fix(backup): store a backup in sealed pieces instead of one value

A stored backup was one value in data_backups.data. v1.39.1 wrote it a
piece at a time but still joined the pieces into that one value, and one
value has to pass through the app whole, so a copy was capped at a fifth of
the heap: 105 MB in the default 1 GB container. An account with 1.75
million readings is past that and could not be backed up at all (#1031).

A copy is now kept as ordered pieces of about a megabyte in the new
data_backup_chunks table (migration 0352). Each piece is sealed on its own
with AES-256-GCM, and its sealed header names the copy (a random stream id
kept on the row), its position and whether it is the last. A reader checks
every piece before it releases a byte, so a piece that is missing, moved,
altered, taken from another copy, or a copy cut short is refused with
backup.payload.undecryptable before a restore deletes anything, and the
second read during the restore checks each piece again.

The write is one transaction, so the previous copy stays whole and readable
until the new one commits. Pieces are written through the model API: a raw
query with a large parameter is kept alive by Prisma's query plan cache,
which left 160 MB of heap behind per hundred one-megabyte inserts.

Restore, preview, download, the restore job and the admin list read both
forms; copies written before this change keep their single value and still
restore. The heap-derived limit is gone. The one limit left is on storage,
BACKUP_MAX_STORED_MB (default 2048), and its message names the setting. Key
rotation covers the pieces, the wipe and account deletion take them with
their backup, and the backup plan lists the table as not carried.

Measured on an account of 2.6 million readings in a 1 GB container with web
and worker in one process: backup 87 s, 97 MiB in 96 pieces, preview 20 s,
download 25 s, restore 264 s with every reading back exactly, peak 567 MiB,
no restart.
This commit is contained in:
Marc-André Bombeck committed 2026-09-26 01:02:48 +02:00
1 parent b0f9bbe530
commit 1e09f30eff
39 files changed
+1700 -546

No files matched your search

@@ -15,7 +15,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
import { unpackBackupBlob } from "@/lib/export/backup-blob";
import { readStoredBackup } from "./stored-backup-read";
import { cookieJar, headerJar } from "./mock-next-headers";
import { getPrismaClient, truncateAllTables } from "./setup";
@@ -170,7 +170,7 @@ describe("POST /api/admin/backups/upload", () => {
// received. Through the envelope rather than through `decrypt` alone: an
// uploaded file is stored exactly the way the weekly worker stores one, so
// the restore path reads both without knowing where the row came from.
const decrypted = JSON.parse(unpackBackupBlob(row!.data));
const decrypted = JSON.parse(await readStoredBackup(prisma, row!.id));
expect(decrypted.userId).toBe(admin.id);
expect(decrypted.measurements).toHaveLength(1);
@@ -0,0 +1,567 @@
/**
* #1031 — a stored backup kept in pieces.
*
* v1.39.1 wrote a backup a piece at a time but still joined the pieces into
* one value in `data_backups.data`, and one value has to pass through the app
* whole. So a copy could be at most a fifth of the heap, 105 MB in a 1 GB
* container, and an account with 1.75 million readings could not be backed up
* at all. A copy is now a run of sealed pieces in `data_backup_chunks`, and
* this file pins what that has to keep true:
*
* - the weekly copy restores exactly, and the size it lists is what it takes;
* - a piece dropped, moved, altered or cut off the end is refused, and the
* refusal comes before the restore deletes anything;
* - a copy stored in one value by an earlier release still restores;
* - a run that fails halfway leaves the previous copy whole and readable;
* - replacing, wiping or deleting a copy takes its pieces with it.
*
* The pieces are made small here (`chunkBytes`) so a record that fits a test
* run still spans many of them.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
import { cookieJar, headerJar } from "./mock-next-headers";
import { getPrismaClient, truncateAllTables } from "./setup";
import { readStoredBackup } from "./stored-backup-read";
import { legacyStreamedBlobFrom } from "@/__tests__/helpers/legacy-backup-blob";
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
import { restoreBackup } from "@/lib/export/restore-backup";
import { storeBackupBlob } from "@/lib/export/store-backup-blob";
import { STORED_BACKUP_SELECT } from "@/lib/export/stored-backup";
vi.mock("next/headers", async () => {
const { cookieJar, headerJar } = await import("./mock-next-headers");
return {
headers: vi.fn(async () => ({
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
})),
cookies: vi.fn(async () => ({
get: (name: string) => {
const value = cookieJar.get(name);
return value ? { name, value } : undefined;
},
set: (name: string, value: string) => {
cookieJar.set(name, value);
},
delete: (name: string) => {
cookieJar.delete(name);
},
})),
};
});
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
const COUNT = 12_000;
/** Small pieces, so a test-sized record spans many of them. */
const SMALL = { chunkBytes: 4 * 1024 };
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
cookieJar.clear();
headerJar.clear();
});
async function seedAccount() {
const prisma = getPrismaClient();
const user = await prisma.user.create({
data: {
username: "chunked-owner",
email: "chunked-owner@example.test",
role: "ADMIN",
},
});
const session = await prisma.session.create({
data: { userId: user.id, expiresAt: new Date(Date.now() + 600_000) },
});
cookieJar.set("healthlog_session", session.id);
const start = Date.UTC(2024, 0, 1);
await prisma.measurement.createMany({
data: Array.from({ length: COUNT }, (_, i) => ({
id: `m-${String(i).padStart(5, "0")}`,
userId: user.id,
type: i % 2 === 0 ? ("PULSE" as const) : ("WEIGHT" as const),
value: 50 + (i % 40) + 0.5,
unit: i % 2 === 0 ? "bpm" : "kg",
source: "APPLE_HEALTH" as const,
measuredAt: new Date(start + i * 600_000),
externalId: `hk-${i}`,
})),
});
return user;
}
async function measurementsOf(userId: string) {
return getPrismaClient().measurement.findMany({
where: { userId },
orderBy: { id: "asc" },
omit: { createdAt: true, updatedAt: true },
});
}
/** The weekly job's own write, with small pieces. */
async function weeklyCopy(
userId: string,
options: { chunkBytes: number; maxBytes?: number } = SMALL,
) {
const prisma = getPrismaClient();
return storeBackupBlob(
prisma,
{ userId, type: "WEEKLY_AUTO" },
(write) =>
streamFullBackupJson(prisma, userId, write, {
purpose: "disaster-recovery",
}),
options,
);
}
async function restoreRoute(id: string) {
const { POST } = await import("./restore-job-driver");
return POST(
new Request(`http://localhost/api/admin/backups/${id}/restore`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ confirm: "RESTORE" }),
}) as never,
{ params: Promise.resolve({ id }) },
);
}
/** The restore itself, without the route's admission check in front of it. */
async function restoreDirect(id: string, actorUserId: string) {
const backup = await getPrismaClient().dataBackup.findUniqueOrThrow({
where: { id },
select: STORED_BACKUP_SELECT,
});
return restoreBackup({
backup,
actorUserId,
ipAddress: null,
restoreInstanceSettings: false,
});
}
async function chunksOf(backupId: string) {
return getPrismaClient().dataBackupChunk.findMany({
where: { backupId },
orderBy: { seq: "asc" },
});
}
describe("a backup stored in pieces", () => {
it("restores every measurement exactly, from many pieces", async () => {
const user = await seedAccount();
const before = await measurementsOf(user.id);
const { id, chunks } = await weeklyCopy(user.id);
expect(chunks).toBeGreaterThan(5);
const prisma = getPrismaClient();
const row = await prisma.dataBackup.findUniqueOrThrow({ where: { id } });
expect(row.data).toBeNull();
expect(row.chunkCount).toBe(chunks);
expect(await chunksOf(id)).toHaveLength(chunks);
await prisma.measurement.deleteMany({ where: { userId: user.id } });
const res = await restoreRoute(id);
expect(res.status, JSON.stringify(await res.clone().json())).toBe(200);
expect(await measurementsOf(user.id)).toEqual(before);
});
it("previews and downloads the copy the job stored", async () => {
const user = await seedAccount();
const { id } = await weeklyCopy(user.id);
const params = { params: Promise.resolve({ id }) };
const { GET: summary } =
await import("@/app/api/admin/backups/[id]/summary/route");
const preview = await summary(new Request("http://x") as never, params);
expect(preview.status).toBe(200);
const { data } = (await preview.json()) as {
data: { summary: { measurements: number } };
};
expect(data.summary.measurements).toBe(COUNT);
const { GET: download } =
await import("@/app/api/admin/backups/[id]/download/route");
const file = await download(new Request("http://x") as never, params);
expect(file.status).toBe(200);
const text = await file.text();
expect(Number(file.headers.get("content-length"))).toBe(
Buffer.byteLength(text),
);
expect(text).toBe(await readStoredBackup(getPrismaClient(), id));
expect(JSON.parse(text).measurements).toHaveLength(COUNT);
});
it("lists the size the copy takes in the database, in either form", async () => {
const user = await seedAccount();
const prisma = getPrismaClient();
const { id, bytes } = await weeklyCopy(user.id);
const legacy = await prisma.dataBackup.create({
data: {
userId: user.id,
type: "MANUAL_UPLOAD_1",
data: await legacyStreamedBlobFrom((write) =>
streamFullBackupJson(prisma, user.id, write, {
purpose: "disaster-recovery",
}),
),
},
});
const { GET } = await import("@/app/api/admin/backups/route");
const res = await GET();
const { data } = (await res.json()) as {
data: { rows: Array<{ id: string; sizeBytes: number }> };
};
const sizes = new Map(data.rows.map((b) => [b.id, b.sizeBytes]));
expect(sizes.get(id)).toBe(bytes);
expect(sizes.get(legacy.id)).toBe(legacy.data!.length);
});
});
describe("a stored copy whose pieces do not add up is refused before anything is deleted", () => {
type Tamper = (backupId: string) => Promise<void>;
const prisma = () => getPrismaClient();
const cases: Array<[string, Tamper]> = [
[
"a piece dropped from the middle",
async (backupId) => {
await prisma().dataBackupChunk.deleteMany({
where: { backupId, seq: 2 },
});
},
],
[
"two pieces swapped",
async (backupId) => {
const [a, b] = [
await prisma().dataBackupChunk.findUniqueOrThrow({
where: { backupId_seq: { backupId, seq: 1 } },
}),
await prisma().dataBackupChunk.findUniqueOrThrow({
where: { backupId_seq: { backupId, seq: 2 } },
}),
];
await prisma().dataBackupChunk.update({
where: { id: a.id },
data: { data: b.data },
});
await prisma().dataBackupChunk.update({
where: { id: b.id },
data: { data: a.data },
});
},
],
[
"one byte of a piece altered",
async (backupId) => {
const piece = await prisma().dataBackupChunk.findUniqueOrThrow({
where: { backupId_seq: { backupId, seq: 3 } },
});
const altered = new Uint8Array(piece.data);
altered[altered.byteLength - 20]! ^= 0x01;
await prisma().dataBackupChunk.update({
where: { id: piece.id },
data: { data: altered },
});
},
],
[
"the last piece cut off, and the count lowered to match",
async (backupId) => {
const row = await prisma().dataBackup.findUniqueOrThrow({
where: { id: backupId },
});
const last = row.chunkCount! - 1;
await prisma().dataBackupChunk.deleteMany({
where: { backupId, seq: last },
});
await prisma().dataBackup.update({
where: { id: backupId },
data: { chunkCount: last },
});
},
],
];
for (const [name, tamper] of cases) {
it(`${name}: the route refuses, the job refuses, nothing changes`, async () => {
const user = await seedAccount();
const { id } = await weeklyCopy(user.id);
await tamper(id);
const before = await measurementsOf(user.id);
expect(before).toHaveLength(COUNT);
const res = await restoreRoute(id);
expect(res.status).toBe(422);
const body = (await res.json()) as { meta?: { errorCode?: string } };
expect(body.meta?.errorCode).toBe("backup.payload.undecryptable");
// The job refuses on its own too, for a copy tampered with after the
// route admitted it.
const outcome = await restoreDirect(id, user.id);
expect(outcome.ok).toBe(false);
if (!outcome.ok)
expect(outcome.code).toBe("backup.payload.undecryptable");
expect(await measurementsOf(user.id)).toEqual(before);
});
}
it("a piece from another account's copy put in its place", async () => {
const user = await seedAccount();
const other = await getPrismaClient().user.create({
data: { username: "other", email: "other@example.test" },
});
await getPrismaClient().measurement.createMany({
data: Array.from({ length: COUNT }, (_, i) => ({
userId: other.id,
type: "PULSE" as const,
value: 70,
unit: "bpm",
source: "APPLE_HEALTH" as const,
measuredAt: new Date(Date.UTC(2025, 0, 1) + i * 60_000),
})),
});
const mine = await weeklyCopy(user.id);
const theirs = await weeklyCopy(other.id);
const foreign = await getPrismaClient().dataBackupChunk.findUniqueOrThrow({
where: { backupId_seq: { backupId: theirs.id, seq: 1 } },
});
await getPrismaClient().dataBackupChunk.update({
where: { backupId_seq: { backupId: mine.id, seq: 1 } },
data: { data: foreign.data },
});
const before = await measurementsOf(user.id);
expect((await restoreRoute(mine.id)).status).toBe(422);
const outcome = await restoreDirect(mine.id, user.id);
expect(outcome.ok).toBe(false);
expect(await measurementsOf(user.id)).toEqual(before);
});
it("the preview and the download refuse the same copy with the same answer", async () => {
const user = await seedAccount();
const { id } = await weeklyCopy(user.id);
await getPrismaClient().dataBackupChunk.deleteMany({
where: { backupId: id, seq: 1 },
});
const params = { params: Promise.resolve({ id }) };
const { GET: summary } =
await import("@/app/api/admin/backups/[id]/summary/route");
const { GET: download } =
await import("@/app/api/admin/backups/[id]/download/route");
for (const res of [
await summary(new Request("http://x") as never, params),
await download(new Request("http://x") as never, params),
]) {
expect(res.status).toBe(422);
const body = (await res.json()) as { meta?: { errorCode?: string } };
expect(body.meta?.errorCode).toBe("backup.payload.undecryptable");
}
});
});
describe("an uploaded backup file", () => {
it("is stored in pieces under the account the file names, not the admin who sent it", async () => {
const admin = await seedAccount();
const prisma = getPrismaClient();
const owner = await prisma.user.create({
data: { username: "file-owner", email: "file-owner@example.test" },
});
await prisma.measurement.create({
data: {
userId: owner.id,
type: "WEIGHT",
value: 71.5,
unit: "kg",
source: "MANUAL",
measuredAt: new Date(Date.UTC(2025, 5, 1)),
},
});
const weekly = await weeklyCopy(owner.id);
const file = await readStoredBackup(prisma, weekly.id);
const { POST: upload } =
await import("@/app/api/admin/backups/upload/route");
const res = await upload(
new Request("http://localhost/api/admin/backups/upload", {
method: "POST",
headers: { "content-type": "application/json" },
body: file,
}) as never,
);
expect(res.status).toBe(201);
const { data } = (await res.json()) as { data: { id: string } };
const row = await prisma.dataBackup.findUniqueOrThrow({
where: { id: data.id },
});
expect(row.userId).toBe(owner.id);
expect(row.userId).not.toBe(admin.id);
expect(row.data).toBeNull();
expect(await readStoredBackup(prisma, data.id)).toBe(file);
});
});
describe("a copy stored in one value by an earlier release", () => {
it("still restores exactly", async () => {
const user = await seedAccount();
const prisma = getPrismaClient();
const before = await measurementsOf(user.id);
const legacy = await prisma.dataBackup.create({
data: {
userId: user.id,
type: "WEEKLY_AUTO",
data: await legacyStreamedBlobFrom((write) =>
streamFullBackupJson(prisma, user.id, write, {
purpose: "disaster-recovery",
}),
),
},
});
await prisma.measurement.deleteMany({ where: { userId: user.id } });
expect((await restoreRoute(legacy.id)).status).toBe(200);
expect(await measurementsOf(user.id)).toEqual(before);
});
it("is replaced in pieces by the next weekly run", async () => {
const user = await seedAccount();
const prisma = getPrismaClient();
const legacy = await prisma.dataBackup.create({
data: {
userId: user.id,
type: "WEEKLY_AUTO",
data: await legacyStreamedBlobFrom(async (write) => {
await write("{}");
}),
},
});
const { id, chunks } = await weeklyCopy(user.id);
expect(id).toBe(legacy.id);
const row = await prisma.dataBackup.findUniqueOrThrow({ where: { id } });
expect(row.data).toBeNull();
expect(row.chunkCount).toBe(chunks);
});
});
describe("a restore queued against a copy that is then replaced", () => {
it("is refused as backup_changed, and nothing is deleted", async () => {
const user = await seedAccount();
const { id } = await weeklyCopy(user.id);
const { acceptingBoss } = await import("./restore-job-driver");
const { POST: queueRestore } =
await import("@/app/api/admin/backups/[id]/restore/route");
const { getGlobalBoss, setGlobalBoss } =
await import("@/lib/jobs/boss-instance");
const { readBackupRestoreJob, runBackupRestoreJob } =
await import("@/lib/jobs/backup-restore");
const previous = getGlobalBoss();
setGlobalBoss(acceptingBoss);
let queued: Response;
try {
queued = await queueRestore(
new Request(`http://localhost/api/admin/backups/${id}/restore`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ confirm: "RESTORE" }),
}) as never,
{ params: Promise.resolve({ id }) },
);
} finally {
setGlobalBoss(previous);
}
expect(queued.status).toBe(202);
const { data } = (await queued.json()) as { data: { jobId: string } };
// The next weekly run writes a new copy into the same row.
await weeklyCopy(user.id);
const before = await measurementsOf(user.id);
await runBackupRestoreJob(data.jobId).catch(() => undefined);
const job = await readBackupRestoreJob(data.jobId);
expect(job?.failure?.code).toBe("backup_changed");
expect(await measurementsOf(user.id)).toEqual(before);
});
});
describe("writing and removing copies", () => {
it("keeps the previous copy whole when a run fails after writing pieces", async () => {
const user = await seedAccount();
const prisma = getPrismaClient();
const first = await weeklyCopy(user.id);
const before = await prisma.dataBackup.findUniqueOrThrow({
where: { id: first.id },
});
const text = await readStoredBackup(prisma, first.id);
let written = 0;
await expect(
storeBackupBlob(
prisma,
{ userId: user.id, type: "WEEKLY_AUTO" },
async (write) => {
await streamFullBackupJson(
prisma,
user.id,
async (chunk) => {
await write(chunk);
written += chunk.length;
if (written > 200 * 1024) throw new Error("read failed halfway");
},
{ purpose: "disaster-recovery" },
);
},
SMALL,
),
).rejects.toThrow("read failed halfway");
const after = await prisma.dataBackup.findUniqueOrThrow({
where: { id: first.id },
});
expect(after.chunkStreamId).toBe(before.chunkStreamId);
expect(after.chunkCount).toBe(before.chunkCount);
expect(after.createdAt).toEqual(before.createdAt);
expect(await chunksOf(first.id)).toHaveLength(first.chunks);
expect(await readStoredBackup(prisma, first.id)).toBe(text);
});
it("stops at the stored-copy limit and keeps the previous copy", async () => {
const user = await seedAccount();
const first = await weeklyCopy(user.id);
await expect(
weeklyCopy(user.id, { ...SMALL, maxBytes: 16 * 1024 }),
).rejects.toThrow(/BACKUP_MAX_STORED_MB/);
expect(await chunksOf(first.id)).toHaveLength(first.chunks);
});
it("replacing a copy leaves only the new copy's pieces", async () => {
const user = await seedAccount();
const prisma = getPrismaClient();
const first = await weeklyCopy(user.id);
const second = await weeklyCopy(user.id);
expect(second.id).toBe(first.id);
expect(await prisma.dataBackupChunk.count()).toBe(second.chunks);
});
it("wiping the account's data and deleting the account take the pieces with them", async () => {
const user = await seedAccount();
const prisma = getPrismaClient();
await weeklyCopy(user.id);
expect(await prisma.dataBackupChunk.count()).toBeGreaterThan(0);
// The wipe deletes `DataBackup` by owner; the pieces follow by cascade.
await prisma.dataBackup.deleteMany({ where: { userId: user.id } });
expect(await prisma.dataBackupChunk.count()).toBe(0);
await weeklyCopy(user.id);
await prisma.user.delete({ where: { id: user.id } });
expect(await prisma.dataBackupChunk.count()).toBe(0);
});
});
+2 -2
View File
@@ -66,7 +66,7 @@ import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
import { UNREADABLE_EXPORT_MARKER } from "@/lib/export/unreadable-marker";
import { decryptNoteFromBytes } from "@/lib/labs/store";
import { decryptContextFromBytes } from "@/lib/labs/biomarker-store";
import { packBackupBlobStreaming } from "@/lib/export/backup-blob";
import { legacyStreamedBlobFrom } from "@/__tests__/helpers/legacy-backup-blob";
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
import { TWO_ENDED_MODELS, type TwoEndedModel } from "@/lib/export/backup-plan";
import { POST } from "./restore-job-driver";
@@ -1356,7 +1356,7 @@ describe("every model the plan claims two-ended survives a real restore", () =>
// form, which is what every row written before any of this existed
// looks like — an operator's newest usable copy may well be one of
// those, so every arm has to reach the restore route.
data: await packBackupBlobStreaming(async (write) => {
data: await legacyStreamedBlobFrom(async (write) => {
await write(streamedJson);
}),
},
@@ -32,11 +32,9 @@ import vm from "node:vm";
import { beforeAll, afterAll, describe, expect, it } from "vitest";
import {
BackupBlobTooLargeError,
packBackupBlobStreaming,
unpackBackupBlob,
} from "@/lib/export/backup-blob";
import { BackupBlobTooLargeError } from "@/lib/export/backup-blob";
import { storeBackupBlob } from "@/lib/export/store-backup-blob";
import { readStoredBackup } from "./stored-backup-read";
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
import { getPrismaClient, truncateAllTables } from "./setup";
@@ -233,26 +231,30 @@ describe("weekly backup under a memory budget", () => {
let peakHeld = 0;
let chunks = 0;
const blob = await packBackupBlobStreaming(async (write) => {
const counts = await streamFullBackupJson(
prisma,
OWNER_ID,
async (chunk) => {
await write(chunk);
// Sampled rather than continuous: a forced collection per chunk
// would dominate the runtime, and one in forty still lands inside
// every phase of the walk.
if (chunks++ % 40 === 0) {
peakHeld = Math.max(peakHeld, liveHeapBytes() - baseline);
}
},
// The purpose the weekly job asks for: tombstones and ciphertext ride
// verbatim, which is the arm that has to fit in memory.
{ purpose: "disaster-recovery" },
);
expect(counts.measurements).toBe(MEASUREMENT_ROWS);
expect(counts.moodEntries).toBe(MOOD_ROWS);
});
const { id: blob } = await storeBackupBlob(
prisma,
{ userId: OWNER_ID, type: "WEEKLY_AUTO" },
async (write) => {
const counts = await streamFullBackupJson(
prisma,
OWNER_ID,
async (chunk) => {
await write(chunk);
// Sampled rather than continuous: a forced collection per chunk
// would dominate the runtime, and one in forty still lands inside
// every phase of the walk.
if (chunks++ % 40 === 0) {
peakHeld = Math.max(peakHeld, liveHeapBytes() - baseline);
}
},
// The purpose the weekly job asks for: tombstones and ciphertext ride
// verbatim, which is the arm that has to fit in memory.
{ purpose: "disaster-recovery" },
);
expect(counts.measurements).toBe(MEASUREMENT_ROWS);
expect(counts.moodEntries).toBe(MOOD_ROWS);
},
);
peakHeld = Math.max(peakHeld, liveHeapBytes() - baseline);
reportPeak("large record", peakHeld, liveHeapBytes());
@@ -263,7 +265,7 @@ describe("weekly backup under a memory budget", () => {
).toBeLessThan(STREAM_BUDGET_BYTES);
// A blob that writes but does not read is worse than none.
const restored = JSON.parse(unpackBackupBlob(blob)) as {
const restored = JSON.parse(await readStoredBackup(prisma, blob)) as {
measurements: Array<{ deletedAt: string | null }>;
moodEntries: unknown[];
};
@@ -307,21 +309,25 @@ describe("weekly backup under a memory budget", () => {
let peak = baseline;
let jsonBytes = 0;
const blob = await packBackupBlobStreaming(async (write) => {
const counts = await streamFullBackupJson(
prisma,
TINY_OWNER_ID,
async (chunk) => {
jsonBytes += chunk.length;
peak = Math.max(peak, process.memoryUsage().heapUsed);
await write(chunk);
},
{ purpose: "disaster-recovery" },
);
expect(counts.measurements).toBe(1);
});
const { id: blob } = await storeBackupBlob(
prisma,
{ userId: TINY_OWNER_ID, type: "WEEKLY_AUTO" },
async (write) => {
const counts = await streamFullBackupJson(
prisma,
TINY_OWNER_ID,
async (chunk) => {
jsonBytes += chunk.length;
peak = Math.max(peak, process.memoryUsage().heapUsed);
await write(chunk);
},
{ purpose: "disaster-recovery" },
);
expect(counts.measurements).toBe(1);
},
);
const restored = JSON.parse(unpackBackupBlob(blob)) as {
const restored = JSON.parse(await readStoredBackup(prisma, blob)) as {
measurements: unknown[];
};
expect(restored.measurements).toHaveLength(1);
@@ -336,14 +342,14 @@ describe("weekly backup under a memory budget", () => {
reportPeak("tiny record", peak - baseline, liveHeapBytes());
}, 180_000);
it("fails as a job rather than as a process when the stored copy does not fit", async () => {
// The bound that remains is on the one copy the pipeline cannot stream
// away: the blob itself, counted in bytes it produced. An account whose
// backup outgrows what this process can hold as a single value is now a
// failed backup for that account instead of a restart for every account
// on the host.
it("fails as a job rather than as a process when the stored copy passes its limit", async () => {
// The one bound left is a storage limit on the stored copy, counted in
// bytes it produced. An account past it is a failed backup for that
// account, not a restart for every account on the host.
await expect(
packBackupBlobStreaming(
storeBackupBlob(
prisma,
{ userId: OWNER_ID, type: "WEEKLY_AUTO" },
(write) =>
streamFullBackupJson(prisma, OWNER_ID, write, {
purpose: "disaster-recovery",
@@ -40,8 +40,11 @@ import {
type CorpusClient,
} from "@/lib/crypto/encryption-corpus";
import { packBackupBlob, unpackBackupBlob } from "@/lib/export/backup-blob";
import { extractKeyIdFromBytes } from "@/lib/crypto";
import { storeBackupBlob } from "@/lib/export/store-backup-blob";
import { legacyStreamedBlob } from "@/__tests__/helpers/legacy-backup-blob";
import { getPrismaClient, truncateAllTables } from "./setup";
import { readStoredBackup } from "./stored-backup-read";
const TEST_USER_ID = "user-rotation-blobs";
const BACKUP_JSON = JSON.stringify({
@@ -117,8 +120,8 @@ describe("key rotation over the non-suffixed ciphertext columns", () => {
for (const id of [gzipped.id, plainEnvelope.id]) {
const row = await prisma.dataBackup.findUniqueOrThrow({ where: { id } });
expect(extractKeyId(row.data)).toBe("v2");
expect(unpackBackupBlob(row.data)).toBe(BACKUP_JSON);
expect(extractKeyId(row.data!)).toBe("v2");
expect(unpackBackupBlob(row.data!)).toBe(BACKUP_JSON);
}
// Idempotent: a second pass finds nothing left to do.
@@ -151,7 +154,7 @@ describe("key rotation over the non-suffixed ciphertext columns", () => {
const row = await prisma.dataBackup.findUniqueOrThrow({
where: { id: backup.id },
});
expect(unpackBackupBlob(row.data)).toBe(BACKUP_JSON);
expect(unpackBackupBlob(row.data!)).toBe(BACKUP_JSON);
});
it("rotates the single stream v1.39.1 stored, and it stays readable", async () => {
@@ -185,6 +188,50 @@ describe("key rotation over the non-suffixed ciphertext columns", () => {
expect(unpackBackupBlob(after.data!)).toBe(BACKUP_JSON);
});
it("rotates every piece of a chunked copy, which reads back once the old key is gone", async () => {
const prisma = getPrismaClient();
process.env.ENCRYPTION_ACTIVE_KEY_ID = "v1";
_resetCryptoCacheForTests();
const { id, chunks } = await storeBackupBlob(
prisma,
{ userId: TEST_USER_ID, type: "WEEKLY_AUTO" },
async (write) => {
await write('{"rows":[');
for (let i = 0; i < 20_000; i++) {
await write(`${i ? "," : ""}{"i":${i},"r":${Math.random()}}`);
}
await write("]}");
},
{ chunkBytes: 4 * 1024 },
);
expect(chunks).toBeGreaterThan(3);
const expected = await readStoredBackup(prisma, id);
process.env.ENCRYPTION_ACTIVE_KEY_ID = "v2";
_resetCryptoCacheForTests();
const client = {
dataBackupChunk: prisma.dataBackupChunk,
} as unknown as CorpusClient;
const result = await rotateColumn(
client,
column("DataBackupChunk", "data"),
);
expect(result).toMatchObject({
scanned: chunks,
rotated: chunks,
errors: 0,
});
const again = await rotateColumn(client, column("DataBackupChunk", "data"));
expect(again.rotated).toBe(0);
process.env.ENCRYPTION_KEYS = JSON.stringify({ v2: "2".repeat(64) });
_resetCryptoCacheForTests();
for (const piece of await prisma.dataBackupChunk.findMany()) {
expect(extractKeyIdFromBytes(Buffer.from(piece.data))).toBe("v2");
}
expect(await readStoredBackup(prisma, id)).toBe(expected);
});
it("drops an unreadable idempotency row instead of failing the run", async () => {
const prisma = getPrismaClient();
const seed = async (key: string, body: string) =>
@@ -223,6 +270,7 @@ describe("key rotation over the non-suffixed ciphertext columns", () => {
it("keeps both columns in the registry the script and the job read", () => {
const keys = ENCRYPTED_COLUMNS.map(encryptedColumnKey);
expect(keys).toContain("DataBackup.data");
expect(keys).toContain("DataBackupChunk.data");
expect(keys).toContain("IdempotencyKey.responseBody");
});
});
+14 -6
View File
@@ -22,8 +22,8 @@ import { beforeEach, describe, expect, it } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention";
import { consolidateDailyMean } from "@/lib/measurements/consolidate-daily-mean";
import { unpackBackupBlob } from "@/lib/export/backup-blob";
import { storeBackupBlob } from "@/lib/export/store-backup-blob";
import { readStoredBackup } from "./stored-backup-read";
const TZ = "Europe/Berlin";
const RUN_A = "user-1031-stopped";
@@ -214,7 +214,7 @@ describe("storeBackupBlob (#1031)", () => {
it("stores a copy that reads back as exactly the JSON produced", async () => {
const prisma = getPrismaClient();
const { bytes } = await storeBackupBlob(
const { id, bytes, chunks } = await storeBackupBlob(
prisma,
{ userId: RUN_A, type: "WEEKLY_AUTO" },
producer(20_000),
@@ -223,8 +223,15 @@ describe("storeBackupBlob (#1031)", () => {
const row = await prisma.dataBackup.findUniqueOrThrow({
where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } },
});
expect(row.data.length).toBe(bytes);
expect(unpackBackupBlob(row.data)).toBe(expectedJson(20_000));
expect(row.id).toBe(id);
expect(row.data).toBeNull();
expect(row.chunkCount).toBe(chunks);
const pieces = await prisma.dataBackupChunk.findMany({
where: { backupId: id },
});
expect(pieces).toHaveLength(chunks);
expect(pieces.reduce((sum, p) => sum + p.data.byteLength, 0)).toBe(bytes);
expect(await readStoredBackup(prisma, id)).toBe(expectedJson(20_000));
});
it("keeps the previous copy when a run fails halfway, and the next run still stores", async () => {
@@ -249,8 +256,9 @@ describe("storeBackupBlob (#1031)", () => {
const after = await prisma.dataBackup.findUniqueOrThrow({
where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } },
});
expect(after.data).toBe(before.data);
expect(after.chunkStreamId).toBe(before.chunkStreamId);
expect(after.createdAt.getTime()).toBe(before.createdAt.getTime());
expect(await readStoredBackup(prisma, after.id)).toBe(expectedJson(10));
// The temporary table went with the failed transaction.
await storeBackupBlob(
@@ -261,6 +269,6 @@ describe("storeBackupBlob (#1031)", () => {
const replaced = await prisma.dataBackup.findUniqueOrThrow({
where: { userId_type: { userId: RUN_A, type: "WEEKLY_AUTO" } },
});
expect(unpackBackupBlob(replaced.data)).toBe(expectedJson(30));
expect(await readStoredBackup(prisma, replaced.id)).toBe(expectedJson(30));
});
});
+23
View File
@@ -0,0 +1,23 @@
/**
* Read a stored backup back as its JSON text, whichever form it was stored
* in, through the same opener the restore, download and preview use.
*/
import type { PrismaClient } from "@/generated/prisma/client";
import {
openStoredBackup,
STORED_BACKUP_SELECT,
} from "@/lib/export/stored-backup";
export async function readStoredBackup(
prisma: PrismaClient,
id: string,
): Promise<string> {
const backup = await prisma.dataBackup.findUniqueOrThrow({
where: { id },
select: STORED_BACKUP_SELECT,
});
const source = await openStoredBackup(prisma, backup);
const parts: Buffer[] = [];
for await (const chunk of source()) parts.push(Buffer.from(chunk));
return Buffer.concat(parts).toString("utf8");
}