perf(backup): take the weekly copy's preview from the writer, not a second parse

Since v1.39.6 the preview scanner parsed the JSON on its way into the
pieces, kept every section but the three bulk tables, and validated the
whole document at the end: a second copy of the record beside the
writer's own, inside the weekly job. On an account with a document vault
that is most of the record.

The writer now shows each section and each bulk row to an observer just
before writing it. The preview collects key uses from that and takes its
counts from the writer's own FullBackupCounts; nothing is parsed or kept.
An upload hands over the summary and key uses it has already validated.
The round trip pins the writer's counts and key uses to a full read of
the copy on an account with a row in every model, and a memory test holds
what the pass keeps once the producer is done.

Measured on 70,000 measurements, 3,000 mood entries and 40 one-megabyte
documents under --max-old-space-size=524: held after the producer
56 MB before, 3 MB after; peak heapUsed 398 MB before, 334 MB after.
This commit is contained in:
Marc-André Bombeck committed 2026-10-04 15:56:56 +02:00
1 parent 5345c8be1e
commit d55e329c48
10 files changed
+426 -144

No files matched your search

@@ -0,0 +1,183 @@
/**
* The preview a weekly copy carries is worked out from the counts the writer
* already produces, not by parsing the copy a second time.
*
* What this pins. From v1.39.6 the preview scanner parsed the JSON on its way
* into the pieces, kept every section that is not one of the three bulk tables
* (documents with their ciphertext, coach turns, profiles), and validated the
* whole document at the end. On an account with a document vault that is a
* second full copy of the record beside the writer's own, inside the weekly
* job that shares the app's heap.
*
* How it measures. Like `backup-streaming-memory.test.ts`: every reading is
* taken after a forced collection, so what is compared is what the pass
* HOLDS. The reading at the end of the producer is the one that matters: the
* writer has released its sections by then, so anything still held belongs
* to the preview. Run under `--max-old-space-size=524` (the V8 limit of a
* 1 GB container) for the production ceiling.
*/
import { randomBytes } from "node:crypto";
import v8 from "node:v8";
import vm from "node:vm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { createBackupPreviewCollector } from "@/lib/export/backup-preview";
import type { FullBackupCounts } from "@/lib/export/full-backup-payload";
import { storeBackupBlob } from "@/lib/export/store-backup-blob";
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
import { getPrismaClient, truncateAllTables } from "./setup";
const OWNER_ID = "backup-preview-memory-owner";
const MEASUREMENT_ROWS = 70_000;
const MOOD_ROWS = 3_000;
const DOCUMENTS = 40;
const DOCUMENT_BYTES = 1024 * 1024;
/**
* What the pass may still hold once the producer is done. Measured: the
* scanner that kept the sections held about twice the document vault at
* that point (the parsed sections and the base64 text inside them); the
* pass without it holds a few megabytes.
*/
const HELD_AT_END_BUDGET_BYTES = 16 * 1024 * 1024;
const prisma = getPrismaClient();
const forceGc = ((): (() => void) => {
v8.setFlagsFromString("--expose-gc");
const gc = vm.runInNewContext("gc") as () => void;
v8.setFlagsFromString("--no-expose-gc");
return gc;
})();
function liveHeapBytes(): number {
forceGc();
forceGc();
return process.memoryUsage().heapUsed;
}
const mb = (bytes: number): number => Math.round(bytes / 1024 / 1024);
async function seed(): Promise<void> {
await prisma.user.create({
data: { id: OWNER_ID, username: "backup-preview-memory" },
});
await prisma.$executeRawUnsafe(
`INSERT INTO measurements (
id, user_id, type, value, unit, source, measured_at, notes,
notes_encrypted, external_id, created_at, updated_at, sync_version)
SELECT
'pm' || lpad(g::text, 10, '0'), $1, 'PULSE'::measurement_type,
60 + (g % 40), 'bpm', 'APPLE_HEALTH'::measurement_source,
timestamp '2019-01-01 00:00:00' + (g * interval '30 seconds'),
CASE WHEN g % 40 = 0 THEN 'a note recorded with reading ' || g END,
CASE WHEN g % 25 = 0
THEN decode(md5(g::text) || md5((g + 1)::text), 'hex') END,
'preview-' || g,
timestamp '2019-01-01 00:00:00' + (g * interval '30 seconds'),
timestamp '2019-01-01 00:00:00' + (g * interval '30 seconds'),
1
FROM generate_series(1, ${MEASUREMENT_ROWS}) AS g`,
OWNER_ID,
);
await prisma.$executeRawUnsafe(
`INSERT INTO mood_entries (
id, user_id, date, mood, score, source, mood_logged_at, synced_at,
created_at, updated_at, tz, note, sync_version)
SELECT
'pd' || lpad(g::text, 10, '0'), $1,
to_char(timestamp '2019-01-01' + (g * interval '1 day'), 'YYYY-MM-DD'),
'okay', 3, 'MOODLOG', timestamp '2019-01-01' + (g * interval '1 day'),
now(), now(), now(), 'Europe/Berlin', 'a journal line about day ' || g, 1
FROM generate_series(1, ${MOOD_ROWS}) AS g`,
OWNER_ID,
);
for (let i = 0; i < DOCUMENTS; i++) {
await prisma.inboundDocument.create({
data: {
userId: OWNER_ID,
mimeType: "application/pdf",
byteSize: DOCUMENT_BYTES,
contentEncrypted: new Uint8Array(randomBytes(DOCUMENT_BYTES)),
contentCodec: "binary2",
title: `document ${i}`,
},
});
}
}
describe("weekly copy preview under a memory budget", () => {
beforeAll(async () => {
await truncateAllTables(prisma);
await seed();
}, 240_000);
afterAll(async () => {
await truncateAllTables(prisma);
});
it("holds nothing of the record once the producer is done", async () => {
const baseline = liveHeapBytes();
let peakHeld = 0;
let heldAtEnd = 0;
let rawPeak = 0;
let chunks = 0;
const poll = setInterval(() => {
rawPeak = Math.max(rawPeak, process.memoryUsage().heapUsed);
}, 5);
// Wired exactly as the weekly job wires it (`backup-handlers.ts`).
const preview = createBackupPreviewCollector();
let counts: FullBackupCounts | undefined;
try {
await storeBackupBlob(
prisma,
{
userId: OWNER_ID,
type: "WEEKLY_AUTO",
preview: () => (counts ? preview.finish(counts) : null),
},
async (write) => {
counts = await streamFullBackupJson(
prisma,
OWNER_ID,
async (chunk) => {
await write(chunk);
if (chunks++ % 40 === 0) {
peakHeld = Math.max(peakHeld, liveHeapBytes() - baseline);
}
},
{ purpose: "disaster-recovery", observe: preview.observe },
);
heldAtEnd = liveHeapBytes() - baseline;
peakHeld = Math.max(peakHeld, heldAtEnd);
},
);
} finally {
clearInterval(poll);
}
rawPeak = Math.max(rawPeak, process.memoryUsage().heapUsed);
process.stderr.write(
`[preview-memory] held at end of producer ${mb(heldAtEnd)} MB, ` +
`peak held ${mb(peakHeld)} MB, raw peak heapUsed ${mb(rawPeak)} MB ` +
`(baseline ${mb(baseline)} MB), ` +
`${mb(v8.getHeapStatistics().heap_size_limit)} MB heap limit\n`,
);
const row = await prisma.dataBackup.findFirstOrThrow({
where: { userId: OWNER_ID, type: "WEEKLY_AUTO" },
select: { preview: true },
});
expect(row.preview).toMatchObject({
summary: {
measurements: MEASUREMENT_ROWS,
moodEntries: MOOD_ROWS,
documents: DOCUMENTS,
},
});
expect(
heldAtEnd,
`the pass still held ${mb(heldAtEnd)} MB after the producer finished`,
).toBeLessThan(HELD_AT_END_BUDGET_BYTES);
}, 600_000);
});
+3 -2
View File
@@ -9,8 +9,9 @@
* and its preview kept; a key dropped after the copy was written is still
* named, because the verdict is taken again at every read.
*
* Mutations that must turn this red: drop the preview scanner from
* `storeBackupBlob` (no preview on the row, the route opens the copy), or
* Mutations that must turn this red: drop the `preview` the weekly job or
* the upload route hands `storeBackupBlob` (no preview on the row, the route
* opens the copy), or
* answer from the stored verdict instead of taking it again (the dropped key
* is not named).
*/
+38 -4
View File
@@ -78,6 +78,8 @@ import {
import { legacyStreamedBlobFrom } from "@/__tests__/helpers/legacy-backup-blob";
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
import { BackupKeyIdCollector } from "@/lib/export/backup-key-ids";
import { createBackupPreviewCollector } from "@/lib/export/backup-preview";
import { parseBackupPayload, summarizeBackup } from "@/lib/validations/backup";
import {
TWO_ENDED_MODELS,
USER_COLUMN_BACKUP_CLASS,
@@ -1492,13 +1494,18 @@ describe("every model the plan claims two-ended survives a real restore", () =>
// below then covers BOTH, since the row that gets restored comes from the
// streamed bytes.
let streamedJson = "";
await streamFullBackupJson(
const previewCollector = createBackupPreviewCollector();
const streamedCounts = await streamFullBackupJson(
prisma,
OWNER_ID,
(chunk) => {
streamedJson += chunk;
},
{ purpose: "disaster-recovery", exportedAt },
{
purpose: "disaster-recovery",
exportedAt,
observe: previewCollector.observe,
},
);
expect(
streamedJson,
@@ -1506,6 +1513,20 @@ describe("every model the plan claims two-ended survives a real restore", () =>
"weekly backup no longer contains what this test proves restorable",
).toBe(JSON.stringify(payload));
// The weekly copy's restore preview is taken from the writer's counts and
// the sections it was shown, never by reading the copy. On an account
// with a row in every model, it has to say what a full read of the copy
// says, count for count and key use for key use.
const fromWriter = previewCollector.finish(streamedCounts);
expect(fromWriter).not.toBeNull();
expect(
fromWriter!.summary,
"the preview taken from the writer's counts disagrees with a full read",
).toEqual(summarizeBackup(parseBackupPayload(streamedJson)));
const readBack = new BackupKeyIdCollector();
readBack.visit(JSON.parse(streamedJson));
expect(fromWriter!.keys.toStored()).toEqual(readBack.toStored());
// The tombstone has to be IN the file. A disaster-recovery payload that
// carries only live rows is smaller and cheaper and wrong: the restore
// wipes the account first, so a deletion that is not in the file is a
@@ -3757,13 +3778,26 @@ describe("every column of every two-ended model survives a real restore", () =>
const exportedAt = new Date("2026-08-01T00:00:00.000Z");
let json = "";
await streamFullBackupJson(
const previewCollector = createBackupPreviewCollector();
const counts = await streamFullBackupJson(
prisma,
OWNER_ID,
(chunk) => {
json += chunk;
},
{ purpose: "disaster-recovery", exportedAt },
{
purpose: "disaster-recovery",
exportedAt,
observe: previewCollector.observe,
},
);
// Every sealed column is filled here: the key uses the preview takes from
// the sections as they are written are the ones a read of the file finds.
const keysReadBack = new BackupKeyIdCollector();
keysReadBack.visit(JSON.parse(json));
expect(keysReadBack.keyIds().length).toBeGreaterThan(0);
expect(previewCollector.finish(counts)!.keys.toStored()).toEqual(
keysReadBack.toStored(),
);
// ── No ciphertext in a portable file ──────────────────────────────────