Merge pull request #1064 from MBombeck/fix/backup-summary-manifest

Answer the restore preview from counts kept with the backup
This commit is contained in:
Marc-André Bombeck authored and GitHub committed 2026-09-30 23:25:26 +02:00
commit a5d19bfe98
19 files changed
+1605 -101

No files matched your search

@@ -0,0 +1,222 @@
/**
* The preview worked out while a copy is written has to say what the preview
* route used to derive by reading the stored copy: the same counts, the same
* key uses. Held here against that derivation over the same file, fed in
* pieces that split values, strings and multi-byte characters.
*/
import { afterEach, describe, expect, it } from "vitest";
import { _resetCryptoCacheForTests, encrypt } from "@/lib/crypto";
import {
BackupKeyIdCollector,
assessBackupKeys,
} from "@/lib/export/backup-key-ids";
import {
buildBackupPreview,
createBackupPreviewScanner,
storedCopyIdentity,
storedPreviewFor,
} from "@/lib/export/backup-preview";
import { backupPayloadSchema, summarizeBackup } from "@/lib/validations/backup";
const ts = "2026-07-19T07:00:00.000Z";
/** A bytes-column value holding the string codec under key id `v3`. */
const sealedV3 = Buffer.from(`v3.${"QUJD".repeat(20)}`, "utf8").toString(
"base64",
);
const FILE = {
schemaVersion: "2",
exportedAt: ts,
userId: "u1",
measurements: [1, 2, 3].map((i) => ({
id: `m${i}`,
type: "WEIGHT",
value: 70 + i,
unit: "kg",
measuredAt: ts,
source: "MANUAL",
notes: "Grüße, ☕",
...(i === 2 ? { notesEncrypted: sealedV3 } : {}),
})),
intakeEvents: [
{ medication: "Example", scheduledFor: ts, source: "WEB" },
{ medication: "Example", scheduledFor: ts, source: "WEB" },
],
moodEntries: [
{
date: "2026-05-08",
mood: "GUT",
score: 4,
source: "MOODLOG",
loggedAt: ts,
},
],
labResults: [{ analyte: "HbA1c", unit: "%", takenAt: ts, source: "MANUAL" }],
illnessEpisodes: [
{
id: "ep-1",
label: "Cold",
type: "INFECTION",
lifecycle: "ACUTE",
onsetAt: ts,
dayLogs: [
{ date: "2026-04-01", symptoms: [] },
{ date: "2026-04-02", symptoms: [] },
],
},
],
};
async function scan(pieces: Array<string | Uint8Array>) {
const scanner = createBackupPreviewScanner();
for (const piece of pieces) await scanner.feed(piece);
return scanner.finish();
}
function split(text: string, size: number): Uint8Array[] {
const bytes = Buffer.from(text, "utf8");
const out: Uint8Array[] = [];
for (let at = 0; at < bytes.length; at += size) {
out.push(bytes.subarray(at, at + size));
}
return out;
}
describe("createBackupPreviewScanner", () => {
const json = JSON.stringify(FILE);
// What the preview route derived from a stored copy.
const expected = summarizeBackup(backupPayloadSchema.parse(FILE));
const expectedKeys = new BackupKeyIdCollector();
expectedKeys.visit(FILE);
it.each([1, 7, 64, 100_000])(
"counts what the full read counts, in pieces of %i bytes",
async (size) => {
const result = await scan(split(json, size));
expect(result).not.toBeNull();
expect(result!.summary).toEqual(expected);
expect(result!.summary.measurements).toBe(3);
expect(result!.summary.intakeEvents).toBe(2);
expect(result!.summary.moodEntries).toBe(1);
expect(result!.summary.illnessDayLogs).toBe(2);
expect(result!.keys.toStored()).toEqual(expectedKeys.toStored());
expect(result!.keys.keyIds()).toEqual(["v3"]);
},
);
it("takes text pieces the way the weekly writer hands them over", async () => {
const result = await scan([json.slice(0, 50), json.slice(50)]);
expect(result!.summary).toEqual(expected);
});
it("answers null for a file that is not a backup, and never holds the writer", async () => {
expect(await scan(["{ not json", "more", "and more"])).toBeNull();
expect(await scan([JSON.stringify({ schemaVersion: 2 })])).toBeNull();
expect(await scan([json.slice(0, 40)])).toBeNull();
});
});
describe("storedPreviewFor", () => {
const chunked = { data: null, chunkCount: 4, chunkStreamId: "s1" };
const preview = () =>
buildBackupPreview(
storedCopyIdentity(chunked)!,
summarizeBackup(backupPayloadSchema.parse(FILE)),
new BackupKeyIdCollector(),
);
it("reads a preview of the copy the row holds", () => {
const stored = JSON.parse(JSON.stringify(preview()));
expect(storedPreviewFor(stored, chunked)?.summary.measurements).toBe(3);
});
it("ignores a preview of another copy", () => {
expect(
storedPreviewFor(preview(), { ...chunked, chunkStreamId: "s2" }),
).toBeNull();
expect(
storedPreviewFor(preview(), { ...chunked, chunkCount: 5 }),
).toBeNull();
});
it("describes nothing on a row holding both forms, or none", () => {
expect(storedPreviewFor(preview(), { ...chunked, data: "x" })).toBeNull();
expect(
storedPreviewFor(preview(), {
data: null,
chunkCount: null,
chunkStreamId: null,
}),
).toBeNull();
});
it("ignores a malformed preview", () => {
expect(storedPreviewFor({ version: 1 }, chunked)).toBeNull();
expect(storedPreviewFor(null, chunked)).toBeNull();
});
it("names a single stored value by its length and head", () => {
const a = storedCopyIdentity({
data: "v1.aaaa",
chunkCount: null,
chunkStreamId: null,
});
const b = storedCopyIdentity({
data: "v1.aaab",
chunkCount: null,
chunkStreamId: null,
});
expect(a).toMatch(/^value:[0-9a-f]{64}$/);
expect(a).not.toBe(b);
});
});
describe("BackupKeyIdCollector.toStored / fromStored", () => {
const saved = {
key: process.env.ENCRYPTION_KEY,
keys: process.env.ENCRYPTION_KEYS,
active: process.env.ENCRYPTION_ACTIVE_KEY_ID,
};
function useKeys(keys: Record<string, string>, active: string) {
delete process.env.ENCRYPTION_KEY;
process.env.ENCRYPTION_KEYS = JSON.stringify(keys);
process.env.ENCRYPTION_ACTIVE_KEY_ID = active;
_resetCryptoCacheForTests();
}
afterEach(() => {
for (const [name, value] of [
["ENCRYPTION_KEY", saved.key],
["ENCRYPTION_KEYS", saved.keys],
["ENCRYPTION_ACTIVE_KEY_ID", saved.active],
] as const) {
if (value === undefined) delete process.env[name];
else process.env[name] = value;
}
_resetCryptoCacheForTests();
});
it("keeps what the verdict needs, so it can be taken again later", () => {
// Written under `old` and `other` on the host that made the copy.
useKeys({ old: "11".repeat(32), other: "22".repeat(32) }, "old");
const collector = new BackupKeyIdCollector();
collector.add("moodEntries", encrypt("a note"));
process.env.ENCRYPTION_ACTIVE_KEY_ID = "other";
_resetCryptoCacheForTests();
collector.add("labResults", encrypt("another note"));
collector.add("appSettings", encrypt("a setting"));
const stored = JSON.parse(JSON.stringify(collector.toStored()));
const options = { ignoreSections: new Set(["appSettings"]) };
// Later, on a host that dropped `old` and holds different material
// under `other`.
useKeys({ other: "33".repeat(32) }, "other");
const back = BackupKeyIdCollector.fromStored(stored);
const verdict = assessBackupKeys(back, options);
expect(verdict.missing).toEqual(["old"]);
expect(verdict.unreadable).toEqual(["other"]);
expect(verdict).toEqual(assessBackupKeys(collector, options));
expect(back.toStored()).toEqual(collector.toStored());
});
});
+37
View File
@@ -185,6 +185,43 @@ export class BackupKeyIdCollector {
entries(): ReadonlyMap<string, KeyUse> {
return this.uses;
}
/**
* What was collected, as plain data, for a copy's stored preview
* (`backup-preview.ts`): enough to hold it against this host's keys again
* later without reading the copy.
*/
toStored(): StoredKeyUse[] {
return [...this.uses.entries()]
.sort(([a], [b]) => a.localeCompare(b))
.map(([keyId, use]) => ({
keyId,
count: use.count,
sections: [...use.sections].sort(),
sample: use.sample ? { ...use.sample } : null,
}));
}
/** The collector {@link toStored} was taken from. */
static fromStored(stored: readonly StoredKeyUse[]): BackupKeyIdCollector {
const collector = new BackupKeyIdCollector();
for (const entry of stored) {
collector.uses.set(entry.keyId, {
count: entry.count,
sections: new Set(entry.sections),
sample: entry.sample ? { ...entry.sample } : null,
});
}
return collector;
}
}
/** One key id a copy needs, as {@link BackupKeyIdCollector.toStored} keeps it. */
export interface StoredKeyUse {
keyId: string;
count: number;
sections: string[];
sample: { value: string; form: InnerCiphertextForm } | null;
}
export interface BackupKeyAssessment {
+223
View File
@@ -0,0 +1,223 @@
/**
* What the restore preview shows for a stored backup, worked out while the
* copy is written and kept on its row (`DataBackup.preview`).
*
* Why. The preview used to read the whole stored copy inside the request that
* asked for it: open every piece, parse every measurement, validate the rest.
* For an account of 1.8 million readings on slow hardware that took two
* minutes, the dialog gave up long before, and the operator was told the
* contents could not be read (#1031). The copy passes through this process
* once anyway, on its way into the pieces, so the counts are taken there.
*
* What is kept. The `summarizeBackup` counts of the copy, its schema version,
* and the inner key ids with the sections they sit in and one sample value
* each. The verdicts are not kept: whether this release can restore the
* schema version, and whether this server holds the keys the copy needs, can
* both change after the copy was written, so the route takes them again from
* these inputs on every read.
*
* Which copy. A preview names the copy it describes (`copy`), and a reader
* ignores one that names another: a copy replaced by a writer that does not
* know about previews (an older release, after a downgrade) must not be
* described by the numbers of the one it replaced.
*/
import { createHash } from "node:crypto";
import { z } from "zod/v4";
import {
BackupKeyIdCollector,
type StoredKeyUse,
} from "@/lib/export/backup-key-ids";
import { scanBackupJson } from "@/lib/export/backup-json-scan";
import type { StoredBackupRef } from "@/lib/export/stored-backup";
import {
backupPayloadSchema,
summarizeBackup,
type BackupSummary,
} from "@/lib/validations/backup";
export interface BackupPreview {
version: 1;
/** The copy this describes (`storedCopyIdentity`). */
copy: string;
summary: BackupSummary;
keys: StoredKeyUse[];
}
const storedPreviewSchema = z.object({
version: z.literal(1),
copy: z.string().min(1),
summary: z
.object({
schemaVersion: z.string(),
userId: z.string(),
exportedAt: z.string(),
measurements: z.number(),
})
.catchall(z.union([z.string(), z.number()])),
keys: z.array(
z.object({
keyId: z.string(),
count: z.number().int().nonnegative(),
sections: z.array(z.string()),
sample: z
.object({
value: z.string(),
form: z.enum(["string", "bytes-string", "binary"]),
})
.nullable(),
}),
),
});
/**
* How a single stored value is told apart from another: its length and its
* first characters, which carry the key id and the random nonce of the
* envelope. Hashed, so the preview does not repeat the envelope's head.
*/
export function singleValueIdentity(length: number, head: string): string {
return `value:${createHash("sha256").update(`${length}:${head}`).digest("hex")}`;
}
/** How many leading characters of a single stored value the identity reads. */
export const SINGLE_VALUE_IDENTITY_HEAD = 128;
/**
* The name of the copy a row holds now, or null for a row nothing may
* describe: one holding both forms, or neither.
*/
export function storedCopyIdentity(
backup: Pick<StoredBackupRef, "data" | "chunkCount" | "chunkStreamId">,
): string | null {
if (backup.data != null && backup.chunkStreamId != null) return null;
if (backup.chunkStreamId != null) {
return `chunks:${backup.chunkStreamId}:${backup.chunkCount ?? ""}`;
}
if (backup.data != null) {
return singleValueIdentity(
backup.data.length,
backup.data.slice(0, SINGLE_VALUE_IDENTITY_HEAD),
);
}
return null;
}
export function buildBackupPreview(
copy: string,
summary: BackupSummary,
keys: BackupKeyIdCollector,
): BackupPreview {
return { version: 1, copy, summary, keys: keys.toStored() };
}
/**
* The stored preview of a row, when it describes the copy the row holds now;
* null when there is none, it is malformed, or it describes another copy.
*/
export function storedPreviewFor(
stored: unknown,
backup: Pick<StoredBackupRef, "data" | "chunkCount" | "chunkStreamId">,
): BackupPreview | null {
const copy = storedCopyIdentity(backup);
if (copy === null || stored == null) return null;
const parsed = storedPreviewSchema.safeParse(stored);
if (!parsed.success || parsed.data.copy !== copy) return null;
return parsed.data as unknown as BackupPreview;
}
/**
* The sections that grow with a record. Their elements are counted and their
* key ids read as they pass, and never kept, so the scan holds no more of the
* copy than the writer itself does (`full-backup-stream.ts`).
*/
const STREAMED = new Set(["measurements", "intakeEvents", "moodEntries"]);
export interface BackupPreviewScanner {
/**
* Hand over the next piece of the JSON. Resolves once the scan has taken
* it, so a writer never runs more than a piece ahead of the scan.
*/
feed(piece: string | Uint8Array): Promise<void>;
/**
* The counts and key uses, once every piece has been fed; null when the
* JSON was not a backup the schema accepts. Never rejects: a copy is
* stored whether or not its preview could be worked out.
*/
finish(): Promise<{
summary: BackupSummary;
keys: BackupKeyIdCollector;
} | null>;
}
/**
* Work out a preview from a copy's JSON as it is written. The counts are the
* ones the preview route derives from the stored copy (`summarizeBackup` over
* the parsed file), the three bulk sections counted rather than parsed.
*/
export function createBackupPreviewScanner(): BackupPreviewScanner {
const queue: Array<{ chunk: Uint8Array; taken: () => void }> = [];
let ended = false;
let stopped = false;
let wake: (() => void) | null = null;
const signal = () => {
const w = wake;
wake = null;
w?.();
};
async function* pieces(): AsyncGenerator<Uint8Array> {
for (;;) {
const next = queue.shift();
if (next) {
next.taken();
yield next.chunk;
continue;
}
if (ended) return;
await new Promise<void>((resolve) => {
wake = resolve;
});
}
}
const keys = new BackupKeyIdCollector();
const result = scanBackupJson(pieces(), {
streamKeys: STREAMED,
onElement: (key, element) => keys.visit(element, key),
})
.then(({ document, streamedCounts }) => {
keys.visit(document);
const parsed = backupPayloadSchema.safeParse(document);
if (!parsed.success) return null;
const summary: BackupSummary = {
...summarizeBackup(parsed.data),
measurements: streamedCounts.measurements ?? 0,
intakeEvents: streamedCounts.intakeEvents ?? 0,
moodEntries: streamedCounts.moodEntries ?? 0,
};
return { summary, keys };
})
.catch(() => null)
.finally(() => {
// A scan that stopped early takes nothing more: release the writer.
stopped = true;
for (const waiting of queue.splice(0)) waiting.taken();
});
return {
feed(piece) {
if (stopped) return Promise.resolve();
const chunk =
typeof piece === "string" ? Buffer.from(piece, "utf8") : piece;
return new Promise<void>((taken) => {
queue.push({ chunk, taken });
signal();
});
},
finish() {
ended = true;
signal();
return result;
},
};
}
+66 -5
View File
@@ -23,7 +23,7 @@
* this module exists to remove. A model create is planned once for its shape
* and keeps nothing.
*/
import type { PrismaClient } from "@/generated/prisma/client";
import { Prisma, type PrismaClient } from "@/generated/prisma/client";
import { isStreamCiphertext, openStreamDecryptor } from "@/lib/crypto";
import {
@@ -35,6 +35,14 @@ import {
type PackBackupOptions,
} from "@/lib/export/backup-blob";
import { createBackupKeyIdTextScanner } from "@/lib/export/backup-key-ids";
import {
buildBackupPreview,
createBackupPreviewScanner,
SINGLE_VALUE_IDENTITY_HEAD,
singleValueIdentity,
storedCopyIdentity,
type BackupPreview,
} from "@/lib/export/backup-preview";
import {
BackupIntegrityError,
newChunkStreamId,
@@ -118,6 +126,9 @@ export async function storeBackupBlob(
// what is inside them, so this is what tells an operator how long a
// retired key is still needed (`GET /api/admin/encryption/status`).
const keyScanner = createBackupKeyIdTextScanner();
// And what the restore preview shows, from the same pass, so the
// preview never has to read the copy back (`backup-preview.ts`).
const previewScanner = createBackupPreviewScanner();
const { chunks, bytes } = await packBackupChunks(
async (sealed, seq) => {
await tx.dataBackupChunk.create({
@@ -129,10 +140,12 @@ export async function storeBackupBlob(
(write) =>
producer(async (chunk) => {
keyScanner.feed(chunk);
await previewScanner.feed(chunk);
await write(chunk);
}),
options,
);
const scanned = await previewScanner.finish();
// Again at the end: a restore may have been queued while this ran. The
// window left is the few milliseconds to the commit, and a restore
@@ -147,6 +160,18 @@ export async function storeBackupBlob(
chunkStreamId: streamId,
innerKeyIds: keyScanner.keyIds(),
innerKeyIdsRecorded: true,
// Prisma's JSON null clears a preview left by the previous copy.
preview: scanned
? (buildBackupPreview(
storedCopyIdentity({
data: null,
chunkCount: chunks,
chunkStreamId: streamId,
})!,
scanned.summary,
scanned.keys,
) as unknown as Prisma.InputJsonValue)
: Prisma.DbNull,
createdAt: new Date(),
},
});
@@ -156,6 +181,17 @@ export async function storeBackupBlob(
);
}
/** A single value's stored preview, when it describes that value. */
function previewOfSingleValue(
stored: unknown,
len: number,
head: string,
): BackupPreview | null {
if (!stored || typeof stored !== "object") return null;
const preview = stored as BackupPreview;
return preview.copy === singleValueIdentity(len, head) ? preview : null;
}
/**
* The first characters of a single stored value, which carry its key id, or
* null when the row no longer holds one. For the key-rotation scan, which must
@@ -208,10 +244,16 @@ export async function convertSingleValueBackup(
`SET LOCAL idle_in_transaction_session_timeout = '${STORE_IDLE_TIMEOUT}'`,
);
const [row] = await tx.$queryRaw<
Array<{ head: string; len: number; stream_id: string | null }>
Array<{
head: string;
len: number;
stream_id: string | null;
preview: unknown;
}>
>`
SELECT left(data, 128) AS head, length(data)::int AS len,
chunk_stream_id AS stream_id
SELECT left(data, ${SINGLE_VALUE_IDENTITY_HEAD}::int) AS head,
length(data)::int AS len,
chunk_stream_id AS stream_id, preview
FROM data_backups
WHERE id = ${id} AND data IS NOT NULL
FOR UPDATE
@@ -272,9 +314,28 @@ export async function convertSingleValueBackup(
// limit on writing a new one.
{ maxBytes: Number.MAX_SAFE_INTEGER },
);
// The content is the same copy in another form, so a preview of it
// still holds; it is moved over to the new form's name.
const preview = previewOfSingleValue(row.preview, row.len, row.head);
await tx.dataBackup.update({
where: { id },
data: { data: null, chunkCount: chunks, chunkStreamId: streamId },
data: {
data: null,
chunkCount: chunks,
chunkStreamId: streamId,
...(preview
? {
preview: {
...preview,
copy: storedCopyIdentity({
data: null,
chunkCount: chunks,
chunkStreamId: streamId,
})!,
} as unknown as Prisma.InputJsonValue,
}
: {}),
},
});
return "converted";
},
+66 -18
View File
@@ -141,6 +141,71 @@ async function* openedChunks(
}
}
/**
* Refuse a chunked copy whose row does not list as many pieces as are stored:
* one missing, or one this code never wrote. A count, without reading a
* piece; `openStoredBackup` goes on to authenticate every piece.
*/
async function checkStoredBackupPieceCount(
prisma: ChunkReader,
id: string,
streamId: string,
count: number,
): Promise<void> {
if (count < 1) {
throw new BackupIntegrityError("The stored copy lists no pieces.");
}
// Pieces beyond the count would be ignored by the reads, but they are not
// something this code ever writes, so their presence is refused too.
const stored = await prisma.dataBackupChunk.count({
where: { backupId: id },
});
if (stored !== count) {
const now = await prisma.dataBackup.findUnique({
where: { id },
select: { chunkStreamId: true },
});
if (now?.chunkStreamId !== streamId) throw new BackupReplacedError();
throw new BackupIntegrityError(
`The stored copy lists ${count} pieces but ${stored} are stored.`,
);
}
}
/**
* The checks on a stored copy that need no piece read: both forms, a count
* without a stream id or the reverse, no content, and the pieces a chunked
* copy lists against those stored. Throws what `openStoredBackup` would
* throw for the same row. For a reader that answers from what was kept about
* the copy (the restore preview) and must still not describe a copy the
* restore would refuse on sight.
*/
export async function checkStoredBackupShape(
prisma: ChunkReader,
backup: StoredBackupRef,
): Promise<void> {
if (backup.data != null && backup.chunkStreamId != null) {
throw new BackupFormsConflictError();
}
if ((backup.chunkStreamId == null) !== (backup.chunkCount == null)) {
throw new BackupIntegrityError(
"The stored copy lists pieces without saying how many, or the reverse.",
);
}
if (backup.chunkStreamId == null || backup.chunkCount == null) {
if (backup.data == null) {
throw new BackupIntegrityError("The stored copy has no content.");
}
return;
}
await checkStoredBackupPieceCount(
prisma,
backup.id,
backup.chunkStreamId,
backup.chunkCount,
);
}
/**
* Open a stored copy as a source of its JSON bytes, openable as many times as
* the caller needs. Rejects when the copy cannot be read whole: a key that is
@@ -166,24 +231,7 @@ export async function openStoredBackup(
return openBackupBlob(backup.data);
}
const { id, chunkStreamId: streamId, chunkCount: count } = backup;
if (count < 1) {
throw new BackupIntegrityError("The stored copy lists no pieces.");
}
// Pieces beyond the count would be ignored by the reads below, but they are
// not something this code ever writes, so their presence is refused too.
const stored = await prisma.dataBackupChunk.count({
where: { backupId: id },
});
if (stored !== count) {
const now = await prisma.dataBackup.findUnique({
where: { id },
select: { chunkStreamId: true },
});
if (now?.chunkStreamId !== streamId) throw new BackupReplacedError();
throw new BackupIntegrityError(
`The stored copy lists ${count} pieces but ${stored} are stored.`,
);
}
await checkStoredBackupPieceCount(prisma, id, streamId, count);
// The whole copy, checked once before anyone reads a byte of it.
for await (const piece of openedChunks(prisma, id, streamId, count)) {
void piece;