mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 12:32:43 +00:00
An account of 1.25 million measurements could not export its data or be restored from any backup (#1031). Measured in a 1 GB container: - The full export and the passphrase export built the whole payload and its JSON string in memory; the app ran out of heap and restarted. - The weekly backup's JSON is 662 MB, longer than any string V8 can hold, so `unpackBackupBlob` threw on every read. Restore, download and the restore preview all answered "could not be decrypted". - Upload accepted 10 MB, a fraction of a large export. Every reader and writer of a backup file now streams it: - `scanBackupJson` reads a backup's bytes once, keeps every section but the named bulk arrays, and hands their elements out one at a time. `readStreamedBackup` uses it twice: a first pass that validates each measurement against the element schema and keeps the rest of the file, and a second pass that hands the measurements to the writer in batches. Stored copies and off-host objects are authenticated whole on their compressed ciphertext before any plaintext is read; only the decompression is streamed. - Restore writes the measurements with one `INSERT … FROM unnest` per batch. Prisma caches a query plan per `createMany` row count, and one 1 000-row plan kept about 145 MB alive, which alone put the restore past the heap limit. The transaction budget grows with the measurement count. The awards and ECG sections are handed the ids they reference that were written, not every written id. - The export routes write the file into the response as it is produced. The passphrase export spools its ciphertext to a temporary file because the HLX1 format puts the tag first; the format is unchanged, so existing archives open as before. - Upload accepts the raw body, JSON or gzip, up to the 512 MB request ceiling, checks it as it arrives and stores it in the same pass; the admin page compresses the file in the browser before sending it. Multipart stays for small files. - Download streams the stored JSON; the preview counts measurements from the stream. The monthly restore drill and `scripts/restore-backup.ts` read the off-host object the same way; the script can write `.json.gz`. On the seeded account in a 1 GB container: export 25 s at 389 MB peak, passphrase export 25 s, both uploads 19 s, restore of the weekly backup 94 s at 554 MB peak, and export, upload and restore round-trip to exactly the same rows. The encrypted-column guard now also recognises the raw SQL statement that writes `DataBackup.data`.
156 lines
5.2 KiB
TypeScript
156 lines
5.2 KiB
TypeScript
/**
|
|
* scripts/restore-backup.ts <s3-key> [output-file]
|
|
*
|
|
* Downloads a single off-host backup object, decrypts it with
|
|
* `BACKUP_ENCRYPTION_KEY`, and writes the JSON dump to disk.
|
|
*
|
|
* Wire format (binary):
|
|
* magic(4)="HLBK" || version(1)=0x01 || iv(12) || tag(16) || ciphertext
|
|
*
|
|
* Usage:
|
|
* BACKUP_S3_ENDPOINT=... \
|
|
* BACKUP_S3_BUCKET=... \
|
|
* BACKUP_S3_ACCESS_KEY=... \
|
|
* BACKUP_S3_SECRET_KEY=... \
|
|
* BACKUP_S3_REGION=auto \
|
|
* BACKUP_ENCRYPTION_KEY=<hex64> \
|
|
* pnpm dlx tsx scripts/restore-backup.ts 2026-05-08/user-clx123.json.enc /tmp/restored.json
|
|
*
|
|
* An output name ending in `.gz` writes the JSON compressed, which is the
|
|
* form to hand to the admin upload for a large account: the JSON of 1.25
|
|
* million measurements is 662 MB, the compressed file about a tenth of it.
|
|
* The file is written as a stream in either case and never held whole.
|
|
*/
|
|
import { createWriteStream } from "node:fs";
|
|
import { rename } from "node:fs/promises";
|
|
import { PassThrough, Readable } from "node:stream";
|
|
import { pipeline } from "node:stream/promises";
|
|
import { createGzip } from "node:zlib";
|
|
|
|
import { scanBackupJson } from "@/lib/export/backup-json-scan";
|
|
import {
|
|
loadOffhostConfig,
|
|
getS3Client,
|
|
openBackupObject,
|
|
OffhostBackupNotConfiguredError,
|
|
} from "@/lib/jobs/offhost-backup";
|
|
|
|
/**
|
|
* Schema-validate the decrypted JSON before writing it to disk. The wire
|
|
* format from `runOffhostBackup` always carries `userId` and the four data
|
|
* arrays — anything else means the operator may be looking at a tampered or
|
|
* mis-targeted object. We reject rather than silently writing a body that a
|
|
* later import script would trust.
|
|
*/
|
|
function validateBackupShape(payload: unknown): {
|
|
exportedAt: string;
|
|
userId: string;
|
|
measurements: unknown[];
|
|
medications: unknown[];
|
|
intakeEvents: unknown[];
|
|
moodEntries: unknown[];
|
|
} {
|
|
if (!payload || typeof payload !== "object") {
|
|
throw new Error("backup payload is not a JSON object");
|
|
}
|
|
const p = payload as Record<string, unknown>;
|
|
const required = [
|
|
"exportedAt",
|
|
"userId",
|
|
"measurements",
|
|
"medications",
|
|
"intakeEvents",
|
|
"moodEntries",
|
|
] as const;
|
|
for (const k of required) {
|
|
if (!(k in p)) throw new Error(`backup is missing required field: ${k}`);
|
|
}
|
|
if (typeof p.exportedAt !== "string")
|
|
throw new Error("backup.exportedAt must be string");
|
|
if (typeof p.userId !== "string" || p.userId.length === 0)
|
|
throw new Error("backup.userId must be a non-empty string");
|
|
for (const k of [
|
|
"measurements",
|
|
"medications",
|
|
"intakeEvents",
|
|
"moodEntries",
|
|
] as const) {
|
|
if (!Array.isArray(p[k])) throw new Error(`backup.${k} must be an array`);
|
|
}
|
|
return p as ReturnType<typeof validateBackupShape>;
|
|
}
|
|
|
|
async function main() {
|
|
const key = process.argv[2];
|
|
const out = process.argv[3] ?? "./restored.json";
|
|
// Optional positional arg: --user-id=<id> requires the dump's `userId`
|
|
// field to match. Defends against an operator pointing the script at the
|
|
// wrong S3 key (e.g. another tenant's dump under a similar path) and
|
|
// silently importing the wrong user's data downstream.
|
|
const expectedUserId = process.argv
|
|
.find((a) => a.startsWith("--user-id="))
|
|
?.slice("--user-id=".length);
|
|
if (!key) {
|
|
console.error(
|
|
"Usage: pnpm dlx tsx scripts/restore-backup.ts <s3-key> [output-file] [--user-id=<id>]",
|
|
);
|
|
process.exit(1);
|
|
}
|
|
|
|
const cfg = loadOffhostConfig();
|
|
if (!cfg) {
|
|
throw new OffhostBackupNotConfiguredError(
|
|
"Off-host backup is not configured. Set BACKUP_* env vars.",
|
|
);
|
|
}
|
|
|
|
const s3 = await getS3Client(cfg);
|
|
console.log(
|
|
`Downloading s3://${cfg.bucket}/${key} from ${cfg.endpoint} (region=${cfg.region})`,
|
|
);
|
|
const ciphertext = await s3.getObject(key);
|
|
const source = openBackupObject(ciphertext, cfg.encryptionKey);
|
|
|
|
// Checked first, in one streamed pass that keeps only the small sections,
|
|
// so nothing is written for an object that is not a backup.
|
|
const { document, streamedCounts } = await scanBackupJson(source(), {
|
|
streamKeys: new Set(["measurements", "intakeEvents", "moodEntries"]),
|
|
});
|
|
const body = validateBackupShape({
|
|
...document,
|
|
measurements: [],
|
|
intakeEvents: [],
|
|
moodEntries: [],
|
|
});
|
|
if (expectedUserId && body.userId !== expectedUserId) {
|
|
throw new Error(
|
|
`Refusing to write: backup.userId='${body.userId}' does not match --user-id='${expectedUserId}'`,
|
|
);
|
|
}
|
|
|
|
// The restored file is decrypted PHI in plaintext. Written 0o600 so it is
|
|
// not world-readable on the operator's disk, to a temporary name first so
|
|
// an interrupted run never leaves a truncated file under the real one.
|
|
const partial = `${out}.part`;
|
|
let bytes = 0;
|
|
const counter = new PassThrough();
|
|
counter.on("data", (chunk: Buffer) => {
|
|
bytes += chunk.length;
|
|
});
|
|
await pipeline(
|
|
Readable.from(source()),
|
|
counter,
|
|
...(out.endsWith(".gz") ? [createGzip()] : []),
|
|
createWriteStream(partial, { mode: 0o600 }),
|
|
);
|
|
await rename(partial, out);
|
|
console.log(
|
|
`Restored userId=${body.userId} (${bytes} bytes of JSON, ${streamedCounts.measurements ?? 0} measurements) -> ${out}`,
|
|
);
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error("Restore failed:", err);
|
|
process.exit(2);
|
|
});
|