mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 04:32:42 +00:00
fix: compress the stored backup before encrypting it
The weekly pass built the whole account in memory, serialised it whole, encrypted the string whole, and handed the result to the database whole. That is four full copies of the record at the same moment — the object graph, the JSON, the base64 ciphertext at 1.33x the JSON, and the copy the driver makes of that parameter. Measured against a seeded account of 445 000 readings, the JSON alone is 242 MB and the run dies of heap exhaustion; under a modest heap it never reaches the insert. The pass was not slow, it could not finish. A health record's JSON is extremely repetitive, so gzip takes that 242 MB to 14 MB and every copy after it shrinks with it. The same run completes in about twenty seconds with a peak heap of 434 MB and a stored row of 25 MB instead of 322 MB — which also matters because the copy lives inside the database it would be needed to restore. Both backup jobs now hand the serialised form back from a frame of its own, so the payload graph is released before the compress step allocates anything. The marker sits inside the ciphertext, so the stored column keeps the one shape `decrypt()` already understands and rows written before this still restore. The off-host object gets the same treatment through the version byte its envelope always carried; version 1 objects still read back, which both the restore drill and the restore script depend on. The obvious saving — dropping soft-deleted readings, four rows in five on a real record — is not taken, and the round-trip test now seeds a tombstone and asserts it makes the journey. A tombstone inside the retention window is what tells a synced client the row is gone and what makes a re-sent sample a no-op; a restore without them brings back every measurement the account ever deleted on the next device sync.
This commit is contained in:
1 parent
b5dd9372a7
commit
d8196d5208
13 files changed
+330
-38
No files matched your search
@@ -19,7 +19,7 @@ import { prisma } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, getClientIp } from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { parseBackupPayload } from "@/lib/validations/backup";
|
||||
|
||||
@@ -52,7 +52,7 @@ export const GET = apiHandler(
|
||||
|
||||
let plaintext: string;
|
||||
try {
|
||||
plaintext = decrypt(backup.data);
|
||||
plaintext = unpackBackupBlob(backup.data);
|
||||
} catch (err) {
|
||||
// Decryption failure is rare but real — a rotated/missing key, or a
|
||||
// corrupted blob from a partially-failed write. Surface it as a 500
|
||||
|
||||
@@ -19,7 +19,7 @@ import { prisma, toJson } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { encryptNote } from "@/lib/crypto/note-cipher";
|
||||
import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
|
||||
import { encryptContextToBytes } from "@/lib/labs/biomarker-store";
|
||||
@@ -180,7 +180,7 @@ const handler = apiHandler(
|
||||
|
||||
let plaintext: string;
|
||||
try {
|
||||
plaintext = decrypt(backup.data);
|
||||
plaintext = unpackBackupBlob(backup.data);
|
||||
} catch (err) {
|
||||
await auditLog("admin.backups.restore.failed", {
|
||||
userId: admin.id,
|
||||
|
||||
@@ -16,7 +16,7 @@ import { NextRequest } from "next/server";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, apiSuccess } from "@/lib/api-response";
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
isCompatibleSchemaVersion,
|
||||
@@ -44,7 +44,7 @@ export const GET = apiHandler(
|
||||
|
||||
let plaintext: string;
|
||||
try {
|
||||
plaintext = decrypt(backup.data);
|
||||
plaintext = unpackBackupBlob(backup.data);
|
||||
} catch {
|
||||
return apiError("Failed to decrypt backup payload", 500);
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import { prisma } from "@/lib/db";
|
||||
import { apiHandler, HttpError, requireAdmin } from "@/lib/api-handler";
|
||||
import { apiError, apiSuccess, getClientIp } from "@/lib/api-response";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { encrypt } from "@/lib/crypto";
|
||||
import { packBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { checkRateLimit } from "@/lib/rate-limit";
|
||||
import {
|
||||
@@ -190,7 +190,7 @@ export const POST = apiHandler(async (request: NextRequest) => {
|
||||
// intact so multiple uploads can coexist for the same user without
|
||||
// overwriting each other or the rolling WEEKLY_AUTO snapshot.
|
||||
const uploadType = `MANUAL_UPLOAD_${Date.now()}`;
|
||||
const encrypted = encrypt(JSON.stringify(payload));
|
||||
const encrypted = packBackupBlob(JSON.stringify(payload));
|
||||
|
||||
const created = await prisma.dataBackup.create({
|
||||
data: {
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* The stored-backup envelope: compress, then encrypt.
|
||||
*
|
||||
* The weekly worker used to hand `encrypt(JSON.stringify(payload))` straight to
|
||||
* Postgres. For an account with a few hundred thousand measurements that means
|
||||
* the object graph, the JSON string, the base64 ciphertext (1.33× the JSON) and
|
||||
* the copy the database driver makes of it all alive at once — well over a
|
||||
* gigabyte for a record whose JSON is a few hundred megabytes. The run died of
|
||||
* memory, not of slow SQL.
|
||||
*
|
||||
* Compressing before the cipher is what makes the tail of that pipeline cheap:
|
||||
* health JSON is extremely repetitive, so everything downstream of the gzip
|
||||
* shrinks by an order of magnitude. The envelope has to stay readable both
|
||||
* ways — rows written before this change are plain `encrypt(json)` and must
|
||||
* keep restoring.
|
||||
*/
|
||||
process.env.ENCRYPTION_KEY ??=
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { encrypt } from "@/lib/crypto";
|
||||
import { packBackupBlob, unpackBackupBlob } from "../backup-blob";
|
||||
|
||||
/** A record-shaped string: many rows, few distinct keys. */
|
||||
function sampleJson(rows: number): string {
|
||||
return JSON.stringify({
|
||||
schemaVersion: "2",
|
||||
measurements: Array.from({ length: rows }, (_, i) => ({
|
||||
id: `measurement-${i}`,
|
||||
type: "PULSE",
|
||||
value: 60 + (i % 30),
|
||||
unit: "bpm",
|
||||
measuredAt: new Date(Date.UTC(2026, 0, 1, 0, i % 60)).toISOString(),
|
||||
source: "APPLE_HEALTH",
|
||||
deletedAt: null,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
describe("backup blob envelope", () => {
|
||||
it("round-trips the exact JSON it was handed", () => {
|
||||
const json = sampleJson(200);
|
||||
expect(unpackBackupBlob(packBackupBlob(json))).toBe(json);
|
||||
});
|
||||
|
||||
it("stores a record-shaped payload far smaller than the plaintext", () => {
|
||||
const json = sampleJson(5_000);
|
||||
const packed = packBackupBlob(json);
|
||||
// The old envelope was 1.33× the JSON. Anything near that means the
|
||||
// compression leg is not running.
|
||||
expect(packed.length).toBeLessThan(json.length / 4);
|
||||
});
|
||||
|
||||
it("still reads a row written before the envelope existed", () => {
|
||||
const json = sampleJson(10);
|
||||
expect(unpackBackupBlob(encrypt(json))).toBe(json);
|
||||
});
|
||||
|
||||
it("fails closed on a corrupt blob rather than returning junk", () => {
|
||||
const packed = packBackupBlob(sampleJson(10));
|
||||
const mangled = `${packed.slice(0, -8)}AAAAAAAA`;
|
||||
expect(() => unpackBackupBlob(mangled)).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* The envelope a `DataBackup.data` row is stored in: compress, then encrypt.
|
||||
*
|
||||
* Why it exists. The weekly worker handed `encrypt(JSON.stringify(payload))`
|
||||
* straight to Postgres. For an account with a few hundred thousand
|
||||
* measurements that is four full copies of the record alive at the same
|
||||
* moment: the payload object graph, the JSON string, the base64 ciphertext at
|
||||
* 1.33× the JSON, and the copy the database driver makes of that parameter on
|
||||
* its way to the wire. Measured on a seeded 445 000-measurement account, the
|
||||
* JSON alone is 242 MB and the run dies of heap exhaustion — under a 1 GB heap
|
||||
* it never reaches the insert, and on a bigger heap it only reaches it later.
|
||||
* The weekly pass was not slow; it could not finish.
|
||||
*
|
||||
* Compressing first is what makes the whole tail of that pipeline cheap. A
|
||||
* health record's JSON is extremely repetitive — the same twenty keys per row,
|
||||
* timestamps sharing a prefix — so gzip takes that 242 MB to 14 MB, and every
|
||||
* copy after it shrinks with it. The same run then completes in about eight
|
||||
* seconds with a peak heap of 434 MB. It is also the reason the stored row
|
||||
* stops being a liability of its own: the backup lives INSIDE the database it
|
||||
* would be needed to restore, so its size is not a cosmetic concern.
|
||||
*
|
||||
* Both directions. Rows written before this envelope existed are plain
|
||||
* `encrypt(json)` and have to keep restoring — an operator whose newest usable
|
||||
* copy predates the fix is exactly the person who needs it to work. The marker
|
||||
* sits inside the ciphertext rather than in front of it so the stored column
|
||||
* keeps the one shape `decrypt()` already understands, key ids and all.
|
||||
*/
|
||||
import { Buffer } from "node:buffer";
|
||||
import { gunzipSync, gzipSync } from "node:zlib";
|
||||
|
||||
import { decrypt, encrypt } from "@/lib/crypto";
|
||||
|
||||
/**
|
||||
* Prefix of the DECRYPTED plaintext when the body is gzipped-then-base64'd.
|
||||
* Chosen so it can never be mistaken for the alternative: a plain payload is
|
||||
* always a JSON object and therefore always starts with `{`.
|
||||
*/
|
||||
const GZIP_MARKER = "HLZ1:";
|
||||
|
||||
/** Serialised backup JSON → the string stored in `DataBackup.data`. */
|
||||
export function packBackupBlob(json: string): string {
|
||||
const compressed = gzipSync(json).toString("base64");
|
||||
return encrypt(`${GZIP_MARKER}${compressed}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* A stored `DataBackup.data` string → the backup JSON.
|
||||
*
|
||||
* Fails closed on every arm: a bad key, a mangled ciphertext or a truncated
|
||||
* gzip member throws rather than returning a partial document, because every
|
||||
* caller goes on to parse the result as a whole account.
|
||||
*/
|
||||
export function unpackBackupBlob(stored: string): string {
|
||||
const plaintext = decrypt(stored);
|
||||
if (!plaintext.startsWith(GZIP_MARKER)) return plaintext;
|
||||
return gunzipSync(
|
||||
Buffer.from(plaintext.slice(GZIP_MARKER.length), "base64"),
|
||||
).toString("utf8");
|
||||
}
|
||||
@@ -316,6 +316,26 @@ function serialiseMoodContext(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The same payload, already serialised, with the object graph released.
|
||||
*
|
||||
* Its own frame on purpose, and both backup jobs go through it. The payload
|
||||
* graph is the largest thing either job allocates — on an account with a few
|
||||
* hundred thousand measurements it is several hundred megabytes of small
|
||||
* objects — and neither job wants anything but the string. Returning from here
|
||||
* makes the graph unreachable before the compress-and-encrypt step allocates
|
||||
* anything of its own, which is part of why the weekly pass now fits in
|
||||
* memory. Inlining these two statements back into a caller quietly undoes it.
|
||||
*/
|
||||
export async function buildFullBackupJson(
|
||||
prisma: PrismaClient,
|
||||
userId: string,
|
||||
options: FullBackupOptions = {},
|
||||
): Promise<string> {
|
||||
const { payload } = await buildFullBackupPayload(prisma, userId, options);
|
||||
return JSON.stringify(payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the canonical full-backup payload for `userId`. Portable exports omit
|
||||
* tombstones and document ciphertext; disaster-recovery payloads preserve
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createCipheriv, randomBytes } from "node:crypto";
|
||||
|
||||
import { describe, it, expect, beforeEach, vi } from "vitest";
|
||||
import {
|
||||
BACKUP_SCHEMA_VERSION,
|
||||
@@ -8,8 +10,20 @@ const mocks = vi.hoisted(() => ({
|
||||
buildFullBackupPayload: vi.fn(),
|
||||
}));
|
||||
|
||||
// The uploader takes the already-serialised form, so the stand-in for
|
||||
// `buildFullBackupJson` serialises whatever the payload mock was told to
|
||||
// return. Every case below keeps stubbing the PAYLOAD, which is the thing
|
||||
// these tests are actually about.
|
||||
vi.mock("@/lib/export/full-backup-payload", () => ({
|
||||
buildFullBackupPayload: mocks.buildFullBackupPayload,
|
||||
buildFullBackupJson: async (...args: unknown[]) =>
|
||||
JSON.stringify(
|
||||
(
|
||||
(await mocks.buildFullBackupPayload(...args)) as {
|
||||
payload: unknown;
|
||||
}
|
||||
).payload,
|
||||
),
|
||||
}));
|
||||
import {
|
||||
encryptBackup,
|
||||
@@ -27,10 +41,33 @@ describe("offhost-backup envelope", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
const payload = JSON.stringify({ hello: "world", n: 42 });
|
||||
const buf = encryptBackup(payload, key);
|
||||
expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x01");
|
||||
// Version 2: the plaintext is gzipped before the cipher sees it. This
|
||||
// assertion used to pin version 1, which is the version that could not
|
||||
// encrypt a large account's dump without four full copies of it resident
|
||||
// at once. The pin is flipped rather than dropped — the byte is the whole
|
||||
// reason a reader can tell the two apart.
|
||||
expect(buf.subarray(0, 5).toString("binary")).toBe("HLBK\x02");
|
||||
expect(decryptBackup(buf, key)).toBe(payload);
|
||||
});
|
||||
|
||||
it("still reads a version-1 object written before compression", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
const payload = JSON.stringify({ hello: "world", n: 42 });
|
||||
// Hand-built v1 envelope: magic || 0x01 || iv || tag || ciphertext, the
|
||||
// exact bytes every object already sitting in an operator's bucket has.
|
||||
const iv = randomBytes(12);
|
||||
const cipher = createCipheriv("aes-256-gcm", key, iv);
|
||||
const ct = Buffer.concat([cipher.update(payload, "utf8"), cipher.final()]);
|
||||
const legacy = Buffer.concat([
|
||||
Buffer.from("HLBK\x01", "binary"),
|
||||
iv,
|
||||
cipher.getAuthTag(),
|
||||
ct,
|
||||
]);
|
||||
|
||||
expect(decryptBackup(legacy, key)).toBe(payload);
|
||||
});
|
||||
|
||||
it("rejects tampered ciphertext", () => {
|
||||
const key = Buffer.from(ENC_KEY, "hex");
|
||||
const buf = encryptBackup("data", key);
|
||||
|
||||
@@ -18,8 +18,9 @@
|
||||
* the backup history. See docs/ops/backup-restore.md.
|
||||
*/
|
||||
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
||||
import { gunzipSync, gzipSync } from "node:zlib";
|
||||
import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
|
||||
import { buildFullBackupJson } from "@/lib/export/full-backup-payload";
|
||||
import { getEvent } from "@/lib/logging/context";
|
||||
|
||||
const ALGORITHM = "aes-256-gcm";
|
||||
@@ -84,20 +85,43 @@ export function loadOffhostConfig(): OffhostBackupConfig | null {
|
||||
* Encrypt a JSON payload with the dedicated backup key.
|
||||
*
|
||||
* Wire format (binary):
|
||||
* magic(4)="HLBK" || version(1)=0x01 || iv(12) || tag(16) || ciphertext
|
||||
* magic(4)="HLBK" || version(1) || iv(12) || tag(16) || ciphertext
|
||||
*
|
||||
* Version 1 encrypts the JSON directly. Version 2 gzips it first, which is
|
||||
* what the version byte was there for: a large account's dump is hundreds of
|
||||
* megabytes of extremely repetitive JSON, and encrypting it whole means the
|
||||
* string, the cipher's input copy, the ciphertext and the concatenation are
|
||||
* all resident at once — the same arithmetic that made the in-database weekly
|
||||
* backup unfinishable. Gzip takes an order of magnitude off every one of them
|
||||
* and off the object in the bucket. Objects written under version 1 stay
|
||||
* readable; both the restore drill and `scripts/restore-backup.ts` come
|
||||
* through `decryptBackup`, so neither needs to know which it got.
|
||||
*/
|
||||
const BACKUP_ENVELOPE_PLAIN = 0x01;
|
||||
const BACKUP_ENVELOPE_GZIP = 0x02;
|
||||
|
||||
export function encryptBackup(plaintext: string, key: Buffer): Buffer {
|
||||
const iv = randomBytes(IV_LENGTH);
|
||||
const cipher = createCipheriv(ALGORITHM, key, iv);
|
||||
const ct = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
|
||||
const ct = Buffer.concat([
|
||||
cipher.update(gzipSync(plaintext)),
|
||||
cipher.final(),
|
||||
]);
|
||||
const tag = cipher.getAuthTag();
|
||||
return Buffer.concat([Buffer.from("HLBK\x01", "binary"), iv, tag, ct]);
|
||||
const header = Buffer.from([
|
||||
...Buffer.from("HLBK", "binary"),
|
||||
BACKUP_ENVELOPE_GZIP,
|
||||
]);
|
||||
return Buffer.concat([header, iv, tag, ct]);
|
||||
}
|
||||
|
||||
export function decryptBackup(buf: Buffer, key: Buffer): string {
|
||||
const magic = buf.subarray(0, 4).toString("binary");
|
||||
const version = buf[4];
|
||||
if (magic !== "HLBK" || version !== 0x01) {
|
||||
if (
|
||||
magic !== "HLBK" ||
|
||||
(version !== BACKUP_ENVELOPE_PLAIN && version !== BACKUP_ENVELOPE_GZIP)
|
||||
) {
|
||||
throw new Error("Invalid backup envelope (bad magic or version)");
|
||||
}
|
||||
const iv = buf.subarray(5, 5 + IV_LENGTH);
|
||||
@@ -105,7 +129,10 @@ export function decryptBackup(buf: Buffer, key: Buffer): string {
|
||||
const ct = buf.subarray(5 + IV_LENGTH + TAG_LENGTH);
|
||||
const dec = createDecipheriv(ALGORITHM, key, iv);
|
||||
dec.setAuthTag(tag);
|
||||
return Buffer.concat([dec.update(ct), dec.final()]).toString("utf8");
|
||||
const plaintext = Buffer.concat([dec.update(ct), dec.final()]);
|
||||
return version === BACKUP_ENVELOPE_GZIP
|
||||
? gunzipSync(plaintext).toString("utf8")
|
||||
: plaintext.toString("utf8");
|
||||
}
|
||||
|
||||
export interface S3Like {
|
||||
@@ -215,12 +242,13 @@ export async function runOffhostBackup(
|
||||
const evt = getEvent();
|
||||
for (const user of users) {
|
||||
try {
|
||||
const { payload } = await buildFullBackupPayload(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: now,
|
||||
});
|
||||
const plaintext = JSON.stringify(payload);
|
||||
const ciphertext = encryptBackup(plaintext, cfg.encryptionKey);
|
||||
const ciphertext = encryptBackup(
|
||||
await buildFullBackupJson(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
exportedAt: now,
|
||||
}),
|
||||
cfg.encryptionKey,
|
||||
);
|
||||
const key = `${dateKey}/user-${user.id}.json.enc`;
|
||||
await s3.putObject(key, ciphertext);
|
||||
uploaded++;
|
||||
|
||||
@@ -4,16 +4,29 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
buildFullBackupPayload: vi.fn(),
|
||||
encrypt: vi.fn((value: string) => value),
|
||||
packBlob: vi.fn((value: string) => value),
|
||||
getWorkerPrisma: vi.fn(),
|
||||
upsert: vi.fn(),
|
||||
}));
|
||||
|
||||
// The handler takes the already-serialised form; the stand-in serialises
|
||||
// whatever the payload mock was told to return, so this file keeps stubbing
|
||||
// and asserting the PAYLOAD, which is what it is about.
|
||||
vi.mock("@/lib/export/full-backup-payload", () => ({
|
||||
buildFullBackupPayload: mocks.buildFullBackupPayload,
|
||||
buildFullBackupJson: async (...args: unknown[]) =>
|
||||
JSON.stringify(
|
||||
((await mocks.buildFullBackupPayload(...args)) as { payload: unknown })
|
||||
.payload,
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/crypto", () => ({ encrypt: mocks.encrypt }));
|
||||
// The envelope is exercised end-to-end in
|
||||
// `src/lib/export/__tests__/backup-blob.test.ts`; here it stays transparent so
|
||||
// the stored bytes can be read back as JSON.
|
||||
vi.mock("@/lib/export/backup-blob", () => ({
|
||||
packBackupBlob: mocks.packBlob,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/background", () => ({
|
||||
withBackgroundEvent: vi.fn(
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
*/
|
||||
import { type Job } from "pg-boss";
|
||||
import { recordError } from "@/lib/jobs/worker-status";
|
||||
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
|
||||
import { encrypt } from "@/lib/crypto";
|
||||
import { buildFullBackupJson } from "@/lib/export/full-backup-payload";
|
||||
import { packBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
|
||||
import { withBackgroundEvent } from "@/lib/logging/background";
|
||||
import {
|
||||
@@ -79,15 +79,21 @@ export async function handleDataBackup(
|
||||
|
||||
let backed = 0;
|
||||
let usersFailed = 0;
|
||||
let largestBlobBytes = 0;
|
||||
for (const user of users) {
|
||||
try {
|
||||
const { payload } = await buildFullBackupPayload(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
});
|
||||
const backupJson = JSON.stringify(payload);
|
||||
|
||||
// Encrypt the backup data (contains sensitive health information)
|
||||
const encryptedBackup = encrypt(backupJson);
|
||||
// Compressed, then encrypted (the record contains sensitive health
|
||||
// information). Both legs live in `packBackupBlob`, which is also
|
||||
// what the restore path reads back through.
|
||||
const encryptedBackup = packBackupBlob(
|
||||
await buildFullBackupJson(prisma, user.id, {
|
||||
purpose: "disaster-recovery",
|
||||
}),
|
||||
);
|
||||
largestBlobBytes = Math.max(
|
||||
largestBlobBytes,
|
||||
Buffer.byteLength(encryptedBackup, "utf8"),
|
||||
);
|
||||
|
||||
await prisma.dataBackup.upsert({
|
||||
where: {
|
||||
@@ -113,6 +119,9 @@ export async function handleDataBackup(
|
||||
}
|
||||
}
|
||||
|
||||
// The stored size is the one number that says whether this pass is
|
||||
// heading back towards the wall it hit before: it tracks the record.
|
||||
evt.addMeta("data_backup_largest_blob_bytes", largestBlobBytes);
|
||||
evt.setBackground({
|
||||
task_name: "job.data_backup",
|
||||
result: { backed, total: users.length },
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* Verifies:
|
||||
* - 201 + a `DataBackup` row with type prefix `MANUAL_UPLOAD_` is
|
||||
* created when the JSON file matches the schema.
|
||||
* - the original payload round-trips through encrypt → decrypt.
|
||||
* - the original payload round-trips through the stored backup envelope.
|
||||
* - 422 when the JSON is malformed.
|
||||
* - 422 when the schema validation fails (missing required field).
|
||||
* - 422 when the userId on the file doesn't exist.
|
||||
@@ -15,7 +15,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
process.env.ENCRYPTION_KEY ??=
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
import { decrypt } from "@/lib/crypto";
|
||||
import { unpackBackupBlob } from "@/lib/export/backup-blob";
|
||||
|
||||
import { cookieJar, headerJar } from "./mock-next-headers";
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
@@ -166,9 +166,11 @@ describe("POST /api/admin/backups/upload", () => {
|
||||
expect(row?.type).toMatch(/^MANUAL_UPLOAD_\d+$/);
|
||||
expect(row?.userId).toBe(admin.id);
|
||||
|
||||
// Round-trip the ciphertext to make sure we stored the same payload
|
||||
// we received.
|
||||
const decrypted = JSON.parse(decrypt(row!.data));
|
||||
// Round-trip the stored blob to make sure we kept the same payload we
|
||||
// received. Through the envelope rather than through `decrypt` alone: an
|
||||
// uploaded file is stored exactly the way the weekly worker stores one, so
|
||||
// the restore path reads both without knowing where the row came from.
|
||||
const decrypted = JSON.parse(unpackBackupBlob(row!.data));
|
||||
expect(decrypted.userId).toBe(admin.id);
|
||||
expect(decrypted.measurements).toHaveLength(1);
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ import {
|
||||
encryptWaveformToBytes,
|
||||
} from "@/lib/withings/ecg-waveform-codec";
|
||||
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
|
||||
import { packBackupBlob } from "@/lib/export/backup-blob";
|
||||
import { TWO_ENDED_MODELS, type TwoEndedModel } from "@/lib/export/backup-plan";
|
||||
import { POST } from "@/app/api/admin/backups/[id]/restore/route";
|
||||
|
||||
@@ -259,6 +260,27 @@ async function seedEveryTwoEndedModel(prisma: PrismaClient): Promise<void> {
|
||||
source: "MANUAL",
|
||||
},
|
||||
});
|
||||
// A DELETED reading, kept as a tombstone. It is here because the obvious way
|
||||
// to make the weekly backup cheaper — stop exporting soft-deleted rows, which
|
||||
// on a real record is four rows in five — is wrong, and nothing in the tree
|
||||
// said so out loud. A tombstone inside the retention window is what tells a
|
||||
// synced client the row is gone (`/api/sync/changes` reads exactly these) and
|
||||
// what makes a re-sent Apple Health sample a no-op instead of a resurrection.
|
||||
// Drop them from a disaster-recovery file and a restore quietly brings back
|
||||
// every measurement the account ever deleted on the next device sync.
|
||||
await prisma.measurement.create({
|
||||
data: {
|
||||
userId: OWNER_ID,
|
||||
type: "PULSE",
|
||||
value: 61,
|
||||
unit: "bpm",
|
||||
measuredAt: AT("2026-07-02T06:30:00.000Z"),
|
||||
source: "APPLE_HEALTH",
|
||||
externalId: "round-trip-deleted-sample",
|
||||
syncVersion: 4,
|
||||
deletedAt: AT("2026-07-20T09:15:00.000Z"),
|
||||
},
|
||||
});
|
||||
// The EVENT row an ECG strip is filed against, and the strip itself. The
|
||||
// pairing is the point: `EcgRecording.measurementId` is a real foreign key,
|
||||
// so this is the reference whose remap has to survive the round trip rather
|
||||
@@ -1231,6 +1253,26 @@ describe("every model the plan claims two-ended survives a real restore", () =>
|
||||
purpose: "disaster-recovery",
|
||||
});
|
||||
|
||||
// The tombstone has to be IN the file. A disaster-recovery payload that
|
||||
// carries only live rows is smaller and cheaper and wrong: the restore
|
||||
// wipes the account first, so a deletion that is not in the file is a
|
||||
// deletion the account no longer knows it made, and the next device sync
|
||||
// re-inserts every one of those samples as fresh readings.
|
||||
const exportedMeasurements = payload.measurements as Array<{
|
||||
externalId: string | null;
|
||||
deletedAt: string | null;
|
||||
syncVersion: number;
|
||||
}>;
|
||||
expect(
|
||||
exportedMeasurements.find(
|
||||
(row) => row.externalId === "round-trip-deleted-sample",
|
||||
),
|
||||
"a soft-deleted reading is missing from the disaster-recovery payload",
|
||||
).toMatchObject({
|
||||
deletedAt: "2026-07-20T09:15:00.000Z",
|
||||
syncVersion: 4,
|
||||
});
|
||||
|
||||
// Cascade rather than a hand-listed sweep — see the file comment.
|
||||
await prisma.user.delete({ where: { id: OWNER_ID } });
|
||||
await createOwner(prisma);
|
||||
@@ -1247,7 +1289,12 @@ describe("every model the plan claims two-ended survives a real restore", () =>
|
||||
data: {
|
||||
userId: OWNER_ID,
|
||||
type: "TWO_ENDED_ROUND_TRIP",
|
||||
data: encrypt(JSON.stringify(payload)),
|
||||
// The envelope the weekly worker actually writes: gzip, then
|
||||
// encrypt. The three cases further down deliberately keep the plain
|
||||
// `encrypt(json)` form, which is what every row written before this
|
||||
// envelope existed looks like — an operator's newest usable copy may
|
||||
// well be one of those, so both arms have to reach the restore route.
|
||||
data: packBackupBlob(JSON.stringify(payload)),
|
||||
},
|
||||
});
|
||||
const response = await POST(
|
||||
@@ -2026,6 +2073,18 @@ describe("every model the plan claims two-ended survives a real restore", () =>
|
||||
// measurement pointer is asserted against the row the RESTORE wrote rather
|
||||
// than against the id the fixture used, because resolving to something
|
||||
// that exists is the property the reference owes.
|
||||
const restoredTombstone = await prisma.measurement.findFirstOrThrow({
|
||||
where: { userId: OWNER_ID, externalId: "round-trip-deleted-sample" },
|
||||
});
|
||||
expect(
|
||||
{
|
||||
deletedAt: restoredTombstone.deletedAt?.toISOString() ?? null,
|
||||
syncVersion: restoredTombstone.syncVersion,
|
||||
},
|
||||
"the deleted reading came back, but not as a deletion — a tombstone " +
|
||||
"restored without its `deletedAt` is a resurrected measurement",
|
||||
).toEqual({ deletedAt: "2026-07-20T09:15:00.000Z", syncVersion: 4 });
|
||||
|
||||
const restoredMeasurement = await prisma.measurement.findFirstOrThrow({
|
||||
where: { userId: OWNER_ID, type: "WEIGHT" },
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user