mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-15 15:02:04 +00:00
fix: compress the stored backup before encrypting it
The weekly pass built the whole account in memory, serialised it whole, encrypted the string whole, and handed the result to the database whole. That is four full copies of the record at the same moment — the object graph, the JSON, the base64 ciphertext at 1.33x the JSON, and the copy the driver makes of that parameter. Measured against a seeded account of 445 000 readings, the JSON alone is 242 MB and the run dies of heap exhaustion; under a modest heap it never reaches the insert. The pass was not slow, it could not finish. A health record's JSON is extremely repetitive, so gzip takes that 242 MB to 14 MB and every copy after it shrinks with it. The same run completes in about twenty seconds with a peak heap of 434 MB and a stored row of 25 MB instead of 322 MB — which also matters because the copy lives inside the database it would be needed to restore. Both backup jobs now hand the serialised form back from a frame of its own, so the payload graph is released before the compress step allocates anything. The marker sits inside the ciphertext, so the stored column keeps the one shape `decrypt()` already understands and rows written before this still restore. The off-host object gets the same treatment through the version byte its envelope always carried; version 1 objects still read back, which both the restore drill and the restore script depend on. The obvious saving — dropping soft-deleted readings, four rows in five on a real record — is not taken, and the round-trip test now seeds a tombstone and asserts it makes the journey. A tombstone inside the retention window is what tells a synced client the row is gone and what makes a re-sent sample a no-op; a restore without them brings back every measurement the account ever deleted on the next device sync.
This commit is contained in:
1 parent
b5dd9372a7
commit
d8196d5208
13 files changed
+330
-38
No files matched your search
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* The stored-backup envelope: compress, then encrypt.
|
||||
*
|
||||
* The weekly worker used to hand `encrypt(JSON.stringify(payload))` straight to
|
||||
* Postgres. For an account with a few hundred thousand measurements that means
|
||||
* the object graph, the JSON string, the base64 ciphertext (1.33× the JSON) and
|
||||
* the copy the database driver makes of it all alive at once — well over a
|
||||
* gigabyte for a record whose JSON is a few hundred megabytes. The run died of
|
||||
* memory, not of slow SQL.
|
||||
*
|
||||
* Compressing before the cipher is what makes the tail of that pipeline cheap:
|
||||
* health JSON is extremely repetitive, so everything downstream of the gzip
|
||||
* shrinks by an order of magnitude. The envelope has to stay readable both
|
||||
* ways — rows written before this change are plain `encrypt(json)` and must
|
||||
* keep restoring.
|
||||
*/
|
||||
process.env.ENCRYPTION_KEY ??=
|
||||
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { encrypt } from "@/lib/crypto";
|
||||
import { packBackupBlob, unpackBackupBlob } from "../backup-blob";
|
||||
|
||||
/** A record-shaped string: many rows, few distinct keys. */
|
||||
function sampleJson(rows: number): string {
|
||||
return JSON.stringify({
|
||||
schemaVersion: "2",
|
||||
measurements: Array.from({ length: rows }, (_, i) => ({
|
||||
id: `measurement-${i}`,
|
||||
type: "PULSE",
|
||||
value: 60 + (i % 30),
|
||||
unit: "bpm",
|
||||
measuredAt: new Date(Date.UTC(2026, 0, 1, 0, i % 60)).toISOString(),
|
||||
source: "APPLE_HEALTH",
|
||||
deletedAt: null,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
describe("backup blob envelope", () => {
|
||||
it("round-trips the exact JSON it was handed", () => {
|
||||
const json = sampleJson(200);
|
||||
expect(unpackBackupBlob(packBackupBlob(json))).toBe(json);
|
||||
});
|
||||
|
||||
it("stores a record-shaped payload far smaller than the plaintext", () => {
|
||||
const json = sampleJson(5_000);
|
||||
const packed = packBackupBlob(json);
|
||||
// The old envelope was 1.33× the JSON. Anything near that means the
|
||||
// compression leg is not running.
|
||||
expect(packed.length).toBeLessThan(json.length / 4);
|
||||
});
|
||||
|
||||
it("still reads a row written before the envelope existed", () => {
|
||||
const json = sampleJson(10);
|
||||
expect(unpackBackupBlob(encrypt(json))).toBe(json);
|
||||
});
|
||||
|
||||
it("fails closed on a corrupt blob rather than returning junk", () => {
|
||||
const packed = packBackupBlob(sampleJson(10));
|
||||
const mangled = `${packed.slice(0, -8)}AAAAAAAA`;
|
||||
expect(() => unpackBackupBlob(mangled)).toThrow();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user