HealthLog/tests/integration/backup-preview-memory.test.ts

/**
 * The preview a weekly copy carries is worked out from the counts the writer
 * already produces, not by parsing the copy a second time.
 *
 * What this pins. From v1.39.6 the preview scanner parsed the JSON on its way
 * into the pieces, kept every section that is not one of the three bulk tables
 * (documents with their ciphertext, coach turns, profiles), and validated the
 * whole document at the end. On an account with a document vault that is a
 * second full copy of the record beside the writer's own, inside the weekly
 * job that shares the app's heap.
 *
 * How it measures. Like `backup-streaming-memory.test.ts`: every reading is
 * taken after a forced collection, so what is compared is what the pass
 * HOLDS. The reading at the end of the producer is the one that matters: the
 * writer has released its sections by then, so anything still held belongs
 * to the preview. Run under `--max-old-space-size=524` (the V8 limit of a
 * 1 GB container) for the production ceiling.
 */
import { randomBytes } from "node:crypto";
import v8 from "node:v8";
import vm from "node:vm";

import { afterAll, beforeAll, describe, expect, it } from "vitest";

import { createBackupPreviewCollector } from "@/lib/export/backup-preview";
import type { FullBackupCounts } from "@/lib/export/full-backup-payload";
import { storeBackupBlob } from "@/lib/export/store-backup-blob";
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
import { getPrismaClient, truncateAllTables } from "./setup";

const OWNER_ID = "backup-preview-memory-owner";
const MEASUREMENT_ROWS = 70_000;
const MOOD_ROWS = 3_000;
const DOCUMENTS = 40;
const DOCUMENT_BYTES = 1024 * 1024;

/**
 * What the pass may still hold once the producer is done. Measured: the
 * scanner that kept the sections held about twice the document vault at
 * that point (the parsed sections and the base64 text inside them); the
 * pass without it holds a few megabytes.
 */
const HELD_AT_END_BUDGET_BYTES = 16 * 1024 * 1024;

const prisma = getPrismaClient();

const forceGc = ((): (() => void) => {
  v8.setFlagsFromString("--expose-gc");
  const gc = vm.runInNewContext("gc") as () => void;
  v8.setFlagsFromString("--no-expose-gc");
  return gc;
})();

function liveHeapBytes(): number {
  forceGc();
  forceGc();
  return process.memoryUsage().heapUsed;
}

const mb = (bytes: number): number => Math.round(bytes / 1024 / 1024);

async function seed(): Promise<void> {
  await prisma.user.create({
    data: { id: OWNER_ID, username: "backup-preview-memory" },
  });
  await prisma.$executeRawUnsafe(
    `INSERT INTO measurements (
       id, user_id, type, value, unit, source, measured_at, notes,
       notes_encrypted, external_id, created_at, updated_at, sync_version)
     SELECT
       'pm' || lpad(g::text, 10, '0'), $1, 'PULSE'::measurement_type,
       60 + (g % 40), 'bpm', 'APPLE_HEALTH'::measurement_source,
       timestamp '2019-01-01 00:00:00' + (g * interval '30 seconds'),
       CASE WHEN g % 40 = 0 THEN 'a note recorded with reading ' || g END,
       CASE WHEN g % 25 = 0
            THEN decode(md5(g::text) || md5((g + 1)::text), 'hex') END,
       'preview-' || g,
       timestamp '2019-01-01 00:00:00' + (g * interval '30 seconds'),
       timestamp '2019-01-01 00:00:00' + (g * interval '30 seconds'),
       1
     FROM generate_series(1, ${MEASUREMENT_ROWS}) AS g`,
    OWNER_ID,
  );
  await prisma.$executeRawUnsafe(
    `INSERT INTO mood_entries (
       id, user_id, date, mood, score, source, mood_logged_at, synced_at,
       created_at, updated_at, tz, note, sync_version)
     SELECT
       'pd' || lpad(g::text, 10, '0'), $1,
       to_char(timestamp '2019-01-01' + (g * interval '1 day'), 'YYYY-MM-DD'),
       'okay', 3, 'MOODLOG', timestamp '2019-01-01' + (g * interval '1 day'),
       now(), now(), now(), 'Europe/Berlin', 'a journal line about day ' || g, 1
     FROM generate_series(1, ${MOOD_ROWS}) AS g`,
    OWNER_ID,
  );
  for (let i = 0; i < DOCUMENTS; i++) {
    await prisma.inboundDocument.create({
      data: {
        userId: OWNER_ID,
        mimeType: "application/pdf",
        byteSize: DOCUMENT_BYTES,
        contentEncrypted: new Uint8Array(randomBytes(DOCUMENT_BYTES)),
        contentCodec: "binary2",
        title: `document ${i}`,
      },
    });
  }
}

describe("weekly copy preview under a memory budget", () => {
  beforeAll(async () => {
    await truncateAllTables(prisma);
    await seed();
  }, 240_000);

  afterAll(async () => {
    await truncateAllTables(prisma);
  });

  it("holds nothing of the record once the producer is done", async () => {
    // The backup schema the preview checks each section against is a module
    // loaded once per process (about 12 MB, `backup-summary.ts`), not memory
    // that grows with the record: load it before the baseline, as a running
    // server has long since done.
    await import("@/lib/validations/backup");
    const baseline = liveHeapBytes();
    let peakHeld = 0;
    let heldAtEnd = 0;
    let rawPeak = 0;
    let chunks = 0;
    const poll = setInterval(() => {
      rawPeak = Math.max(rawPeak, process.memoryUsage().heapUsed);
    }, 5);
    // Wired exactly as the weekly job wires it (`backup-handlers.ts`).
    const preview = await createBackupPreviewCollector();
    let counts: FullBackupCounts | undefined;
    try {
      await storeBackupBlob(
        prisma,
        {
          userId: OWNER_ID,
          type: "WEEKLY_AUTO",
          preview: () => (counts ? preview.finish(counts) : null),
        },
        async (write) => {
          counts = await streamFullBackupJson(
            prisma,
            OWNER_ID,
            async (chunk) => {
              await write(chunk);
              if (chunks++ % 40 === 0) {
                peakHeld = Math.max(peakHeld, liveHeapBytes() - baseline);
              }
            },
            { purpose: "disaster-recovery", observe: preview.observe },
          );
          heldAtEnd = liveHeapBytes() - baseline;
          peakHeld = Math.max(peakHeld, heldAtEnd);
        },
      );
    } finally {
      clearInterval(poll);
    }
    rawPeak = Math.max(rawPeak, process.memoryUsage().heapUsed);
    process.stderr.write(
      `[preview-memory] held at end of producer ${mb(heldAtEnd)} MB, ` +
        `peak held ${mb(peakHeld)} MB, raw peak heapUsed ${mb(rawPeak)} MB ` +
        `(baseline ${mb(baseline)} MB), ` +
        `${mb(v8.getHeapStatistics().heap_size_limit)} MB heap limit\n`,
    );

    const row = await prisma.dataBackup.findFirstOrThrow({
      where: { userId: OWNER_ID, type: "WEEKLY_AUTO" },
      select: { preview: true },
    });
    expect(row.preview).toMatchObject({
      summary: {
        measurements: MEASUREMENT_ROWS,
        moodEntries: MOOD_ROWS,
        documents: DOCUMENTS,
      },
    });
    expect(
      heldAtEnd,
      `the pass still held ${mb(heldAtEnd)} MB after the producer finished`,
    ).toBeLessThan(HELD_AT_END_BUDGET_BYTES);
  }, 600_000);
});