From 3fcae204f738fbd14444a21a4dfc9a3c2150200b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 8 Jul 2026 23:45:04 +0200 Subject: [PATCH] perf(backup): keyset-page the intake/mood/cycle reads in the off-host DR dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measurements were already paged, but the medication-intake, mood, and cycle tables in the same per-user payload were still unbounded full-table findMany reads — peak heap tracked the largest tenant's history. Give them the same id-keyset page walk (full row projection, so payload bytes are unchanged) so at most one page is resident per round-trip. --- src/lib/jobs/__tests__/offhost-backup.test.ts | 51 ++++++++++++++ src/lib/jobs/offhost-backup.ts | 68 +++++++++++++++++-- 2 files changed, 112 insertions(+), 7 deletions(-) diff --git a/src/lib/jobs/__tests__/offhost-backup.test.ts b/src/lib/jobs/__tests__/offhost-backup.test.ts index b2ffc0398..307617c02 100644 --- a/src/lib/jobs/__tests__/offhost-backup.test.ts +++ b/src/lib/jobs/__tests__/offhost-backup.test.ts @@ -6,6 +6,7 @@ import { runOffhostBackup, runOffhostRoundtripTest, } from "../offhost-backup"; +import { MEASUREMENT_BACKUP_PAGE_SIZE } from "../backup/measurement-page"; const ENC_KEY = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; @@ -138,6 +139,56 @@ describe("runOffhostBackup", () => { expect(parsed.userId).toBe("u1"); }); + it("keyset-pages the intake/mood/cycle tables across multiple pages", async () => { + const s3 = makeS3Mock(); + // A full first page (=== page size) followed by a short page proves the + // reader walks the id cursor instead of one unbounded findMany. + const fullPage = Array.from( + { length: MEASUREMENT_BACKUP_PAGE_SIZE }, + (_, i) => ({ + id: `e${String(i).padStart(5, "0")}`, + userId: "u1", + takenAt: new Date("2026-05-01T00:00:00Z"), + }), + ); + const tailPage = [ + { id: "z9999", userId: "u1", takenAt: new Date("2026-05-02T00:00:00Z") }, + ]; + const intakeFindMany = vi.fn( + async (args: { where?: { id?: { gt?: string } } }) => + args?.where?.id?.gt ? tailPage : fullPage, + ); + + const prisma = { + user: { findMany: vi.fn().mockResolvedValue([{ id: "u1" }]) }, + measurement: { findMany: vi.fn().mockResolvedValue([]) }, + medication: { findMany: vi.fn().mockResolvedValue([]) }, + medicationIntakeEvent: { findMany: intakeFindMany }, + moodEntry: { findMany: vi.fn().mockResolvedValue([]) }, + cycleProfile: { findUnique: vi.fn().mockResolvedValue(null) }, + menstrualCycle: { findMany: vi.fn().mockResolvedValue([]) }, + cycleDayLog: { findMany: vi.fn().mockResolvedValue([]) }, + }; + + const report = await runOffhostBackup( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prisma as any, + s3, + new Date("2026-05-08T00:00:00Z"), + ); + expect(report.uploaded).toBe(1); + // Two round-trips: the full page, then the short tail that ends the walk. + expect(intakeFindMany).toHaveBeenCalledTimes(2); + expect(intakeFindMany.mock.calls[0][0]).toMatchObject({ + orderBy: { id: "asc" }, + take: MEASUREMENT_BACKUP_PAGE_SIZE, + }); + // Every paged row lands in the decrypted payload — no rows dropped. + const ct = s3.store.get("2026-05-08/user-u1.json.enc")!; + const parsed = JSON.parse(decryptBackup(ct, Buffer.from(ENC_KEY, "hex"))); + expect(parsed.intakeEvents).toHaveLength(MEASUREMENT_BACKUP_PAGE_SIZE + 1); + }); + it("counts per-user failures without aborting the whole run", async () => { const s3 = makeS3Mock(); const prisma = { diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 2a6118c1a..d0c2ee672 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -279,16 +279,70 @@ export async function runOffhostBackup( where: { userId: user.id }, include: { schedules: true }, }), - prisma.medicationIntakeEvent.findMany({ where: { userId: user.id } }), - prisma.moodEntry.findMany({ where: { userId: user.id } }), + // The intake / mood / cycle tables scale with history depth the same + // way measurements do (multiple intakes + a mood + a cycle-day-log per + // day over years), so they get the same keyset-paged read: at most one + // page is resident per round-trip instead of the entire table landing + // in one array alongside the giant JSON.stringify below. The projection + // keeps the full row (no select), so payload bytes are identical to the + // prior default findMany — only peak heap during the read is bounded. + collectPagedMeasurements({ + fetchPage: (afterId, take) => + prisma.medicationIntakeEvent.findMany({ + where: { + userId: user.id, + ...(afterId ? { id: { gt: afterId } } : {}), + }, + orderBy: { id: "asc" }, + take, + }), + project: (row) => row, + pageSize: MEASUREMENT_BACKUP_PAGE_SIZE, + }), + collectPagedMeasurements({ + fetchPage: (afterId, take) => + prisma.moodEntry.findMany({ + where: { + userId: user.id, + ...(afterId ? { id: { gt: afterId } } : {}), + }, + orderBy: { id: "asc" }, + take, + }), + project: (row) => row, + pageSize: MEASUREMENT_BACKUP_PAGE_SIZE, + }), // v1.15.0 — cycle tables in the DR snapshot. Raw rows (this is a // disaster-recovery dump, not the canonical user-facing backup); - // `notesEncrypted` stays ciphertext. + // `notesEncrypted` stays ciphertext. The profile is a single row, so it + // stays a direct read; the cycle + day-log tables are paged. prisma.cycleProfile.findUnique({ where: { userId: user.id } }), - prisma.menstrualCycle.findMany({ where: { userId: user.id } }), - prisma.cycleDayLog.findMany({ - where: { userId: user.id }, - include: { symptomLinks: true }, + collectPagedMeasurements({ + fetchPage: (afterId, take) => + prisma.menstrualCycle.findMany({ + where: { + userId: user.id, + ...(afterId ? { id: { gt: afterId } } : {}), + }, + orderBy: { id: "asc" }, + take, + }), + project: (row) => row, + pageSize: MEASUREMENT_BACKUP_PAGE_SIZE, + }), + collectPagedMeasurements({ + fetchPage: (afterId, take) => + prisma.cycleDayLog.findMany({ + where: { + userId: user.id, + ...(afterId ? { id: { gt: afterId } } : {}), + }, + include: { symptomLinks: true }, + orderBy: { id: "asc" }, + take, + }), + project: (row) => row, + pageSize: MEASUREMENT_BACKUP_PAGE_SIZE, }), ]);