fix(import): write Apple Health spot rows without Prisma's per-call heap

Each `createManyAndReturn` flush of the export import left about 5 MB of
heap behind until some two dozen calls had piled up. A 300 000-record
synthetic import under a 524 MB heap settled at 53 MB retained; a run of
back-to-back 600-row calls with nothing between them reached 212 MB.

Spot rows now go through the column-array `INSERT … SELECT FROM unnest`
the restore already uses, with `ON CONFLICT DO NOTHING RETURNING`, which
is what `createMany({ skipDuplicates })` emits. The same import holds
11.5 MB and finishes in 35 s instead of 55 s. Ids are minted in the
cuid() shape, defaults are written as the client would write them, and
integration tests compare the returned rows and the resulting table with
`createManyAndReturn` for fresh rows, conflicts on either unique
identity, and duplicates inside one call.
This commit is contained in:
Marc-André Bombeck committed 2026-09-25 07:39:07 +02:00
1 parent d81518b5bf
commit 74dedee645
7 files changed
+472 -45

No files matched your search

@@ -10,7 +10,10 @@ import { beforeEach, describe, expect, it } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import {
insertMeasurementRows,
insertNewMeasurementRows,
newMeasurementId,
type MeasurementInsertRow,
type NewMeasurementRow,
} from "@/lib/export/measurement-bulk-insert";
const USER = "user-bulk-insert";
@@ -137,3 +140,164 @@ describe("insertMeasurementRows", () => {
);
});
});
/**
* `insertNewMeasurementRows` replaced `createManyAndReturn({ skipDuplicates })`
* in the Apple Health export import. Each case below runs the same input
* through both and compares what they return and what they leave in the
* table, so a difference in either direction fails.
*/
describe("insertNewMeasurementRows", () => {
function spot(
externalId: string,
at: string,
overrides: Partial<NewMeasurementRow> = {},
): NewMeasurementRow {
return {
userId: USER,
type: "PULSE",
value: 61,
unit: "bpm",
source: "APPLE_HEALTH",
measuredAt: new Date(at),
externalId,
externalSourceVersion: "17.4",
sleepStage: null,
deviceType: null,
...overrides,
};
}
const SELECT = {
id: true,
type: true,
measuredAt: true,
externalId: true,
} as const;
/** Run `rows` through Prisma, then through the raw path in a fresh table. */
async function both(seed: NewMeasurementRow[], rows: NewMeasurementRow[]) {
const prisma = getPrismaClient();
const run = async (
write: (r: NewMeasurementRow[]) => Promise<
Array<{
id: string;
type: string;
measuredAt: Date;
externalId: string | null;
}>
>,
) => {
await prisma.measurement.deleteMany({ where: { userId: USER } });
if (seed.length > 0) {
await prisma.measurement.createMany({ data: seed as never });
}
const returned = await write(rows);
const table = await prisma.measurement.findMany({
where: { userId: USER },
orderBy: [{ measuredAt: "asc" }, { externalId: "asc" }],
});
return {
returned: returned
.map(({ id: _id, ...rest }) => rest)
.sort((a, b) => `${a.externalId}`.localeCompare(`${b.externalId}`)),
table: table.map(
({ id: _id, createdAt: _c, updatedAt: _u, ...rest }) => rest,
),
ids: returned.map((r) => r.id),
};
};
const viaPrisma = await run((r) =>
prisma.measurement.createManyAndReturn({
data: r as never,
skipDuplicates: true,
select: SELECT,
}),
);
const viaRaw = await run((r) => insertNewMeasurementRows(prisma, r));
return { viaPrisma, viaRaw };
}
it("returns and stores exactly what createManyAndReturn does", async () => {
const rows = [
spot("hr-1", "2026-03-01T08:00:00.123Z"),
spot("sleep-1", "2026-03-01T06:00:00.000Z", {
type: "SLEEP_DURATION",
value: 42.5,
unit: "minutes",
sleepStage: "REM",
deviceType: "watch",
}),
spot("spo2-1", "2026-03-01T09:00:00.000Z", {
type: "OXYGEN_SATURATION",
value: 97,
unit: "%",
externalSourceVersion: null,
}),
];
const { viaPrisma, viaRaw } = await both([], rows);
expect(viaRaw.returned).toEqual(viaPrisma.returned);
expect(viaRaw.table).toEqual(viaPrisma.table);
expect(viaRaw.returned).toHaveLength(3);
for (const r of viaRaw.returned) expect(r.measuredAt).toBeInstanceOf(Date);
});
it("skips a row that meets an existing row on either unique identity", async () => {
const seed = [
// Same external id as `ext-dup`.
spot("ext-dup", "2026-03-02T08:00:00.000Z", { value: 70 }),
// Same (type, measured_at, source, sleep_stage) as `nat-dup`.
spot("seeded-natural", "2026-03-02T09:00:00.000Z", { value: 71 }),
];
const rows = [
spot("ext-dup", "2026-03-02T10:00:00.000Z", { value: 80 }),
spot("nat-dup", "2026-03-02T09:00:00.000Z", { value: 81 }),
spot("fresh", "2026-03-02T11:00:00.000Z", { value: 82 }),
];
const { viaPrisma, viaRaw } = await both(seed, rows);
expect(viaRaw.returned).toEqual(viaPrisma.returned);
expect(viaRaw.table).toEqual(viaPrisma.table);
expect(viaRaw.returned.map((r) => r.externalId)).toEqual(["fresh"]);
});
it("keeps the first of two rows in one call that share an identity", async () => {
const rows = [
spot("twin", "2026-03-03T08:00:00.000Z", { value: 90 }),
spot("twin", "2026-03-03T08:30:00.000Z", { value: 91 }),
spot("other", "2026-03-03T08:00:00.000Z", { value: 92 }),
];
const { viaPrisma, viaRaw } = await both([], rows);
expect(viaRaw.returned).toEqual(viaPrisma.returned);
expect(viaRaw.table).toEqual(viaPrisma.table);
expect(viaRaw.returned).toHaveLength(1);
});
it("mints ids in the shape the cuid() default produces", async () => {
const prisma = getPrismaClient();
await prisma.measurement.deleteMany({ where: { userId: USER } });
const viaPrisma = await prisma.measurement.create({
data: spot("shape-prisma", "2026-03-04T08:00:00.000Z") as never,
});
const [viaRaw] = await insertNewMeasurementRows(prisma, [
spot("shape-raw", "2026-03-04T09:00:00.000Z"),
]);
const shape = /^c[0-9a-z]{24}$/;
expect(viaPrisma.id).toMatch(shape);
expect(viaRaw!.id).toMatch(shape);
const ids = new Set(Array.from({ length: 10_000 }, newMeasurementId));
expect(ids.size).toBe(10_000);
const stored = await prisma.measurement.findUniqueOrThrow({
where: { id: viaRaw!.id },
});
expect(stored.syncVersion).toBe(1);
expect(stored.deletedAt).toBeNull();
expect(Math.abs(Date.now() - stored.createdAt.getTime())).toBeLessThan(
60_000,
);
expect(stored.updatedAt.getTime()).toBe(stored.createdAt.getTime());
});
it("answers an empty call without touching the database", async () => {
expect(await insertNewMeasurementRows(getPrismaClient(), [])).toEqual([]);
});
});