fix(import): write Apple Health spot rows without Prisma's per-call heap

Each `createManyAndReturn` flush of the export import left about 5 MB of
heap behind until some two dozen calls had piled up. A 300 000-record
synthetic import under a 524 MB heap settled at 53 MB retained; a run of
back-to-back 600-row calls with nothing between them reached 212 MB.

Spot rows now go through the column-array `INSERT … SELECT FROM unnest`
the restore already uses, with `ON CONFLICT DO NOTHING RETURNING`, which
is what `createMany({ skipDuplicates })` emits. The same import holds
11.5 MB and finishes in 35 s instead of 55 s. Ids are minted in the
cuid() shape, defaults are written as the client would write them, and
integration tests compare the returned rows and the resulting table with
`createManyAndReturn` for fresh rows, conflicts on either unique
identity, and duplicates inside one call.
This commit is contained in:
Marc-André Bombeck committed 2026-09-25 07:39:07 +02:00
1 parent d81518b5bf
commit 74dedee645
7 files changed
+472 -45

No files matched your search

@@ -130,6 +130,22 @@ const WRITERS: Record<string, Disposition> = {
const MEASUREMENT_WRITE =
/[A-Za-z_$][\w$]*\s*\.\s*measurement\s*\.\s*(create|createMany|createManyAndReturn|upsert)\s*\(/;
/**
* A call into the column-array bulk insert, which writes around the Prisma
* delegate and so never matches {@link MEASUREMENT_WRITE}. The Apple export
* importer writes its spot rows only this way, and was invisible to this
* guard for as long as nothing looked for it. The definitions themselves are
* excluded by the lookbehind; the module that holds them is pinned below as
* the one place raw SQL may insert a measurement.
*/
const BULK_INSERT_CALL = /(?<!function\s)\binsert(?:New)?MeasurementRows\s*\(/;
/** Raw SQL that inserts into the measurements table. */
const RAW_MEASUREMENT_INSERT = /INSERT\s+INTO\s+"?measurements"?[\s(]/i;
/** The only module allowed to hold {@link RAW_MEASUREMENT_INSERT}. */
const BULK_INSERT_MODULE = "lib/export/measurement-bulk-insert.ts";
function sourceFiles(): string[] {
return walkSourceFiles(SRC, { floor: 3000 })
.filter((p) => !p.startsWith("generated/"))
@@ -143,7 +159,10 @@ function read(rel: string): string {
}
function discoveredWriters(): string[] {
return sourceFiles().filter((rel) => MEASUREMENT_WRITE.test(read(rel)));
return sourceFiles().filter((rel) => {
const text = read(rel);
return MEASUREMENT_WRITE.test(text) || BULK_INSERT_CALL.test(text);
});
}
describe("every measurement writer declares how it meets the plausibility domain", () => {
@@ -158,6 +177,21 @@ describe("every measurement writer declares how it meets the plausibility domain
expect(discoveredWriters()).toEqual(Object.keys(WRITERS).sort());
});
it("sees a writer that only calls the bulk insert", () => {
// The importer has no Prisma measurement write left; if the bulk-insert
// matcher stopped matching, it would drop out of the sweep silently.
const importer = "lib/measurements/import-apple-health-export.ts";
expect(MEASUREMENT_WRITE.test(read(importer))).toBe(false);
expect(discoveredWriters()).toContain(importer);
});
it("raw SQL inserts measurements in the bulk-insert module and nowhere else", () => {
const raw = sourceFiles().filter((rel) =>
RAW_MEASUREMENT_INSERT.test(read(rel)),
);
expect(raw).toEqual([BULK_INSERT_MODULE]);
});
it("every provider-fed writer reaches the gate module", () => {
const gated = Object.entries(WRITERS)
.filter(([, d]) => d.kind === "gate")