mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 12:32:43 +00:00
fix(import): write Apple Health spot rows without Prisma's per-call heap
Each `createManyAndReturn` flush of the export import left about 5 MB of
heap behind until some two dozen calls had piled up. A 300 000-record
synthetic import under a 524 MB heap settled at 53 MB retained; a run of
back-to-back 600-row calls with nothing between them reached 212 MB.
Spot rows now go through the column-array `INSERT … SELECT FROM unnest`
the restore already uses, with `ON CONFLICT DO NOTHING RETURNING`, which
is what `createMany({ skipDuplicates })` emits. The same import holds
11.5 MB and finishes in 35 s instead of 55 s. Ids are minted in the
cuid() shape, defaults are written as the client would write them, and
integration tests compare the returned rows and the resulting table with
`createManyAndReturn` for fresh rows, conflicts on either unique
identity, and duplicates inside one call.
This commit is contained in:
1 parent
d81518b5bf
commit
74dedee645
7 files changed
+472
-45
No files matched your search
@@ -130,6 +130,22 @@ const WRITERS: Record<string, Disposition> = {
|
||||
const MEASUREMENT_WRITE =
|
||||
/[A-Za-z_$][\w$]*\s*\.\s*measurement\s*\.\s*(create|createMany|createManyAndReturn|upsert)\s*\(/;
|
||||
|
||||
/**
|
||||
* A call into the column-array bulk insert, which writes around the Prisma
|
||||
* delegate and so never matches {@link MEASUREMENT_WRITE}. The Apple export
|
||||
* importer writes its spot rows only this way, and was invisible to this
|
||||
* guard for as long as nothing looked for it. The definitions themselves are
|
||||
* excluded by the lookbehind; the module that holds them is pinned below as
|
||||
* the one place raw SQL may insert a measurement.
|
||||
*/
|
||||
const BULK_INSERT_CALL = /(?<!function\s)\binsert(?:New)?MeasurementRows\s*\(/;
|
||||
|
||||
/** Raw SQL that inserts into the measurements table. */
|
||||
const RAW_MEASUREMENT_INSERT = /INSERT\s+INTO\s+"?measurements"?[\s(]/i;
|
||||
|
||||
/** The only module allowed to hold {@link RAW_MEASUREMENT_INSERT}. */
|
||||
const BULK_INSERT_MODULE = "lib/export/measurement-bulk-insert.ts";
|
||||
|
||||
function sourceFiles(): string[] {
|
||||
return walkSourceFiles(SRC, { floor: 3000 })
|
||||
.filter((p) => !p.startsWith("generated/"))
|
||||
@@ -143,7 +159,10 @@ function read(rel: string): string {
|
||||
}
|
||||
|
||||
function discoveredWriters(): string[] {
|
||||
return sourceFiles().filter((rel) => MEASUREMENT_WRITE.test(read(rel)));
|
||||
return sourceFiles().filter((rel) => {
|
||||
const text = read(rel);
|
||||
return MEASUREMENT_WRITE.test(text) || BULK_INSERT_CALL.test(text);
|
||||
});
|
||||
}
|
||||
|
||||
describe("every measurement writer declares how it meets the plausibility domain", () => {
|
||||
@@ -158,6 +177,21 @@ describe("every measurement writer declares how it meets the plausibility domain
|
||||
expect(discoveredWriters()).toEqual(Object.keys(WRITERS).sort());
|
||||
});
|
||||
|
||||
it("sees a writer that only calls the bulk insert", () => {
|
||||
// The importer has no Prisma measurement write left; if the bulk-insert
|
||||
// matcher stopped matching, it would drop out of the sweep silently.
|
||||
const importer = "lib/measurements/import-apple-health-export.ts";
|
||||
expect(MEASUREMENT_WRITE.test(read(importer))).toBe(false);
|
||||
expect(discoveredWriters()).toContain(importer);
|
||||
});
|
||||
|
||||
it("raw SQL inserts measurements in the bulk-insert module and nowhere else", () => {
|
||||
const raw = sourceFiles().filter((rel) =>
|
||||
RAW_MEASUREMENT_INSERT.test(read(rel)),
|
||||
);
|
||||
expect(raw).toEqual([BULK_INSERT_MODULE]);
|
||||
});
|
||||
|
||||
it("every provider-fed writer reaches the gate module", () => {
|
||||
const gated = Object.entries(WRITERS)
|
||||
.filter(([, d]) => d.kind === "gate")
|
||||
|
||||
@@ -10,13 +10,28 @@
|
||||
* `INSERT … SELECT FROM unnest(…)` with an array per column is a single small
|
||||
* statement whatever the batch size, and about four times faster.
|
||||
*
|
||||
* The Apple Health export import writes its spot rows through the same
|
||||
* statement (`insertNewMeasurementRows`), for the same reason. There the cost
|
||||
* was per call rather than per shape: every `createManyAndReturn` of a
|
||||
* 650-row flush left about 5 MB behind until roughly two dozen calls had
|
||||
* accumulated, so a 300 000-record import settled at about 53 MB of retained
|
||||
* heap and a run of back-to-back flushes reached well over 100 MB. Through
|
||||
* this statement the same import holds a few megabytes.
|
||||
*
|
||||
* `COLUMNS` maps every field of `MeasurementInsertRow` to its column and
|
||||
* type. It is typed as a complete record over the row's keys, so a field
|
||||
* added to the row without a column here does not compile, and the restore
|
||||
* builds its rows as `MeasurementInsertRow`, so a field it starts writing has
|
||||
* to be added to the row first.
|
||||
*/
|
||||
import type { Prisma } from "@/generated/prisma/client";
|
||||
import { randomInt } from "node:crypto";
|
||||
import { hostname } from "node:os";
|
||||
|
||||
import type {
|
||||
MeasurementType,
|
||||
Prisma,
|
||||
PrismaClient,
|
||||
} from "@/generated/prisma/client";
|
||||
|
||||
/** One measurement as the restore writes it. */
|
||||
export interface MeasurementInsertRow {
|
||||
@@ -134,3 +149,146 @@ export async function insertMeasurementRows(
|
||||
});
|
||||
return tx.$executeRawUnsafe(MEASUREMENT_BULK_INSERT_SQL, ...arrays);
|
||||
}
|
||||
|
||||
/**
|
||||
* The same statement, skipping any row that collides with an existing one on
|
||||
* either unique identity, and answering with the rows it did insert.
|
||||
*
|
||||
* `ON CONFLICT DO NOTHING` with no target is what Prisma emits for
|
||||
* `createMany({ skipDuplicates: true })`: a row that meets either
|
||||
* `(user_id, type, source, external_id)` or
|
||||
* `(user_id, type, measured_at, source, sleep_stage)` is left out, a row that
|
||||
* meets another row of the same statement is left out after the first, and
|
||||
* nothing throws for either.
|
||||
*/
|
||||
export const MEASUREMENT_BULK_INSERT_SKIP_DUPLICATES_SQL = `${MEASUREMENT_BULK_INSERT_SQL}
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING id, type::text AS "type", measured_at AS "measuredAt", external_id AS "externalId"
|
||||
`;
|
||||
|
||||
/** One row `insertNewMeasurementRows` wrote. */
|
||||
export interface InsertedMeasurementRow {
|
||||
id: string;
|
||||
type: MeasurementType;
|
||||
measuredAt: Date;
|
||||
externalId: string | null;
|
||||
}
|
||||
|
||||
/** A row for {@link insertNewMeasurementRows}: the id and the defaults are filled in. */
|
||||
export type NewMeasurementRow = Omit<
|
||||
MeasurementInsertRow,
|
||||
| "id"
|
||||
| "valueMin"
|
||||
| "valueMax"
|
||||
| "notes"
|
||||
| "notesEncrypted"
|
||||
| "aggregationProvenance"
|
||||
| "glucoseContext"
|
||||
| "rhythmClassification"
|
||||
| "syncVersion"
|
||||
| "deletedAt"
|
||||
| "createdAt"
|
||||
| "updatedAt"
|
||||
> &
|
||||
Partial<
|
||||
Pick<
|
||||
MeasurementInsertRow,
|
||||
| "valueMin"
|
||||
| "valueMax"
|
||||
| "notesEncrypted"
|
||||
| "aggregationProvenance"
|
||||
| "glucoseContext"
|
||||
| "rhythmClassification"
|
||||
>
|
||||
>;
|
||||
|
||||
/**
|
||||
* Insert brand-new rows in one statement, skipping duplicates, and return the
|
||||
* rows that landed. The replacement for
|
||||
* `measurement.createManyAndReturn({ data, skipDuplicates: true })`.
|
||||
*
|
||||
* Every column the schema defaults is written with that default: a fresh
|
||||
* cuid-shaped id, `sync_version` 1, not deleted, created and updated at the
|
||||
* moment of the call, and null for every optional column the row does not
|
||||
* carry.
|
||||
*/
|
||||
export async function insertNewMeasurementRows(
|
||||
db: Pick<PrismaClient, "$queryRawUnsafe"> | Prisma.TransactionClient,
|
||||
rows: readonly NewMeasurementRow[],
|
||||
): Promise<InsertedMeasurementRow[]> {
|
||||
if (rows.length === 0) return [];
|
||||
// Stamped here, as the Prisma client stamps `@default(now())` and
|
||||
// `@updatedAt`, rather than left to the database clock.
|
||||
const now = new Date();
|
||||
const full: MeasurementInsertRow[] = rows.map((row) => ({
|
||||
id: newMeasurementId(),
|
||||
userId: row.userId,
|
||||
type: row.type,
|
||||
value: row.value,
|
||||
valueMin: row.valueMin ?? null,
|
||||
valueMax: row.valueMax ?? null,
|
||||
unit: row.unit,
|
||||
source: row.source,
|
||||
measuredAt: row.measuredAt,
|
||||
notes: null,
|
||||
notesEncrypted: row.notesEncrypted ?? null,
|
||||
externalId: row.externalId,
|
||||
externalSourceVersion: row.externalSourceVersion,
|
||||
aggregationProvenance: row.aggregationProvenance ?? null,
|
||||
glucoseContext: row.glucoseContext ?? null,
|
||||
sleepStage: row.sleepStage,
|
||||
rhythmClassification: row.rhythmClassification ?? null,
|
||||
deviceType: row.deviceType,
|
||||
syncVersion: 1,
|
||||
deletedAt: null,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
}));
|
||||
const arrays = FIELDS.map((field) => {
|
||||
const kind = COLUMNS[field][1];
|
||||
return full.map((row) => cell(row[field], kind));
|
||||
});
|
||||
return db.$queryRawUnsafe<InsertedMeasurementRow[]>(
|
||||
MEASUREMENT_BULK_INSERT_SKIP_DUPLICATES_SQL,
|
||||
...arrays,
|
||||
);
|
||||
}
|
||||
|
||||
// ── Ids ──────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// Prisma mints `@default(cuid())` in the client, so a row written around the
|
||||
// client needs one minted here. Same shape as Prisma's own (cuid v1): `c`, the
|
||||
// time in base 36, a rolling counter, a host fingerprint, and eight random
|
||||
// characters, 25 characters in all. Ids stay roughly time-ordered, which the
|
||||
// list reads rely on only as a tiebreaker.
|
||||
|
||||
const BLOCK = 4;
|
||||
const BLOCK_SPACE = 36 ** BLOCK;
|
||||
let counter = randomInt(BLOCK_SPACE);
|
||||
|
||||
function block(n: number): string {
|
||||
return n.toString(36).padStart(BLOCK, "0").slice(-BLOCK);
|
||||
}
|
||||
|
||||
const FINGERPRINT = (() => {
|
||||
const host = hostname();
|
||||
let sum = host.length + 36;
|
||||
for (let i = 0; i < host.length; i++) sum += host.charCodeAt(i);
|
||||
return (
|
||||
(process.pid % 1296).toString(36).padStart(2, "0") +
|
||||
(sum % 1296).toString(36).padStart(2, "0")
|
||||
);
|
||||
})();
|
||||
|
||||
/** A fresh id in the shape Prisma's `cuid()` default produces. */
|
||||
export function newMeasurementId(): string {
|
||||
counter = (counter + 1) % BLOCK_SPACE;
|
||||
return (
|
||||
"c" +
|
||||
Date.now().toString(36).padStart(8, "0") +
|
||||
block(counter) +
|
||||
FINGERPRINT +
|
||||
block(randomInt(BLOCK_SPACE)) +
|
||||
block(randomInt(BLOCK_SPACE))
|
||||
);
|
||||
}
|
||||
@@ -11,6 +11,27 @@ vi.mock("@/lib/daily/morning-refresh-trigger", () => ({
|
||||
maybeEnqueueMorningRefresh: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
// The spot flush writes through one raw `INSERT … ON CONFLICT DO NOTHING
|
||||
// RETURNING` statement. These fakes model the table in memory, so route that
|
||||
// statement to the fake's own skip-duplicates insert; the SQL itself is pinned
|
||||
// against a real Postgres in tests/integration/measurement-bulk-insert.test.ts.
|
||||
vi.mock("@/lib/export/measurement-bulk-insert", async (importOriginal) => ({
|
||||
...(await importOriginal<
|
||||
typeof import("@/lib/export/measurement-bulk-insert")
|
||||
>()),
|
||||
insertNewMeasurementRows: vi.fn(
|
||||
(
|
||||
db: { measurement: { createManyAndReturn: (args: unknown) => unknown } },
|
||||
rows: unknown[],
|
||||
) =>
|
||||
db.measurement.createManyAndReturn({
|
||||
data: rows,
|
||||
skipDuplicates: true,
|
||||
select: { id: true, type: true, measuredAt: true, externalId: true },
|
||||
}),
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/arrivals/emit-shared", () => ({
|
||||
emitDataArrival: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
@@ -35,6 +56,7 @@ import {
|
||||
import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit";
|
||||
import { maybeEnqueueMorningRefresh } from "@/lib/daily/morning-refresh-trigger";
|
||||
import { emitDataArrival } from "@/lib/arrivals/emit-shared";
|
||||
import { insertNewMeasurementRows } from "@/lib/export/measurement-bulk-insert";
|
||||
import {
|
||||
APPLE_HEALTH_TYPE_MAP,
|
||||
CUMULATIVE_HK_TYPES,
|
||||
@@ -731,6 +753,40 @@ describe("streamParseExportXml — cumulative source-day estimates", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("streamParseExportXml — spot-row writer", () => {
|
||||
it("writes spot rows through the bounded bulk insert, never createManyAndReturn", async () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
|
||||
const xmlPath = join(tmp, "export.xml");
|
||||
writeFileSync(xmlPath, tinyExportXml());
|
||||
const prisma = makeFakePrisma();
|
||||
const direct = vi.spyOn(prisma.measurement, "createManyAndReturn");
|
||||
const helper = vi.mocked(insertNewMeasurementRows);
|
||||
helper.mockClear();
|
||||
|
||||
await streamParseExportXml({
|
||||
xmlPath,
|
||||
userId: "user-1",
|
||||
userTimezone: "Europe/Berlin",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
prisma: prisma as any,
|
||||
spotBatchSize: 2,
|
||||
workoutBatchSize: 1,
|
||||
});
|
||||
|
||||
expect(helper).toHaveBeenCalled();
|
||||
// Every createManyAndReturn the fake saw came through the helper's
|
||||
// stand-in above; none came from the import itself.
|
||||
expect(direct).toHaveBeenCalledTimes(helper.mock.calls.length);
|
||||
const written = helper.mock.calls.flatMap(([, rows]) => [...rows]);
|
||||
expect(written.map((r) => r.type)).toEqual(
|
||||
expect.arrayContaining(["WEIGHT", "SLEEP_DURATION"]),
|
||||
);
|
||||
expect(new Set(written.map((r) => r.source))).toEqual(
|
||||
new Set(["APPLE_HEALTH"]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("streamParseExportXml — end-to-end", () => {
|
||||
it("ingests the tiny fixture into per-type / workout / deferred / unknown buckets", async () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
|
||||
|
||||
@@ -43,6 +43,25 @@ vi.mock("@/lib/daily/morning-refresh-trigger", () => ({
|
||||
maybeEnqueueMorningRefresh: async () => undefined,
|
||||
}));
|
||||
|
||||
// The spot flush writes through one raw `INSERT … ON CONFLICT DO NOTHING
|
||||
// RETURNING` statement. These fakes model the table in memory, so route that
|
||||
// statement to the fake's own skip-duplicates insert; the SQL itself is pinned
|
||||
// against a real Postgres in tests/integration/measurement-bulk-insert.test.ts.
|
||||
vi.mock("@/lib/export/measurement-bulk-insert", async (importOriginal) => ({
|
||||
...(await importOriginal<
|
||||
typeof import("@/lib/export/measurement-bulk-insert")
|
||||
>()),
|
||||
insertNewMeasurementRows: (
|
||||
db: { measurement: { createManyAndReturn: (args: unknown) => unknown } },
|
||||
rows: unknown[],
|
||||
) =>
|
||||
db.measurement.createManyAndReturn({
|
||||
data: rows,
|
||||
skipDuplicates: true,
|
||||
select: { id: true, type: true, measuredAt: true, externalId: true },
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/arrivals/emit-shared", () => ({
|
||||
emitDataArrival: async () => undefined,
|
||||
}));
|
||||
|
||||
@@ -41,6 +41,11 @@ import {
|
||||
canonicalDailyTimestamp,
|
||||
} from "@/lib/measurements/drain-per-sample-cumulative";
|
||||
import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement";
|
||||
import {
|
||||
insertNewMeasurementRows,
|
||||
type InsertedMeasurementRow,
|
||||
type NewMeasurementRow,
|
||||
} from "@/lib/export/measurement-bulk-insert";
|
||||
import { resolveHkWorkoutSportType } from "@/lib/measurements/hk-workout-activity-type-map";
|
||||
import {
|
||||
convertHkValue,
|
||||
@@ -309,7 +314,10 @@ export interface StreamParseInput {
|
||||
/** IANA timezone, used to anchor cumulative-type day-keys. */
|
||||
userTimezone: string;
|
||||
/** Prisma client to flush rows through. */
|
||||
prisma: Pick<PrismaClient, "measurement" | "workout" | "$transaction">;
|
||||
prisma: Pick<
|
||||
PrismaClient,
|
||||
"measurement" | "workout" | "$transaction" | "$queryRawUnsafe"
|
||||
>;
|
||||
/**
|
||||
* Live progress hook. Called every `PROGRESS_TICK_RECORDS` records
|
||||
* read, and once on terminal `done`. Best-effort; the parser
|
||||
@@ -457,55 +465,38 @@ export async function streamParseExportXml(
|
||||
type: MeasurementType;
|
||||
measuredAt: Date;
|
||||
}> = [];
|
||||
const createData: Prisma.MeasurementCreateManyInput[] = chunk.map(
|
||||
(row) => ({
|
||||
userId,
|
||||
type: row.type,
|
||||
value: row.value,
|
||||
unit: row.unit,
|
||||
source: "APPLE_HEALTH",
|
||||
measuredAt: row.measuredAt,
|
||||
externalId: row.externalId,
|
||||
externalSourceVersion: row.externalSourceVersion,
|
||||
sleepStage: row.sleepStage ?? null,
|
||||
deviceType: row.deviceType,
|
||||
}),
|
||||
);
|
||||
// Written as one `INSERT … SELECT FROM unnest(…) ON CONFLICT DO NOTHING
|
||||
// RETURNING` rather than `createManyAndReturn`: the Prisma call left about
|
||||
// 5 MB of heap behind per flush until some two dozen had piled up, which
|
||||
// is the import's whole budget in a small container. Same semantics —
|
||||
// duplicates on either unique identity are skipped, never thrown — and the
|
||||
// same answer: the rows that landed. See `insertNewMeasurementRows`.
|
||||
const createData: NewMeasurementRow[] = chunk.map((row) => ({
|
||||
userId,
|
||||
type: row.type,
|
||||
value: row.value,
|
||||
unit: row.unit,
|
||||
source: "APPLE_HEALTH",
|
||||
measuredAt: row.measuredAt,
|
||||
externalId: row.externalId,
|
||||
externalSourceVersion: row.externalSourceVersion,
|
||||
sleepStage: row.sleepStage ?? null,
|
||||
deviceType: row.deviceType,
|
||||
}));
|
||||
const insertStartedAt = Date.now();
|
||||
let createdRows: Array<{
|
||||
id: string;
|
||||
type: MeasurementType;
|
||||
measuredAt: Date;
|
||||
externalId: string | null;
|
||||
}> = [];
|
||||
let createdRows: InsertedMeasurementRow[] = [];
|
||||
const failedInsertIndexes = new Set<number>();
|
||||
try {
|
||||
createdRows = await prisma.measurement.createManyAndReturn({
|
||||
data: createData,
|
||||
skipDuplicates: true,
|
||||
select: {
|
||||
id: true,
|
||||
type: true,
|
||||
measuredAt: true,
|
||||
externalId: true,
|
||||
},
|
||||
});
|
||||
createdRows = await insertNewMeasurementRows(prisma, createData);
|
||||
} catch {
|
||||
// Retain the old per-sample failure isolation if an unexpected database
|
||||
// error rejects the bulk statement. Conflicts remain non-errors because
|
||||
// every retry still uses skipDuplicates.
|
||||
// every retry still skips duplicates.
|
||||
for (let index = 0; index < createData.length; index += 1) {
|
||||
try {
|
||||
const created = await prisma.measurement.createManyAndReturn({
|
||||
data: [createData[index]],
|
||||
skipDuplicates: true,
|
||||
select: {
|
||||
id: true,
|
||||
type: true,
|
||||
measuredAt: true,
|
||||
externalId: true,
|
||||
},
|
||||
});
|
||||
const created = await insertNewMeasurementRows(prisma, [
|
||||
createData[index],
|
||||
]);
|
||||
createdRows.push(...created);
|
||||
} catch {
|
||||
failedInsertIndexes.add(index);
|
||||
|
||||
@@ -150,8 +150,13 @@ describe("WHOOP-written HealthKit records remain source-agnostic", () => {
|
||||
});
|
||||
|
||||
it("persists supported export.xml rows as APPLE_HEALTH, never as a raw writer label", () => {
|
||||
// Spot rows are built as `NewMeasurementRow`s and written through the
|
||||
// column-array bulk insert; the source is fixed where each row is built.
|
||||
expect(appleImporterSource).toMatch(
|
||||
/measurement\.createManyAndReturn\([\s\S]*source:\s*"APPLE_HEALTH"/,
|
||||
/const createData: NewMeasurementRow\[\] = chunk\.map\(\(row\) => \(\{[^}]*source:\s*"APPLE_HEALTH"/,
|
||||
);
|
||||
expect(appleImporterSource).toMatch(
|
||||
/insertNewMeasurementRows\(prisma, createData\)/,
|
||||
);
|
||||
expect(appleImporterSource).not.toMatch(
|
||||
/source:\s*attrs\.(?:sourceName|device)/,
|
||||
|
||||
@@ -10,7 +10,10 @@ import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
import {
|
||||
insertMeasurementRows,
|
||||
insertNewMeasurementRows,
|
||||
newMeasurementId,
|
||||
type MeasurementInsertRow,
|
||||
type NewMeasurementRow,
|
||||
} from "@/lib/export/measurement-bulk-insert";
|
||||
|
||||
const USER = "user-bulk-insert";
|
||||
@@ -137,3 +140,164 @@ describe("insertMeasurementRows", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* `insertNewMeasurementRows` replaced `createManyAndReturn({ skipDuplicates })`
|
||||
* in the Apple Health export import. Each case below runs the same input
|
||||
* through both and compares what they return and what they leave in the
|
||||
* table, so a difference in either direction fails.
|
||||
*/
|
||||
describe("insertNewMeasurementRows", () => {
|
||||
function spot(
|
||||
externalId: string,
|
||||
at: string,
|
||||
overrides: Partial<NewMeasurementRow> = {},
|
||||
): NewMeasurementRow {
|
||||
return {
|
||||
userId: USER,
|
||||
type: "PULSE",
|
||||
value: 61,
|
||||
unit: "bpm",
|
||||
source: "APPLE_HEALTH",
|
||||
measuredAt: new Date(at),
|
||||
externalId,
|
||||
externalSourceVersion: "17.4",
|
||||
sleepStage: null,
|
||||
deviceType: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const SELECT = {
|
||||
id: true,
|
||||
type: true,
|
||||
measuredAt: true,
|
||||
externalId: true,
|
||||
} as const;
|
||||
|
||||
/** Run `rows` through Prisma, then through the raw path in a fresh table. */
|
||||
async function both(seed: NewMeasurementRow[], rows: NewMeasurementRow[]) {
|
||||
const prisma = getPrismaClient();
|
||||
const run = async (
|
||||
write: (r: NewMeasurementRow[]) => Promise<
|
||||
Array<{
|
||||
id: string;
|
||||
type: string;
|
||||
measuredAt: Date;
|
||||
externalId: string | null;
|
||||
}>
|
||||
>,
|
||||
) => {
|
||||
await prisma.measurement.deleteMany({ where: { userId: USER } });
|
||||
if (seed.length > 0) {
|
||||
await prisma.measurement.createMany({ data: seed as never });
|
||||
}
|
||||
const returned = await write(rows);
|
||||
const table = await prisma.measurement.findMany({
|
||||
where: { userId: USER },
|
||||
orderBy: [{ measuredAt: "asc" }, { externalId: "asc" }],
|
||||
});
|
||||
return {
|
||||
returned: returned
|
||||
.map(({ id: _id, ...rest }) => rest)
|
||||
.sort((a, b) => `${a.externalId}`.localeCompare(`${b.externalId}`)),
|
||||
table: table.map(
|
||||
({ id: _id, createdAt: _c, updatedAt: _u, ...rest }) => rest,
|
||||
),
|
||||
ids: returned.map((r) => r.id),
|
||||
};
|
||||
};
|
||||
const viaPrisma = await run((r) =>
|
||||
prisma.measurement.createManyAndReturn({
|
||||
data: r as never,
|
||||
skipDuplicates: true,
|
||||
select: SELECT,
|
||||
}),
|
||||
);
|
||||
const viaRaw = await run((r) => insertNewMeasurementRows(prisma, r));
|
||||
return { viaPrisma, viaRaw };
|
||||
}
|
||||
|
||||
it("returns and stores exactly what createManyAndReturn does", async () => {
|
||||
const rows = [
|
||||
spot("hr-1", "2026-03-01T08:00:00.123Z"),
|
||||
spot("sleep-1", "2026-03-01T06:00:00.000Z", {
|
||||
type: "SLEEP_DURATION",
|
||||
value: 42.5,
|
||||
unit: "minutes",
|
||||
sleepStage: "REM",
|
||||
deviceType: "watch",
|
||||
}),
|
||||
spot("spo2-1", "2026-03-01T09:00:00.000Z", {
|
||||
type: "OXYGEN_SATURATION",
|
||||
value: 97,
|
||||
unit: "%",
|
||||
externalSourceVersion: null,
|
||||
}),
|
||||
];
|
||||
const { viaPrisma, viaRaw } = await both([], rows);
|
||||
expect(viaRaw.returned).toEqual(viaPrisma.returned);
|
||||
expect(viaRaw.table).toEqual(viaPrisma.table);
|
||||
expect(viaRaw.returned).toHaveLength(3);
|
||||
for (const r of viaRaw.returned) expect(r.measuredAt).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
it("skips a row that meets an existing row on either unique identity", async () => {
|
||||
const seed = [
|
||||
// Same external id as `ext-dup`.
|
||||
spot("ext-dup", "2026-03-02T08:00:00.000Z", { value: 70 }),
|
||||
// Same (type, measured_at, source, sleep_stage) as `nat-dup`.
|
||||
spot("seeded-natural", "2026-03-02T09:00:00.000Z", { value: 71 }),
|
||||
];
|
||||
const rows = [
|
||||
spot("ext-dup", "2026-03-02T10:00:00.000Z", { value: 80 }),
|
||||
spot("nat-dup", "2026-03-02T09:00:00.000Z", { value: 81 }),
|
||||
spot("fresh", "2026-03-02T11:00:00.000Z", { value: 82 }),
|
||||
];
|
||||
const { viaPrisma, viaRaw } = await both(seed, rows);
|
||||
expect(viaRaw.returned).toEqual(viaPrisma.returned);
|
||||
expect(viaRaw.table).toEqual(viaPrisma.table);
|
||||
expect(viaRaw.returned.map((r) => r.externalId)).toEqual(["fresh"]);
|
||||
});
|
||||
|
||||
it("keeps the first of two rows in one call that share an identity", async () => {
|
||||
const rows = [
|
||||
spot("twin", "2026-03-03T08:00:00.000Z", { value: 90 }),
|
||||
spot("twin", "2026-03-03T08:30:00.000Z", { value: 91 }),
|
||||
spot("other", "2026-03-03T08:00:00.000Z", { value: 92 }),
|
||||
];
|
||||
const { viaPrisma, viaRaw } = await both([], rows);
|
||||
expect(viaRaw.returned).toEqual(viaPrisma.returned);
|
||||
expect(viaRaw.table).toEqual(viaPrisma.table);
|
||||
expect(viaRaw.returned).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("mints ids in the shape the cuid() default produces", async () => {
|
||||
const prisma = getPrismaClient();
|
||||
await prisma.measurement.deleteMany({ where: { userId: USER } });
|
||||
const viaPrisma = await prisma.measurement.create({
|
||||
data: spot("shape-prisma", "2026-03-04T08:00:00.000Z") as never,
|
||||
});
|
||||
const [viaRaw] = await insertNewMeasurementRows(prisma, [
|
||||
spot("shape-raw", "2026-03-04T09:00:00.000Z"),
|
||||
]);
|
||||
const shape = /^c[0-9a-z]{24}$/;
|
||||
expect(viaPrisma.id).toMatch(shape);
|
||||
expect(viaRaw!.id).toMatch(shape);
|
||||
const ids = new Set(Array.from({ length: 10_000 }, newMeasurementId));
|
||||
expect(ids.size).toBe(10_000);
|
||||
const stored = await prisma.measurement.findUniqueOrThrow({
|
||||
where: { id: viaRaw!.id },
|
||||
});
|
||||
expect(stored.syncVersion).toBe(1);
|
||||
expect(stored.deletedAt).toBeNull();
|
||||
expect(Math.abs(Date.now() - stored.createdAt.getTime())).toBeLessThan(
|
||||
60_000,
|
||||
);
|
||||
expect(stored.updatedAt.getTime()).toBe(stored.createdAt.getTime());
|
||||
});
|
||||
|
||||
it("answers an empty call without touching the database", async () => {
|
||||
expect(await insertNewMeasurementRows(getPrismaClient(), [])).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user