diff --git a/src/app/api/cycle/insights/__tests__/route-locale.test.ts b/src/app/api/cycle/insights/__tests__/route-locale.test.ts index a55be82e6..0a516735c 100644 --- a/src/app/api/cycle/insights/__tests__/route-locale.test.ts +++ b/src/app/api/cycle/insights/__tests__/route-locale.test.ts @@ -37,6 +37,9 @@ vi.mock("@/lib/cycle/phase-crosstab", async (importOriginal) => ({ discoverPhaseCorrelations(args), })); +vi.mock("@/lib/measurements/day-aggregates", () => ({ + readSourceDayAggregates: vi.fn().mockResolvedValue([]), +})); vi.mock("@/lib/db", () => ({ prisma: { auditLog: { create: vi.fn() }, diff --git a/src/app/api/cycle/insights/route.ts b/src/app/api/cycle/insights/route.ts index 4fc89c5b3..fb9d6c1c8 100644 --- a/src/app/api/cycle/insights/route.ts +++ b/src/app/api/cycle/insights/route.ts @@ -43,6 +43,7 @@ import { type SymptomDay, } from "@/lib/cycle/symptom-phase"; import { addDays } from "@/lib/cycle/day-math"; +import { readSourceDayAggregates } from "@/lib/measurements/day-aggregates"; import { moodDateKey } from "@/lib/mood/date-key"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; @@ -130,27 +131,19 @@ export const GET = apiHandler(async () => { orderBy: { measuredAt: "asc" }, select: { measuredAt: true, value: true }, }), - // The outcome metrics the phase contrast compares — soft-delete-scoped, - // canonical-source deduped per day inside `metricDayMap`. Skipped whole - // for a caller outside `measurements`. + // The outcome metrics the phase contrast compares, soft-delete-scoped. + // A year of steps, heart-rate variability and sensor glucose is far too + // many readings to hold as objects (#1023), and the crosstab uses only + // each day's sum and count after picking one source and device per day. + // So the fold runs in SQL, one row per type, local day, source and + // device, the read the coach's cycle snapshot already makes. Skipped + // whole for a caller outside `measurements`. seesMeasurements - ? prisma.measurement.findMany({ - where: { - userId: user.id, - deletedAt: null, - type: { in: PHASE_CROSSTAB_METRIC_TYPES }, - measuredAt: { - gte: new Date(Date.parse(`${from}T00:00:00Z`)), - }, - }, - orderBy: { measuredAt: "asc" }, - select: { - type: true, - value: true, - measuredAt: true, - source: true, - deviceType: true, - }, + ? readSourceDayAggregates({ + userId: user.id, + types: PHASE_CROSSTAB_METRIC_TYPES, + since: new Date(Date.parse(`${from}T00:00:00Z`)), + timeZone: tz, }) : [], prisma.user.findUnique({ @@ -200,10 +193,13 @@ export const GET = apiHandler(async () => { today, ); + // Each row stands for its day's readings: `value` is their sum and + // `count` their number, so `metricDayMap` gets the raw readings' day mean. const measurements: CrossMetricMeasurement[] = measurementRows.map((m) => ({ type: m.type, - value: m.value, - measuredAt: m.measuredAt, + value: m.sum, + count: m.n, + measuredAt: m.firstAt, source: m.source, deviceType: m.deviceType, })); diff --git a/tests/integration/cycle-insights-dense-reads.test.ts b/tests/integration/cycle-insights-dense-reads.test.ts new file mode 100644 index 000000000..545e1d24b --- /dev/null +++ b/tests/integration/cycle-insights-dense-reads.test.ts @@ -0,0 +1,264 @@ +/** + * `GET /api/cycle/insights` reads its outcome channels as per-source day + * aggregates, not as a year of readings. + * + * The route compared a year of resting heart rate, heart-rate variability, + * sleep, steps, weight, temperature and glucose across the cycle phases, and + * read every reading of it. A sensor reporting glucose every five minutes is + * over 100 000 readings in that year. The contrast only uses each local day's + * sum and count after one source and device is picked per day, which is what + * `readSourceDayAggregates` returns. + * + * Two properties on real Postgres: no outcome channel is read as rows, and the + * response is the one the raw readings give. The second runs the route twice, + * once over the SQL fold and once over every raw reading handed in as its own + * one-reading aggregate (exactly the old input), and compares the bodies. + */ +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { cookieJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { SourceDayAggregateRow } from "@/lib/measurements/day-aggregates"; +import { userDayKey } from "@/lib/tz/format"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +/** When set, the fold is replaced by every raw reading as a one-row group. */ +const mode = vi.hoisted(() => ({ raw: false })); + +vi.mock("@/lib/measurements/day-aggregates", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + readSourceDayAggregates: async ( + opts: Parameters[0], + ): Promise => { + if (!mode.raw) return actual.readSourceDayAggregates(opts); + const { getPrismaClient } = await import("./setup"); + const rows = await getPrismaClient().measurement.findMany({ + where: { + userId: opts.userId, + deletedAt: null, + type: { in: [...opts.types] }, + measuredAt: { gte: opts.since }, + }, + orderBy: { measuredAt: "asc" }, + select: { + type: true, + value: true, + measuredAt: true, + source: true, + deviceType: true, + }, + }); + return rows.map((r) => ({ + type: r.type, + day: userDayKey(r.measuredAt, opts.timeZone), + source: r.source, + deviceType: r.deviceType, + n: 1, + sum: r.value, + firstAt: r.measuredAt, + })); + }, + }; +}); + +const prisma = getPrismaClient(); +const OWNER = "cycle-dense-owner"; +const TZ = "Europe/Berlin"; +const CYCLE_LENGTH = 28; +const OBSERVED_CYCLES = 5; +const DAY = 86_400_000; + +function isoDay(offsetDays: number): string { + return new Date(Date.now() + offsetDays * DAY).toISOString().slice(0, 10); +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: OWNER, username: OWNER, gender: "FEMALE", timezone: TZ }, + }); + invalidateUserTimezone(OWNER); + await prisma.cycleProfile.create({ + data: { + userId: OWNER, + cycleTrackingEnabled: true, + predictionEnabled: false, + lutealPhaseLength: 14, + }, + }); + for (let i = OBSERVED_CYCLES; i >= 1; i--) { + await prisma.menstrualCycle.create({ + data: { + userId: OWNER, + startDate: isoDay(-i * CYCLE_LENGTH), + endDate: i > 1 ? isoDay(-(i - 1) * CYCLE_LENGTH - 1) : null, + periodEndDate: isoDay(-i * CYCLE_LENGTH + 4), + tz: "UTC", + }, + }); + } + const start = `${isoDay(-OBSERVED_CYCLES * CYCLE_LENGTH)} 00:00:00`; + // Sensor glucose every five minutes from two sources, higher in the second + // half of each cycle; heart-rate variability every twenty minutes from a + // watch, lower in the second half; steps every fifteen minutes from a watch + // and a phone; weight once a day next to the Berlin midnight. + await prisma.$executeRawUnsafe( + `INSERT INTO measurements (id, user_id, type, value, unit, source, measured_at, created_at, updated_at, sync_version) + SELECT 'cg' || s || '-' || g, $1, 'BLOOD_GLUCOSE'::measurement_type, + (CASE WHEN (g / 288) % 28 >= 14 THEN 120 ELSE 95 END) + (g % 23) + s * 0.5, 'mg/dL', + (CASE WHEN s = 0 THEN 'APPLE_HEALTH' ELSE 'NIGHTSCOUT' END)::measurement_source, + $2::timestamp + (g * interval '5 minutes') + (s * interval '1 second'), + now(), now(), 1 + FROM generate_series(0, ${OBSERVED_CYCLES * CYCLE_LENGTH} * 288 - 1) g, generate_series(0, 1) s`, + OWNER, + start, + ); + await prisma.$executeRawUnsafe( + `INSERT INTO measurements (id, user_id, type, value, unit, source, measured_at, created_at, updated_at, sync_version) + SELECT 'hv' || g, $1, 'HEART_RATE_VARIABILITY'::measurement_type, + (CASE WHEN (g / 72) % 28 >= 14 THEN 38 ELSE 52 END) + (g % 11) * 0.7, 'ms', + 'APPLE_HEALTH'::measurement_source, + $2::timestamp + (g * interval '20 minutes'), now(), now(), 1 + FROM generate_series(0, ${OBSERVED_CYCLES * CYCLE_LENGTH} * 72 - 1) g`, + OWNER, + start, + ); + await prisma.$executeRawUnsafe( + `INSERT INTO measurements (id, user_id, type, value, unit, source, device_type, measured_at, created_at, updated_at, sync_version) + SELECT 'st' || d || '-' || g, $1, 'ACTIVITY_STEPS'::measurement_type, 10 + (g % 7), 'count', + 'APPLE_HEALTH'::measurement_source, CASE WHEN d = 0 THEN 'watch' ELSE 'phone' END, + $2::timestamp + (g * interval '15 minutes') + (d * interval '2 seconds'), + now(), now(), 1 + FROM generate_series(0, ${OBSERVED_CYCLES * CYCLE_LENGTH} * 96 - 1) g, generate_series(0, 1) d + WHERE d = 0 OR g % 3 = 0`, + OWNER, + start, + ); + await prisma.$executeRawUnsafe( + `INSERT INTO measurements (id, user_id, type, value, unit, source, measured_at, created_at, updated_at, sync_version) + SELECT 'wt' || g, $1, 'WEIGHT'::measurement_type, + (CASE WHEN g % 28 >= 14 THEN 71.2 ELSE 70.1 END) + (g % 5) * 0.1, 'kg', + 'WITHINGS'::measurement_source, + $2::timestamp + (g * interval '1 day') + interval '22 hours 50 minutes', now(), now(), 1 + FROM generate_series(0, ${OBSERVED_CYCLES * CYCLE_LENGTH} - 1) g`, + OWNER, + start, + ); + await prisma.$executeRawUnsafe(`ANALYZE measurements`); + const session = await prisma.session.create({ + data: { userId: OWNER, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.clear(); + cookieJar.set("healthlog_session", session.id); +}, 180_000); + +async function readInsights(): Promise<{ status: number; body: unknown }> { + const { GET } = await import("@/app/api/cycle/insights/route"); + const handler = GET as unknown as (request: Request) => Promise; + const res = await handler(new Request("http://localhost/api/cycle/insights")); + return { status: res.status, body: await res.json() }; +} + +function readsOutcomeRows(args: unknown): boolean { + const type = (args as { where?: { type?: unknown } } | undefined)?.where + ?.type; + const list = (type as { in?: unknown } | undefined)?.in; + return ( + type === "BLOOD_GLUCOSE" || + type === "HEART_RATE_VARIABILITY" || + (Array.isArray(list) && + (list.includes("BLOOD_GLUCOSE") || + list.includes("HEART_RATE_VARIABILITY"))) + ); +} + +function expectSameUpToRounding( + actual: unknown, + expected: unknown, + path: string, +): void { + if (typeof expected === "number" && typeof actual === "number") { + const scale = Math.max(Math.abs(expected), Math.abs(actual)); + expect( + Math.abs(actual - expected) <= scale * 1e-9, + `${path}: ${actual} vs ${expected}`, + ).toBe(true); + return; + } + if (Array.isArray(expected)) { + expect(Array.isArray(actual), path).toBe(true); + expect((actual as unknown[]).length, path).toBe(expected.length); + expected.forEach((e, i) => + expectSameUpToRounding((actual as unknown[])[i], e, `${path}[${i}]`), + ); + return; + } + if (expected !== null && typeof expected === "object") { + expect(actual !== null && typeof actual === "object", path).toBe(true); + const a = actual as Record; + const e = expected as Record; + expect(Object.keys(a).sort(), path).toEqual(Object.keys(e).sort()); + for (const key of Object.keys(e)) { + expectSameUpToRounding(a[key], e[key], `${path}.${key}`); + } + return; + } + expect(actual, path).toEqual(expected); +} + +describe("cycle insights on a dense account", () => { + it("reads no outcome channel as rows", async () => { + mode.raw = false; + const spy = vi.spyOn(prisma.measurement, "findMany"); + try { + const started = performance.now(); + const { status } = await readInsights(); + process.stderr.write( + `[cycle-insights-dense] ${(performance.now() - started).toFixed(0)} ms\n`, + ); + expect(status).toBe(200); + expect(spy.mock.calls.filter(([a]) => readsOutcomeRows(a))).toEqual([]); + } finally { + spy.mockRestore(); + } + }, 60_000); + + it("answers exactly what the raw readings give", async () => { + mode.raw = false; + const folded = await readInsights(); + mode.raw = true; + const raw = await readInsights(); + mode.raw = false; + expect(folded.status).toBe(200); + const rows = (folded.body as { data: { rows: unknown[] } }).data.rows; + // The fixture is built so the contrast has something to say. + expect(rows.length).toBeGreaterThan(0); + // A day's sum is added in another order in SQL than in JavaScript, so a + // p-value can differ in its last bits; every figure is otherwise equal. + expectSameUpToRounding(folded.body, raw.body, "body"); + }, 120_000); +});