diff --git a/src/__tests__/pulse-day-mean-reader-guard.test.ts b/src/__tests__/pulse-day-mean-reader-guard.test.ts index 5e36ecbf8..0918eda6a 100644 --- a/src/__tests__/pulse-day-mean-reader-guard.test.ts +++ b/src/__tests__/pulse-day-mean-reader-guard.test.ts @@ -93,7 +93,6 @@ const HELPER: Entry = { helper: true }; /** Files with a database mean over values. */ const SQL_READERS: Record = { - "app/api/measurements/series/route.ts": HELPER, "lib/ai/coach/tools/availability.ts": HELPER, "lib/analytics/summaries-slice.ts": HELPER, "lib/doctor-report/dense-buckets.ts": HELPER, @@ -102,6 +101,8 @@ const SQL_READERS: Record = { // The dry run's hour means, weighed by both methods through the helper. "lib/jobs/pulse-day-mean.ts": HELPER, "lib/measurements/daily-series-read.ts": HELPER, + // The series route's long-window dense read. + "lib/measurements/dense-series-read.ts": HELPER, "lib/rollups/measurement-rollups.ts": HELPER, "lib/rollups/tiered-context.ts": HELPER, "lib/insights/correlation-channel-series.ts": { diff --git a/src/app/api/measurements/series/__tests__/route.test.ts b/src/app/api/measurements/series/__tests__/route.test.ts index ad9c05cf4..0742d921b 100644 --- a/src/app/api/measurements/series/__tests__/route.test.ts +++ b/src/app/api/measurements/series/__tests__/route.test.ts @@ -21,6 +21,51 @@ vi.mock("@/lib/db", () => ({ }, })); +// The route's dense read is one statement (`dense-series-read.ts`, held to +// the two statements it replaced by an integration parity test). Here it +// issues those two statements again, so the route's shaping of buckets and +// figures is pinned against the same queued results as before. +vi.mock("@/lib/measurements/dense-series-read", async (importOriginal) => { + const actual = + await importOriginal< + typeof import("@/lib/measurements/dense-series-read") + >(); + const { usesHourlyMeanDay } = + await import("@/lib/measurements/day-statistic"); + return { + ...actual, + readDenseSeries: async (args: { + userId: string; + type: Parameters[0]; + grain: "day" | "hour"; + userTz: string; + since: Date; + seriesRows: string; + }) => { + const { prisma } = await import("@/lib/db"); + const hourWeighted = args.grain === "day" && usesHourlyMeanDay(args.type); + const bucketRows = await prisma.$queryRawUnsafe( + actual.denseBucketsSql( + hourWeighted, + `${args.seriesRows} WHERE m."measured_at" >= $4`, + ), + args.userId, + args.grain, + args.userTz, + args.since, + ); + const [rawAgg] = (await prisma.$queryRawUnsafe( + `WITH src AS (SELECT m.* FROM ${args.seriesRows} WHERE m."measured_at" >= $4) ${actual.denseFiguresSql(usesHourlyMeanDay(args.type))}`, + args.userId, + args.grain, + args.userTz, + args.since, + )) as unknown[]; + return { bucketRows, rawAgg }; + }, + }; +}); + vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() })); // v1.11.4 — the sleep branch resolves the user's tz to bucket per-night; diff --git a/src/app/api/measurements/series/route.ts b/src/app/api/measurements/series/route.ts index c0e964111..5fccd4d6c 100644 --- a/src/app/api/measurements/series/route.ts +++ b/src/app/api/measurements/series/route.ts @@ -14,6 +14,7 @@ * window, judged by that direction) and `referenceLines` (the person's own * target lines, else the clinical ones, in the unit of the points). */ +import { readDenseSeries } from "@/lib/measurements/dense-series-read"; import { NextRequest } from "next/server"; import { z } from "zod/v4"; import { prisma } from "@/lib/db"; @@ -33,14 +34,7 @@ import { readSleepNights } from "@/lib/measurements/list/sleep-nights"; import { VALUE_RANGES } from "@/lib/validations/measurement"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; -import { - dayWeightedRows, - hourWeightSql, - readingsMean, - weightedHourMeanSql, - windowMeanSql, - zoneDayFrame, -} from "@/lib/measurements/day-mean"; +import { readingsMean } from "@/lib/measurements/day-mean"; import { seriesRowsFrom } from "@/lib/measurements/series-canonical"; import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; import { resolveUserTimezone, userDayKey } from "@/lib/tz/resolver"; @@ -461,135 +455,14 @@ export const GET = apiHandler(async (request: NextRequest) => { // weighted mean of its local hours' means, so a workout hour with hundreds // of times the readings does not outweigh the rest of the day (see // `day-statistic.ts`). The hour grain is already one mean per hour. - const hourWeighted = grain === "day" && usesHourlyMeanDay(type); - const bucketRows = await prisma.$queryRawUnsafe< - Array<{ - bucket_start: Date; - mean: number; - min_value: number; - max_value: number; - }> - >( - hourWeighted - ? ` - WITH localized AS ( - SELECT - m."value", - m."value_min", - m."value_max", - date_trunc( - 'hour', - (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE $3 - ) AS local_hour - FROM ${seriesRows} - WHERE m."measured_at" >= $4 - ), - hourly AS ( - SELECT - date_trunc($2, local_hour) AS local_day, - AVG("value") AS hour_mean, - COUNT(*) AS n, - MIN(COALESCE("value_min", "value")) AS min_value, - MAX(COALESCE("value_max", "value")) AS max_value - FROM localized - GROUP BY local_hour - ) - SELECT - local_day AT TIME ZONE $3 AS bucket_start, - ${weightedHourMeanSql("hour_mean", hourWeightSql("n"))}::double precision AS mean, - MIN(min_value)::double precision AS min_value, - MAX(max_value)::double precision AS max_value - FROM hourly - GROUP BY local_day - ORDER BY bucket_start ASC - ` - : ` - WITH localized AS ( - SELECT - m."value", - m."value_min", - m."value_max", - date_trunc( - $2, - (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE $3 - ) AS local_day - FROM ${seriesRows} - WHERE m."measured_at" >= $4 - ) - SELECT - local_day AT TIME ZONE $3 AS bucket_start, - AVG("value")::double precision AS mean, - MIN(COALESCE("value_min", "value"))::double precision AS min_value, - MAX(COALESCE("value_max", "value"))::double precision AS max_value - FROM localized - GROUP BY local_day - ORDER BY bucket_start ASC - `, - user.id, + const { bucketRows, rawAgg } = await readDenseSeries({ + userId: user.id, + type, grain, userTz, since, - ); - // Stats stay aggregated over the RAW rows (not the day buckets) so the - // strip's mean/min/max/stdDev/count are the same figures the raw path - // computed: `AVG`/`MIN`/`MAX` over `value`, `STDDEV_POP` matching the - // JS population-variance helper, exact row count. Aggregate-only — - // no row transfer. For an hourly-mean type (pulse) the mean is the mean - // of the local days' values, each day the mean of its local hours' - // means, so it is the mean of the day points the chart draws; count, - // min, max and the spread stay over every reading. - const [rawAgg] = usesHourlyMeanDay(type) - ? await prisma.$queryRawUnsafe< - Array<{ - n: number; - mean: number | null; - min: number | null; - max: number | null; - sd: number | null; - }> - >( - ` - WITH src AS ( - SELECT m.* - FROM ${seriesRows} - WHERE m."measured_at" >= $2 - ) - SELECT - COUNT(*)::int AS n, - ${windowMeanSql({ typeColumn: 'm."type"', value: 'm."value"', weight: "m.day_weight" })}::double precision - AS mean, - MIN(m."value")::double precision AS min, - MAX(m."value")::double precision AS max, - STDDEV_POP(m."value")::double precision AS sd - FROM ${dayWeightedRows("src", zoneDayFrame("$3"))} m - GROUP BY m."type" - `, - user.id, - since, - userTz, - ) - : await prisma.$queryRawUnsafe< - Array<{ - n: number; - mean: number | null; - min: number | null; - max: number | null; - sd: number | null; - }> - >( - ` - SELECT - COUNT(*)::int AS n, - AVG(m."value")::double precision AS mean, - MIN(m."value")::double precision AS min, - MAX(m."value")::double precision AS max, - STDDEV_POP(m."value")::double precision AS sd - FROM ${seriesRows} - WHERE m."measured_at" >= $2 - `, - user.id, - since, - ); + seriesRows, + }); const round2 = (v: number) => Math.round(v * 100) / 100; const dayId = (d: Date) => grain === "day" diff --git a/src/lib/measurements/dense-series-read.ts b/src/lib/measurements/dense-series-read.ts new file mode 100644 index 000000000..0160924c4 --- /dev/null +++ b/src/lib/measurements/dense-series-read.ts @@ -0,0 +1,195 @@ +/** + * The long-window read of a sample-dense series (pulse, CGM glucose, and the + * other dense kinds of `/api/measurements/series`): day or hour buckets in + * the account's zone, and the strip's figures over the raw readings. + * + * Both used to be separate statements over the same canonical rows, so a + * ten-year pulse read picked the canonical source per day and scanned the + * readings twice. Now the window's rows are materialised once (`src`) and + * both aggregates run over them in one statement, with exactly the SQL they + * had: `tests/integration/dense-series-read-parity.test.ts` holds the two old + * statements as the reference. + * + * Buckets: a day of a type with an activity-dependent sampling rate (pulse) + * is the weighted mean of its local hours' means, so a workout hour with + * hundreds of times the readings does not outweigh the rest of the day (see + * `day-statistic.ts`); the hour grain is already one mean per hour. + * + * Figures: aggregated over the raw rows, not the buckets, so the strip's + * mean, min, max, spread and count are what the raw path computes (`AVG`, + * `MIN`, `MAX`, `STDDEV_POP`, the exact row count). For an hourly-mean type + * the mean is the mean of the local days' values, each day the mean of its + * local hours' means, so it is the mean of the day points the chart draws; + * count, min, max and the spread stay over every reading. + * + * Parameters: `$1` the user (inside `seriesRows`), `$2` the grain, `$3` the + * zone, `$4` the window start; `seriesRows` is the FROM-clause subquery of + * `seriesRowsFrom`, its literals whitelisted there. + */ +import type { MeasurementType } from "@/generated/prisma/client"; +import { prisma } from "@/lib/db"; +import { + dayWeightedRows, + hourWeightSql, + weightedHourMeanSql, + windowMeanSql, + zoneDayFrame, +} from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; + +export interface DenseBucketRow { + bucket_start: Date; + mean: number; + min_value: number; + max_value: number; +} + +export interface DenseFigures { + n: number; + mean: number | null; + min: number | null; + max: number | null; + sd: number | null; +} + +/** The bucket aggregate over `src`, in the original statement's SQL. */ +export function denseBucketsSql(hourWeighted: boolean, from: string): string { + return hourWeighted + ? ` + WITH localized AS ( + SELECT + m."value", + m."value_min", + m."value_max", + date_trunc( + 'hour', + (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE $3 + ) AS local_hour + FROM ${from} + ), + hourly AS ( + SELECT + date_trunc($2, local_hour) AS local_day, + AVG("value") AS hour_mean, + COUNT(*) AS n, + MIN(COALESCE("value_min", "value")) AS min_value, + MAX(COALESCE("value_max", "value")) AS max_value + FROM localized + GROUP BY local_hour + ) + SELECT + local_day AT TIME ZONE $3 AS bucket_start, + ${weightedHourMeanSql("hour_mean", hourWeightSql("n"))}::double precision AS mean, + MIN(min_value)::double precision AS min_value, + MAX(max_value)::double precision AS max_value + FROM hourly + GROUP BY local_day` + : ` + WITH localized AS ( + SELECT + m."value", + m."value_min", + m."value_max", + date_trunc( + $2, + (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE $3 + ) AS local_day + FROM ${from} + ) + SELECT + local_day AT TIME ZONE $3 AS bucket_start, + AVG("value")::double precision AS mean, + MIN(COALESCE("value_min", "value"))::double precision AS min_value, + MAX(COALESCE("value_max", "value"))::double precision AS max_value + FROM localized + GROUP BY local_day`; +} + +/** The figures aggregate over the CTE `src`, in the original SQL. */ +export function denseFiguresSql(hourlyMean: boolean): string { + return hourlyMean + ? ` + SELECT + COUNT(*)::int AS n, + ${windowMeanSql({ typeColumn: 'm."type"', value: 'm."value"', weight: "m.day_weight" })}::double precision + AS mean, + MIN(m."value")::double precision AS min, + MAX(m."value")::double precision AS max, + STDDEV_POP(m."value")::double precision AS sd + FROM ${dayWeightedRows("src", zoneDayFrame("$3"))} m + GROUP BY m."type"` + : ` + SELECT + COUNT(*)::int AS n, + AVG(m."value")::double precision AS mean, + MIN(m."value")::double precision AS min, + MAX(m."value")::double precision AS max, + STDDEV_POP(m."value")::double precision AS sd + FROM src m`; +} + +/** Buckets and figures of one dense series window, in one statement. */ +export async function readDenseSeries(args: { + userId: string; + type: MeasurementType; + grain: "day" | "hour"; + userTz: string; + since: Date; + seriesRows: string; +}): Promise<{ + bucketRows: DenseBucketRow[]; + rawAgg: DenseFigures | undefined; +}> { + const hourWeighted = args.grain === "day" && usesHourlyMeanDay(args.type); + const rows = await prisma.$queryRawUnsafe< + Array< + { kind: "bucket" | "figures" } & Partial & + Partial + > + >( + ` + WITH src AS MATERIALIZED ( + SELECT m.* + FROM ${args.seriesRows} + WHERE m."measured_at" >= $4 + ), + buckets AS (${denseBucketsSql(hourWeighted, "src m")} + ), + figures AS (${denseFiguresSql(usesHourlyMeanDay(args.type))} + ) + SELECT 'bucket' AS kind, b.bucket_start, b.mean, b.min_value, b.max_value, + NULL::int AS n, NULL::double precision AS min, + NULL::double precision AS max, NULL::double precision AS sd + FROM buckets b + UNION ALL + SELECT 'figures', NULL, f.mean, NULL, NULL, f.n, f.min, f.max, f.sd + FROM figures f + ORDER BY kind ASC, bucket_start ASC + `, + args.userId, + args.grain, + args.userTz, + args.since, + ); + const bucketRows: DenseBucketRow[] = []; + let rawAgg: DenseFigures | undefined; + for (const row of rows) { + if (row.kind === "bucket") { + bucketRows.push({ + bucket_start: row.bucket_start!, + mean: row.mean!, + min_value: row.min_value!, + max_value: row.max_value!, + }); + } else { + rawAgg = { + n: row.n!, + mean: row.mean ?? null, + min: row.min ?? null, + max: row.max ?? null, + sd: row.sd ?? null, + }; + } + } + return { bucketRows, rawAgg }; +} diff --git a/tests/integration/dense-series-read-parity.test.ts b/tests/integration/dense-series-read-parity.test.ts new file mode 100644 index 000000000..09f7c745b --- /dev/null +++ b/tests/integration/dense-series-read-parity.test.ts @@ -0,0 +1,209 @@ +/** + * The dense-series read, against a real Postgres: the shipped single + * statement (`readDenseSeries`) against the two statements it replaced, + * kept below as the reference (the bucket SQL and the figures SQL are the + * same builders, so the reference differs only in reading the canonical rows + * twice instead of once). + * + * Buckets and figures must match for every shape the route asks for: day + * and hour grains; an hourly-mean type (pulse, two ranked sources), an + * unranked dense type (glucose) and a ranked non-hourly one (oxygen + * saturation); zones with a DST change and a half-hour offset; a fixture + * with gaps, tombstones and readings around local and UTC midnight. + */ +import { beforeAll, describe, expect, it } from "vitest"; + +import type { + MeasurementSource, + MeasurementType, +} from "@/generated/prisma/client"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; +import { + denseBucketsSql, + denseFiguresSql, + readDenseSeries, + type DenseBucketRow, + type DenseFigures, +} from "@/lib/measurements/dense-series-read"; +import { seriesRowsFrom } from "@/lib/measurements/series-canonical"; + +import { makeUser } from "../kit/factories"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +const prisma = getPrismaClient(); +const DAY_MS = 86_400_000; + +/** The two statements as the route sent them before. */ +async function legacyDenseSeries(args: { + userId: string; + type: MeasurementType; + grain: "day" | "hour"; + userTz: string; + since: Date; + seriesRows: string; +}) { + const hourWeighted = args.grain === "day" && usesHourlyMeanDay(args.type); + const bucketRows = await prisma.$queryRawUnsafe( + `${denseBucketsSql(hourWeighted, `${args.seriesRows} WHERE m."measured_at" >= $4`)} + ORDER BY bucket_start ASC`, + args.userId, + args.grain, + args.userTz, + args.since, + ); + // The figures statement took the window start as `$2` and the zone as + // `$3`, the latter only for an hourly-mean type. + const hourly = usesHourlyMeanDay(args.type); + const [rawAgg] = await prisma.$queryRawUnsafe( + `WITH src AS (SELECT m.* FROM ${args.seriesRows} WHERE m."measured_at" >= $2) + ${denseFiguresSql(hourly)}`, + args.userId, + args.since, + ...(hourly ? [args.userTz] : []), + ); + return { bucketRows, rawAgg }; +} + +function rng(seed: number) { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +let userId = ""; + +async function seed() { + userId = (await makeUser({ timezone: "Europe/Berlin" })).id; + const random = rng(7); + const now = Date.now(); + const rows: Array<{ + userId: string; + type: MeasurementType; + source: MeasurementSource; + value: number; + valueMin: number | null; + valueMax: number | null; + unit: string; + measuredAt: Date; + deletedAt: Date | null; + }> = []; + const plan: Array<[MeasurementType, MeasurementSource[], number]> = [ + ["PULSE", ["APPLE_HEALTH", "WITHINGS"], 60], + ["BLOOD_GLUCOSE", ["NIGHTSCOUT", "MANUAL"], 70], + ["OXYGEN_SATURATION", ["APPLE_HEALTH", "WITHINGS"], 96], + ]; + // 200 days: across the last two DST changes, with gaps. + for (let day = 0; day < 200; day++) { + if (day % 13 === 4) continue; + const dayStart = Math.floor((now - day * DAY_MS) / DAY_MS) * DAY_MS; + for (const [type, sources, base] of plan) { + for (const source of sources) { + if (random() < 0.3) continue; + const readings = 3 + Math.floor(random() * 12); + for (let r = 0; r < readings; r++) { + const at = dayStart + Math.floor(random() * DAY_MS); + const value = Math.round((base + random() * 30) * 10) / 10; + const banded = type === "PULSE" && random() < 0.3; + rows.push({ + userId, + type, + source, + value, + valueMin: banded ? value - 10 : null, + valueMax: banded ? value + 15 : null, + unit: "x", + measuredAt: new Date(at), + deletedAt: random() < 0.05 ? new Date() : null, + }); + } + } + } + } + const seen = new Set(); + await prisma.measurement.createMany({ + data: rows.filter((row) => { + const key = `${row.type}|${row.source}|${row.measuredAt.getTime()}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }), + }); +} + +const CASES: Array<[MeasurementType, "day" | "hour", number]> = [ + ["PULSE", "day", 180], + ["PULSE", "hour", 60], + ["BLOOD_GLUCOSE", "day", 180], + ["BLOOD_GLUCOSE", "hour", 30], + ["OXYGEN_SATURATION", "day", 120], +]; +const ZONES = ["UTC", "Europe/Berlin", "Asia/Kolkata", "America/New_York"]; + +function close(a: number | null | undefined, b: number | null | undefined) { + if (a == null || b == null) return expect(a ?? null).toBe(b ?? null); + expect(Math.abs(a - b)).toBeLessThanOrEqual(1e-9 * Math.max(1, Math.abs(b))); +} + +describe("dense-series read parity", () => { + beforeAll(async () => { + await truncateAllTables(prisma); + await seed(); + }); + + for (const [type, grain, days] of CASES) { + for (const userTz of ZONES) { + it(`matches the two statements: ${type}, ${grain}, ${days} d, ${userTz}`, async () => { + const args = { + userId, + type, + grain, + userTz, + since: new Date(Date.now() - days * DAY_MS), + seriesRows: seriesRowsFrom(null, type, days), + }; + const [now, before] = await Promise.all([ + readDenseSeries(args), + legacyDenseSeries(args), + ]); + expect(before.bucketRows.length).toBeGreaterThan(10); + expect(now.bucketRows.map((r) => r.bucket_start.getTime())).toEqual( + before.bucketRows.map((r) => r.bucket_start.getTime()), + ); + now.bucketRows.forEach((row, i) => { + const ref = before.bucketRows[i]!; + close(row.mean, ref.mean); + close(row.min_value, ref.min_value); + close(row.max_value, ref.max_value); + }); + expect(now.rawAgg?.n).toBe(before.rawAgg?.n); + close(now.rawAgg?.mean, before.rawAgg?.mean); + close(now.rawAgg?.min, before.rawAgg?.min); + close(now.rawAgg?.max, before.rawAgg?.max); + close(now.rawAgg?.sd, before.rawAgg?.sd); + }); + } + } + + it("answers no figures row for an hourly-mean type without readings, as before", async () => { + const empty = await makeUser(); + const args = { + userId: empty.id, + type: "PULSE" as const, + grain: "day" as const, + userTz: "UTC", + since: new Date(Date.now() - 365 * DAY_MS), + seriesRows: seriesRowsFrom(null, "PULSE", 365), + }; + const [now, before] = await Promise.all([ + readDenseSeries(args), + legacyDenseSeries(args), + ]); + expect(now).toEqual(before); + expect(now.rawAgg).toBeUndefined(); + }); +});