diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index ed7a4c391..b89b1bf48 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -40910,8 +40910,8 @@ components: id: type: string description: Measurement row id; `sleep:` for a sleep night; `day:` for a day bucket (`pulse` / - `glucose` beyond 90 days); `hour:` for an hour bucket (`pulse` within 90 days when the - window holds more than 10 000 readings). + `glucose` beyond 90 days); `hour:` for an hour bucket (`pulse` / `glucose` within 90 days + when the window holds more than 10 000 readings). at: type: string format: date-time @@ -40985,9 +40985,11 @@ components: additionalProperties: false description: "iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an explicit `unit` token, and a summary `stats` block. Sample-dense kinds are bucketed server-side: `pulse` and - `glucose` beyond 90 days per local day, and `pulse` within 90 days per local hour when the window holds more - than 10 000 readings (a bucket carries the mean as `value` and its range as `valueMin` / `valueMax`). `stats` is - always computed over the underlying readings, never over the buckets." + `glucose` beyond 90 days per local day, and `pulse` / `glucose` within 90 days per local hour when the window + holds more than 10 000 readings (a continuous glucose sensor, a watch recording heart rate every minute; a cuff + or fingerstick meter stays raw). A bucket carries the mean as `value`; a `pulse` bucket also carries its range + as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. `stats` + is always computed over the underlying readings, never over the buckets." ListMeasurementsResponse: type: object properties: diff --git a/src/app/api/measurements/series/__tests__/route.test.ts b/src/app/api/measurements/series/__tests__/route.test.ts index de6f8dced..cd220128a 100644 --- a/src/app/api/measurements/series/__tests__/route.test.ts +++ b/src/app/api/measurements/series/__tests__/route.test.ts @@ -631,6 +631,85 @@ describe("GET /api/measurements/series — dense-kind day-bucketing (v1.28.25)", expect(bucketSql.slice(1)).toContain("hour"); }); + it("hour-buckets a CGM glucose window too dense to send raw, in the user's unit", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + glucoseUnit: "mmol/L", + } as never); + // A sensor reading every five minutes for 90 days. + vi.mocked(prisma.measurement.count).mockResolvedValue(25_920 as never); + vi.mocked(prisma.$queryRaw) + .mockResolvedValueOnce([ + { + bucket_start: new Date("2026-06-01T08:00:00Z"), + mean: 100, + min_value: 90, + max_value: 112, + }, + ] as never) + .mockResolvedValueOnce([ + { n: 25_920, mean: 118.004, min: 48, max: 260, sd: 36.004 }, + ] as never); + + const res = await GET(req("kind=glucose&days=90")); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { + unit: string; + points: Array>; + stats: { + count: number; + mean: number; + min: number; + max: number; + stdDev: number; + }; + }; + }; + expect(prisma.measurement.findMany).not.toHaveBeenCalled(); + expect(vi.mocked(prisma.measurement.count).mock.calls[0]?.[0]).toEqual( + expect.objectContaining({ + where: expect.objectContaining({ type: "BLOOD_GLUCOSE" }), + }), + ); + expect(body.data.unit).toBe("mmol/L"); + // Converted once at serialization; no band on a glucose point. + expect(body.data.points).toEqual([ + { + id: "hour:2026-06-01T08:00:00.000Z", + at: "2026-06-01T08:00:00.000Z", + value: 5.5, + secondary: null, + }, + ]); + // Stats describe every reading (mg/dL aggregate, converted once). + expect(body.data.stats).toEqual({ + count: 25_920, + mean: 6.5, + min: 2.7, + max: 14.4, + stdDev: 2, + }); + const bucketSql = vi.mocked(prisma.$queryRaw).mock.calls[0] as unknown[]; + expect(bucketSql.slice(1)).toContain("hour"); + }); + + it("keeps fingerstick glucose at or under the row cap raw", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.measurement.count).mockResolvedValue(360 as never); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([ + { id: "g1", value: 104, measuredAt: new Date("2026-06-01T07:12:00Z") }, + ] as never); + const res = await GET(req("kind=glucose&days=90")); + const body = (await res.json()) as { + data: { points: Array> }; + }; + expect(prisma.$queryRaw).not.toHaveBeenCalled(); + expect(body.data.points).toEqual([ + { id: "g1", at: "2026-06-01T07:12:00.000Z", value: 104, secondary: null }, + ]); + }); + it("keeps a pulse window at or under the row cap raw", async () => { vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); vi.mocked(prisma.measurement.count).mockResolvedValue(10_000 as never); diff --git a/src/app/api/measurements/series/route.ts b/src/app/api/measurements/series/route.ts index 80d5ed035..06d46e07a 100644 --- a/src/app/api/measurements/series/route.ts +++ b/src/app/api/measurements/series/route.ts @@ -52,17 +52,20 @@ const DENSE_SERIES_RAW_WINDOW_DAYS = 90; const DENSE_SERIES_KINDS: ReadonlySet = new Set(["glucose", "pulse"]); /** - * #1023 — inside the raw window, a pulse series with more rows than this is - * bucketed per local hour in SQL instead of sent raw. + * #1023 — inside the raw window, a pulse or glucose series with more rows + * than this is bucketed per local hour in SQL instead of sent raw. * * A watch that records heart rate once a minute puts 43 000 rows in the * default 30-day window and 130 000 in 90 days: a 15 MB response the client * could not draw at that resolution anyway, built from as many objects on the - * server. Hourly buckets carry the hour's mean with its low/high band, the - * shape an hourly-bucket import already has, and cap the answer at 24 points - * a day. The cap sits far above what sparse sources produce (a cuff, hourly - * buckets over 90 days, a watch sampling every five minutes over 30 days), so - * those series stay raw and unchanged. + * server. A CGM reading every five minutes puts 26 000 glucose rows in 90 + * days, the same problem at a smaller scale. Hourly buckets carry the hour's + * mean (pulse also its low/high band, the shape an hourly-bucket import + * already has) and cap the answer at 24 points a day; the statistics stay + * computed over every raw reading. The cap sits far above what sparse sources + * produce (a cuff, fingerstick glucose, hourly buckets over 90 days, a watch + * sampling every five minutes over 30 days), so those series stay raw and + * unchanged. */ const DENSE_SERIES_RAW_ROW_CAP = 10_000; @@ -368,18 +371,18 @@ export const GET = apiHandler(async (request: NextRequest) => { }); } else if ( (DENSE_SERIES_KINDS.has(kind) && days > DENSE_SERIES_RAW_WINDOW_DAYS) || - (kind === "pulse" && + (DENSE_SERIES_KINDS.has(kind) && (await prisma.measurement.count({ where: { userId: user.id, - type: "PULSE", + type: KIND_TO_TYPE[kind], measuredAt: { gte: since }, deletedAt: null, }, })) > DENSE_SERIES_RAW_ROW_CAP) ) { - // Day buckets past the raw window; hour buckets for a pulse stream too - // dense to send raw inside it. + // Day buckets past the raw window; hour buckets for a pulse or CGM + // stream too dense to send raw inside it. const grain = days > DENSE_SERIES_RAW_WINDOW_DAYS ? "day" : "hour"; // v1.28.25 — long-window read of a sample-dense kind (CGM glucose, // per-sample / hourly pulse). Day-bucket in SQL instead of walking diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index e36506223..0795e88b1 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -696,7 +696,7 @@ const seriesPointSchema = z.object({ id: z .string() .describe( - "Measurement row id; `sleep:` for a sleep night; `day:` for a day bucket (`pulse` / `glucose` beyond 90 days); `hour:` for an hour bucket (`pulse` within 90 days when the window holds more than 10 000 readings).", + "Measurement row id; `sleep:` for a sleep night; `day:` for a day bucket (`pulse` / `glucose` beyond 90 days); `hour:` for an hour bucket (`pulse` / `glucose` within 90 days when the window holds more than 10 000 readings).", ), at: z.iso.datetime({ offset: true }).describe("Point timestamp (ISO-8601)."), value: z.number().describe("Primary value in the top-level `unit`."), @@ -751,7 +751,7 @@ const seriesResponse = z .meta({ id: "MeasurementsSeriesResponse", description: - "iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an explicit `unit` token, and a summary `stats` block. Sample-dense kinds are bucketed server-side: `pulse` and `glucose` beyond 90 days per local day, and `pulse` within 90 days per local hour when the window holds more than 10 000 readings (a bucket carries the mean as `value` and its range as `valueMin` / `valueMax`). `stats` is always computed over the underlying readings, never over the buckets.", + "iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an explicit `unit` token, and a summary `stats` block. Sample-dense kinds are bucketed server-side: `pulse` and `glucose` beyond 90 days per local day, and `pulse` / `glucose` within 90 days per local hour when the window holds more than 10 000 readings (a continuous glucose sensor, a watch recording heart rate every minute; a cuff or fingerstick meter stays raw). A bucket carries the mean as `value`; a `pulse` bucket also carries its range as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. `stats` is always computed over the underlying readings, never over the buckets.", }); // ── Batched daily series (v1.18.6 dashboard fetch coalescing) ──────── diff --git a/tests/integration/dense-glucose-raw-window-series.test.ts b/tests/integration/dense-glucose-raw-window-series.test.ts new file mode 100644 index 000000000..ceb6991a6 --- /dev/null +++ b/tests/integration/dense-glucose-raw-window-series.test.ts @@ -0,0 +1,238 @@ +/** + * The per-kind glucose series inside the 90-day raw window, on a continuous + * sensor stream. + * + * Past 90 days glucose already folded per day in SQL. Inside it, a sensor + * reading every five minutes sent every row raw: 26 000 points in 90 days, + * the same class as the per-minute heart-rate stream in + * `dense-pulse-raw-window-reads.test.ts`. Over the row cap the series now + * folds per local hour; a fingerstick meter stays far under it and keeps its + * exact old shape. + * + * The doctor report deliberately keeps glucose raw inside 90 days: its + * clinical panel (time in range, GMI, CV and the risk indices) is computed + * from the raw readings of every source, and the day path computes it from + * canonical-source day buckets, which is not the same figure when a meter and + * a sensor overlap. The last case pins that the panel is still the raw + * engine's answer. + */ +import { NextRequest } from "next/server"; +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { cookieJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import { computeGlucoseClinicalMetrics } from "@/lib/analytics/glucose-metrics"; +import { collectDoctorReportData } from "@/lib/doctor-report-data"; +import { convertGlucose } from "@/lib/glucose"; +import { selectionFromLeaves } from "@/lib/report-selection/selection"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const TZ = "America/New_York"; +const DENSE = "dense-glucose-owner"; +const SPARSE = "sparse-glucose-owner"; +/** Forty days of one reading every five minutes: over the raw-window cap. */ +const DENSE_ROWS = 40 * 288; +const NOW = new Date(Math.floor(Date.now() / 300_000) * 300_000 - 300_000); + +/** A fingerstick meter: a handful of readings a day at irregular times. */ +function sparseReadings(): Array<{ at: Date; value: number }> { + const out: Array<{ at: Date; value: number }> = []; + for (let day = 1; day <= 30; day++) { + const perDay = 2 + (day % 4); + for (let i = 0; i < perDay; i++) { + out.push({ + at: new Date( + NOW.getTime() - day * 86_400_000 + (i * 211 + day * 17) * 60_000, + ), + value: 78 + ((day * 13 + i * 29) % 120), + }); + } + } + return out; +} + +async function session(userId: string): Promise { + const s = await prisma.session.create({ + data: { userId, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.clear(); + cookieJar.set("healthlog_session", s.id); +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: DENSE, username: DENSE, timezone: TZ, glucoseUnit: "mmol/L" }, + }); + await prisma.user.create({ + data: { id: SPARSE, username: SPARSE, timezone: TZ }, + }); + invalidateUserTimezone(DENSE); + invalidateUserTimezone(SPARSE); + await prisma.$executeRawUnsafe( + `INSERT INTO measurements ( + id, user_id, type, value, unit, source, measured_at, + created_at, updated_at, sync_version) + SELECT 'cg' || lpad(g::text, 7, '0'), $1, 'BLOOD_GLUCOSE'::measurement_type, + 62 + ((g * 37) % 170), 'mg/dL', 'NIGHTSCOUT'::measurement_source, + ($2::timestamptz AT TIME ZONE 'UTC') - (g * interval '5 minutes'), + now(), now(), 1 + FROM generate_series(0, $3::int - 1) g`, + DENSE, + NOW.toISOString(), + DENSE_ROWS, + ); + await prisma.measurement.createMany({ + data: sparseReadings().map((r, i) => ({ + id: `fs${String(i).padStart(4, "0")}`, + userId: SPARSE, + type: "BLOOD_GLUCOSE" as const, + value: r.value, + unit: "mg/dL", + source: "MANUAL" as const, + measuredAt: r.at, + })), + }); + await prisma.$executeRawUnsafe(`ANALYZE measurements`); +}, 120_000); + +async function series(userId: string, days: number) { + await session(userId); + const { GET } = await import("@/app/api/measurements/series/route"); + const res = await GET( + new NextRequest( + `http://localhost/api/measurements/series?kind=glucose&days=${days}`, + ), + ); + expect(res.status).toBe(200); + return ( + (await res.json()) as { + data: { + unit: string; + points: Array & { id: string; at: string }>; + stats: { + count: number; + mean: number; + min: number; + max: number; + stdDev: number; + }; + }; + } + ).data; +} + +describe("per-kind glucose series inside the raw window", () => { + it("sends a sensor stream as hour buckets in the user's unit, statistics over every reading", async () => { + const data = await series(DENSE, 90); + const rows = await prisma.measurement.findMany({ + where: { userId: DENSE, type: "BLOOD_GLUCOSE" }, + select: { value: true, measuredAt: true }, + }); + expect(rows).toHaveLength(DENSE_ROWS); + + // Forty days of hours, give or take the partial hours at either end. + expect(data.points.length).toBeLessThanOrEqual(40 * 24 + 2); + expect(data.points.length).toBeGreaterThanOrEqual(40 * 24 - 2); + expect(data.points.every((p) => p.id.startsWith("hour:"))).toBe(true); + expect(data.unit).toBe("mmol/L"); + + // Every bucket is the mean of its hour's readings, converted once. New + // York sits a whole number of hours off UTC, so its local hours are UTC + // hours. + const byHour = new Map(); + for (const r of rows) { + const hour = Math.floor(r.measuredAt.getTime() / 3_600_000) * 3_600_000; + (byHour.get(hour) ?? byHour.set(hour, []).get(hour)!).push(r.value); + } + for (const p of data.points) { + const values = byHour.get(new Date(p.at).getTime()); + expect(values, p.at).toBeDefined(); + const mean = values!.reduce((a, b) => a + b, 0) / values!.length; + expect(p).toEqual({ + id: `hour:${p.at}`, + at: p.at, + value: convertGlucose(mean, "mmol/L"), + secondary: null, + }); + } + + const values = rows.map((r) => r.value); + const mean = values.reduce((a, b) => a + b, 0) / values.length; + expect(data.stats.count).toBe(DENSE_ROWS); + expect(data.stats.min).toBe(convertGlucose(Math.min(...values), "mmol/L")); + expect(data.stats.max).toBe(convertGlucose(Math.max(...values), "mmol/L")); + expect(data.stats.mean).toBe( + convertGlucose(Math.round(mean * 100) / 100, "mmol/L"), + ); + }); + + it("keeps fingerstick readings raw, one point per reading", async () => { + const data = await series(SPARSE, 90); + const expected = sparseReadings() + .map((r, i) => ({ + id: `fs${String(i).padStart(4, "0")}`, + at: r.at.toISOString(), + value: r.value, + secondary: null, + })) + .sort((a, b) => a.at.localeCompare(b.at)); + expect(data.unit).toBe("mg/dL"); + expect(data.points).toEqual(expected); + expect(data.stats.count).toBe(expected.length); + }); +}); + +describe("doctor report keeps glucose raw inside the raw window", () => { + it("computes the clinical panel from every raw reading", async () => { + const end = new Date(); + const start = new Date(end.getTime() - 60 * 86_400_000); + const data = await collectDoctorReportData( + DENSE, + { start, end, days: 60 }, + selectionFromLeaves(["BLOOD_GLUCOSE", "GLUCOSE_PANEL"]), + ); + const rows = await prisma.measurement.findMany({ + where: { + userId: DENSE, + type: "BLOOD_GLUCOSE", + measuredAt: { gte: start, lte: end }, + }, + orderBy: { measuredAt: "asc" }, + select: { value: true, measuredAt: true }, + }); + expect(data.measurements.BLOOD_GLUCOSE).toHaveLength(rows.length); + expect(data.glucoseClinical).toEqual( + computeGlucoseClinicalMetrics( + rows.map((r) => ({ measuredAt: r.measuredAt, mgdl: r.value })), + { windowDays: 60, now: end }, + ), + ); + }); +});