From 05ae8b1e995fc408f54653ade00276b944fcf2bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:27:32 +0200 Subject: [PATCH] perf(dashboard): count all-time readings from the DAY rollups The summary's all-time aggregate grouped every live reading of the account by type: one index entry per reading, millions on an account with a watch, two to four seconds whenever they were not in memory and the slowest read of the request on every cache miss. The count now sums the DAY buckets that already hold a per-day count of every source, counts live only the readings before the fold window and any type not folded yet, and takes lastSeenAt from the newest reading the latest-reading read already returns. Parity with the live count is pinned against a real Postgres. --- .../dashboard-summary-read-plan-guard.test.ts | 2 +- .../dense-measurement-read-guard.test.ts | 2 + .../dashboard/summary/__tests__/route.test.ts | 85 ++++++++--------- src/app/api/dashboard/summary/route.ts | 25 ++--- src/lib/dashboard/summary-reads.ts | 77 ++++++++++++++++ .../dashboard-summary-reads.test.ts | 92 +++++++++++++++++++ 6 files changed, 229 insertions(+), 54 deletions(-) diff --git a/src/__tests__/dashboard-summary-read-plan-guard.test.ts b/src/__tests__/dashboard-summary-read-plan-guard.test.ts index b1b9a14e1..ce7035eb5 100644 --- a/src/__tests__/dashboard-summary-read-plan-guard.test.ts +++ b/src/__tests__/dashboard-summary-read-plan-guard.test.ts @@ -85,7 +85,7 @@ describe("dashboard summary reads — sparkline rollup buckets", () => { // defeat statement reuse. const sql = code(READS); const casts = sql.match(/\}::"measurement_type"\[\]/g) ?? []; - expect(casts.length).toBe(2); + expect(casts.length).toBe(3); expect(sql).not.toMatch(/\$queryRawUnsafe/); }); }); diff --git a/src/__tests__/dense-measurement-read-guard.test.ts b/src/__tests__/dense-measurement-read-guard.test.ts index 75c083e73..07c9d441b 100644 --- a/src/__tests__/dense-measurement-read-guard.test.ts +++ b/src/__tests__/dense-measurement-read-guard.test.ts @@ -134,6 +134,8 @@ const ALLOWED_SQL: Record = { "lib/rollups/measurement-coverage.ts::1": "Selects from `unnest` of the caller's type list; `measurements` is read inside `EXISTS`, one row per type at most.", "lib/dashboard/summary-reads.ts::1": + "Selects from `unnest` of the type list; `measurements` is read only inside COUNT scalar subqueries, before the fold window for a type the DAY rollups cover.", + "lib/dashboard/summary-reads.ts::2": "Selects from `generate_series` of the streak window's local days; `measurements` is read inside `EXISTS`, one row per day at most.", }; diff --git a/src/app/api/dashboard/summary/__tests__/route.test.ts b/src/app/api/dashboard/summary/__tests__/route.test.ts index b74b91c07..c6fd2ca31 100644 --- a/src/app/api/dashboard/summary/__tests__/route.test.ts +++ b/src/app/api/dashboard/summary/__tests__/route.test.ts @@ -78,6 +78,11 @@ vi.mock("@/lib/rollups/measurement-read", () => ({ // covered by `src/lib/analytics/__tests__/mood-series.test.ts`. Pinned to an // empty series by default so the legacy assertions keep their "no mood" // expectation, and overridden per test below. +vi.mock("@/lib/dashboard/summary-reads", async (importOriginal) => ({ + ...(await importOriginal()), + readAllTimeCounts: vi.fn(), +})); + vi.mock("@/lib/analytics/mood-series", () => ({ buildMoodDailySeries: vi.fn(async () => ({ entries: [], @@ -106,6 +111,7 @@ import { getSession } from "@/lib/auth/session"; import { __resetAllCachesForTests } from "@/lib/cache/server-cache"; import { buildMoodDailySeries } from "@/lib/analytics/mood-series"; import { userDayKey } from "@/lib/tz/format"; +import { readAllTimeCounts } from "@/lib/dashboard/summary-reads"; const SESSION_OK = { session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, @@ -133,10 +139,10 @@ beforeEach(() => { // between tests to keep cross-test isolation. __resetAllCachesForTests(); vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); - // v1.4.33 — the route now issues a `groupBy` for - // per-type all-time count + most-recent timestamp. Default to an - // empty aggregate so legacy tests keep their "no data" expectations. - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([] as never); + // The per-type all-time counts (`readAllTimeCounts`). Default to none so + // legacy tests keep their "no data" expectations; the newest reading per + // type, which `lastSeenAt` carries, comes from the latest-reading read. + vi.mocked(readAllTimeCounts).mockResolvedValue([] as never); // v1.11.4 — the sleep tile reads the raw per-stage SLEEP_DURATION rows // via `measurement.findMany` to reconstruct the night total. Default to // no rows so legacy tests keep their "no sleep" expectation. @@ -249,22 +255,26 @@ describe("GET /api/dashboard/summary", () => { } }); - it("surfaces lastSeenAt from the historical aggregate even when the 7-day window is empty", async () => { + it("surfaces lastSeenAt from the newest reading even when the 7-day window is empty", async () => { // Power-user path — the user has logged weight for years but - // hasn't touched the app in two weeks. `groupBy` returns the - // all-time count + most-recent timestamp; the 7-day `findMany` - // returns empty. The route should still emit the weight card + // hasn't touched the app in two weeks. The all-time count and the + // newest reading come back; the sparkline is empty. The route should + // still emit the weight card // with the historical `lastSeenAt` so the iOS client renders the // staleness caption. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const twoWeeksAgo = new Date(Date.now() - 14 * 86_400_000); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "WEIGHT", - _count: { _all: 312 }, - _max: { measuredAt: twoWeeksAgo }, + count: 312, }, ] as never); + // The newest reading, which `lastSeenAt` carries (first `$queryRaw` of + // the round: the latest-reading read). + vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([ + { type: "WEIGHT", value: 81.2, measured_at: twoWeeksAgo }, + ] as never); const res = await callGet(makeReq()); const body = (await res.json()) as { data: { @@ -281,10 +291,9 @@ describe("GET /api/dashboard/summary", () => { expect(weight, "weight card must be present").toBeDefined(); expect(weight?.allTimeCount).toBe(312); expect(weight?.lastSeenAt).toBe(twoWeeksAgo.toISOString()); - // No 7-day reading → latestValue stays null but updatedAt falls - // through to the historical timestamp so the iOS client can - // build the relative-age caption from a single field. - expect(weight?.latestValue).toBeNull(); + // The tile shows the historical reading (REG-11) and dates it, so the + // iOS client builds the relative-age caption from one field. + expect(weight?.latestValue).toBe(81.2); expect(weight?.updatedAt).toBe(twoWeeksAgo.toISOString()); }); @@ -299,16 +308,14 @@ describe("GET /api/dashboard/summary", () => { vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const sixtyDaysAgo = new Date(Date.now() - 60 * 86_400_000); const sixtyOneDaysAgo = new Date(Date.now() - 61 * 86_400_000); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "BLOOD_PRESSURE_SYS", - _count: { _all: 14 }, - _max: { measuredAt: sixtyDaysAgo }, + count: 14, }, { type: "BLOOD_PRESSURE_DIA", - _count: { _all: 14 }, - _max: { measuredAt: sixtyDaysAgo }, + count: 14, }, ] as never); // Promise.all order: [latestEver, sparkline, streakDays]. @@ -406,11 +413,10 @@ describe("GET /api/dashboard/summary", () => { // don't accidentally regress the fresh-data case. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const twoDaysAgo = new Date(Date.now() - 2 * 86_400_000); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "WEIGHT", - _count: { _all: 200 }, - _max: { measuredAt: twoDaysAgo }, + count: 200, }, ] as never); vi.mocked(prisma.$queryRaw) @@ -457,11 +463,10 @@ describe("GET /api/dashboard/summary", () => { // oldest bucket first. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const days = (n: number) => new Date(Date.now() - n * 86_400_000); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "PULSE", - _count: { _all: 42 }, - _max: { measuredAt: days(30) }, + count: 42, }, ] as never); vi.mocked(prisma.$queryRaw) @@ -547,13 +552,15 @@ describe("GET /api/dashboard/summary", () => { // hint). vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const tenDaysAgo = new Date(Date.now() - 10 * 86_400_000); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "BLOOD_GLUCOSE", - _count: { _all: 4 }, - _max: { measuredAt: tenDaysAgo }, + count: 4, }, ] as never); + vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([ + { type: "BLOOD_GLUCOSE", value: 104, measured_at: tenDaysAgo }, + ] as never); const res = await callGet(makeReq()); const body = (await res.json()) as { data: { @@ -579,11 +586,10 @@ describe("GET /api/dashboard/summary", () => { // hours, NOT surface a single stage. Excludes IN_BED + AWAKE. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const wake = new Date("2026-06-04T06:00:00.000Z"); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "SLEEP_DURATION", - _count: { _all: 5 }, - _max: { measuredAt: wake }, + count: 5, }, ] as never); // Raw per-stage rows for the night (the sleep findMany read). @@ -654,11 +660,10 @@ describe("GET /api/dashboard/summary", () => { // total; the additive annotation surfaces the disagreement. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); const wake = new Date("2026-06-04T06:00:00.000Z"); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "SLEEP_DURATION", - _count: { _all: 4 }, - _max: { measuredAt: wake }, + count: 4, }, ] as never); vi.mocked(prisma.measurement.findMany).mockResolvedValue([ @@ -780,11 +785,10 @@ describe("GET /api/dashboard/summary", () => { // returned shape proves the route hands `sum_value` through // instead of reconstructing from `mean * count`. vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "ACTIVITY_STEPS", - _count: { _all: 7 }, - _max: { measuredAt: new Date() }, + count: 7, }, ] as never); // First call: latest7d. Second call: sparkline. Third call: @@ -824,11 +828,10 @@ describe("GET /api/dashboard/summary", () => { it("falls back to mean * count when the legacy sum_value is null (v1.4.39 W-SUM)", async () => { vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); - vi.mocked(prisma.measurement.groupBy).mockResolvedValue([ + vi.mocked(readAllTimeCounts).mockResolvedValue([ { type: "ACTIVITY_STEPS", - _count: { _all: 7 }, - _max: { measuredAt: new Date() }, + count: 7, }, ] as never); vi.mocked(prisma.$queryRaw) diff --git a/src/app/api/dashboard/summary/route.ts b/src/app/api/dashboard/summary/route.ts index 41a60ae4a..4526bf946 100644 --- a/src/app/api/dashboard/summary/route.ts +++ b/src/app/api/dashboard/summary/route.ts @@ -79,6 +79,7 @@ import { buildMedsTodayBlock } from "@/lib/dashboard/meds-today"; import { computeStreak } from "@/lib/dashboard/streak"; import { readActivityDayKeys, + readAllTimeCounts, readLatestEver, readSparkBuckets, } from "@/lib/dashboard/summary-reads"; @@ -468,7 +469,7 @@ async function buildDashboardSummary( // - sparkBuckets: ≤ SPARK_DAYS × N metric types (typically <80, // REG-11: taken via ROW_NUMBER window, no calendar // filter, so a 60-day-old metric still paints a chart) - // - allTimeAggregate: ≤ N metric types (unchanged) + // - allTimeAggregate: ≤ N metric types, summed from the DAY rollups // - medsToday: today-window tally via the shared projector // (≤ daily intake schedule count, unchanged semantics) // - streakActivity: ≤ daily intake count × 365 (unchanged) @@ -492,13 +493,10 @@ async function buildDashboardSummary( time("sparkline", () => readSparkBuckets(prisma, userId, measurementTypes, SPARK_DAYS), ), + // Counted from the DAY rollups where they cover the days, not by one + // index entry per reading (see `readAllTimeCounts`). time("allTime", () => - prisma.measurement.groupBy({ - by: ["type"], - where: { userId, type: { in: measurementTypes }, deletedAt: null }, - _count: { _all: true }, - _max: { measuredAt: true }, - }), + readAllTimeCounts(prisma, userId, measurementTypes, now), ), // v1.4.39 — the dashboard compliance tile pulled // `MedicationIntakeEvent` rows for the today-window only. Daily @@ -583,17 +581,20 @@ async function buildDashboardSummary( // Per-type metadata lookup — typed Map so a metric with no readings // at all falls through `metaForType` to the `{ allTimeCount: 0, - // lastSeenAt: null }` default. The aggregate row's `_count._all` is - // always `number` per Prisma's runtime contract; defending against - // undefined keeps the helper's narrow signature stable. + // lastSeenAt: null }` default. const allTimeByType = new Map< MeasurementType, { allTimeCount: number; lastSeenAt: Date | null } >(); + // The newest reading is the one `latestEver` already read per type; its + // instant is the metric's last-seen time. + const lastSeenByType = new Map( + latestEver.map((row) => [row.type, row.measured_at]), + ); for (const row of allTimeAggregate) { allTimeByType.set(row.type, { - allTimeCount: row._count?._all ?? 0, - lastSeenAt: row._max?.measuredAt ?? null, + allTimeCount: row.count, + lastSeenAt: lastSeenByType.get(row.type) ?? null, }); } function metaForType(type: MeasurementType): { diff --git a/src/lib/dashboard/summary-reads.ts b/src/lib/dashboard/summary-reads.ts index 1a0322d63..8f8550160 100644 --- a/src/lib/dashboard/summary-reads.ts +++ b/src/lib/dashboard/summary-reads.ts @@ -17,6 +17,7 @@ */ import { Prisma, type PrismaClient } from "@/generated/prisma/client"; import type { MeasurementType } from "@/generated/prisma/client"; +import { ROLLUP_FOLD_WINDOW_MS } from "@/lib/rollups/measurement-rollups"; /** v1.4.38 — the single most recent reading per measurement * type, irrespective of age. One row per metric the user has ever @@ -166,6 +167,82 @@ export function readSparkBuckets( `; } +/** One metric's all-time reading count, for a type that has any. */ +export interface AllTimeCountRow { + type: MeasurementType; + count: number; +} + +/** + * The first UTC day the DAY rollups are trusted to hold in full: one day + * inside the fold window. Every fold that ever ran for the account began at + * or before it (its window was anchored on an earlier `now`), and the write + * hooks keep every later day current. + */ +export function allTimeRollupBoundary(now: Date): Date { + const DAY_MS = 24 * 60 * 60 * 1000; + const start = now.getTime() - ROLLUP_FOLD_WINDOW_MS; + return new Date(Math.floor(start / DAY_MS) * DAY_MS + DAY_MS); +} + +/** + * Total live readings per measurement type, all time. + * + * Counting them on `measurements` reads one index entry per reading: on an + * account with a watch that is millions of entries per request, two to four + * seconds whenever they are not in memory, which is the dashboard's slowest + * read by far. The DAY rollups already carry a count per (type, day, source) + * — every live reading of the day, every source — so the sum of a type's DAY + * buckets is its count over the days they cover. Readings before the fold + * window, which the fold never buckets, are counted live; that range is a + * small, bounded index slice. A type with no DAY bucket at all (not folded + * yet) is counted live in full, exactly as before. Types with no readings + * are absent. + */ +export async function readAllTimeCounts( + db: PrismaClient, + userId: string, + measurementTypes: readonly MeasurementType[], + now: Date = new Date(), +): Promise { + const boundary = allTimeRollupBoundary(now); + const rows = await db.$queryRaw< + Array<{ type: MeasurementType; count: number }> + >` + SELECT wanted."type" AS type, + (CASE + WHEN EXISTS ( + SELECT 1 FROM measurement_rollups r + WHERE r."user_id" = ${userId} + AND r."type" = wanted."type" + AND r."granularity" = 'DAY' + ) + THEN COALESCE(( + SELECT SUM(r."count") FROM measurement_rollups r + WHERE r."user_id" = ${userId} + AND r."type" = wanted."type" + AND r."granularity" = 'DAY' + AND r."bucket_start" >= ${boundary} + ), 0) + ( + SELECT COUNT(*) FROM measurements m + WHERE m."user_id" = ${userId} + AND m."type" = wanted."type" + AND m."deleted_at" IS NULL + AND m."measured_at" < ${boundary} + ) + ELSE ( + SELECT COUNT(*) FROM measurements m + WHERE m."user_id" = ${userId} + AND m."type" = wanted."type" + AND m."deleted_at" IS NULL + ) + END)::int AS count + FROM unnest(${[...measurementTypes]}::"measurement_type"[]) AS wanted("type") + ORDER BY wanted."type" + `; + return rows.filter((row) => row.count > 0); +} + /** * The local days in `[since, now]` that hold at least one live reading, as * `YYYY-MM-DD` keys in `timeZone`: the measurement half of the streak. diff --git a/tests/integration/dashboard-summary-reads.test.ts b/tests/integration/dashboard-summary-reads.test.ts index 6dcf1f840..979c6746f 100644 --- a/tests/integration/dashboard-summary-reads.test.ts +++ b/tests/integration/dashboard-summary-reads.test.ts @@ -27,12 +27,15 @@ import { beforeEach, describe, expect, it } from "vitest"; import { getPrismaClient, truncateAllTables } from "./setup"; import { + allTimeRollupBoundary, + readAllTimeCounts, readLatestEver, readSparkBuckets, type LatestEverRow, type SparklineRow, } from "@/lib/dashboard/summary-reads"; import { measurementTypeEnum } from "@/lib/validations/measurement"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; import type { MeasurementType, RollupGranularity, @@ -245,3 +248,92 @@ describe("dashboard summary reads — parity with the pre-v1.38.5 SQL", () => { } }); }); + +describe("dashboard all-time counts — parity with the live count", () => { + /** The pre-rollup read: one index entry per live reading. */ + async function legacyAllTime(userId: string) { + const groups = await getPrismaClient().measurement.groupBy({ + by: ["type"], + where: { userId, type: { in: TYPES }, deletedAt: null }, + _count: { _all: true }, + }); + return new Map(groups.map((g) => [g.type, g._count._all])); + } + + it("counts what the live count counts, from the rollups", async () => { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { username: "all-time-counts", email: "all-time@example.test" }, + }); + const now = new Date(); + const boundary = allTimeRollupBoundary(now); + + // A watch-dense pulse history from two sources: a reading every four + // minutes for two years, written set-wise the way an import lands it. + const twoYears = 2 * 365 * DAY; + for (const [source, offsetMinutes] of [ + ["APPLE_HEALTH", 0], + ["WITHINGS", 2], + ] as const) { + await prisma.$executeRawUnsafe( + `INSERT INTO measurements + (id, user_id, type, value, unit, source, measured_at, created_at, updated_at) + SELECT md5($1 || $2 || g::text), $1, 'PULSE'::"measurement_type", + 60 + (g % 30), 'bpm', $2::"measurement_source", + $3::timestamptz + make_interval(mins => g * 4 + $4), now(), now() + FROM generate_series(0, $5::int) AS g`, + user.id, + source, + new Date(now.getTime() - twoYears), + offsetMinutes, + Math.floor(twoYears / (4 * 60_000)) - 1, + ); + } + // Weigh-ins: some older than the fold window, one tombstoned. + await prisma.measurement.createMany({ + data: [ + ...[1, 2, 3].map((years) => ({ + userId: user.id, + type: "WEIGHT" as MeasurementType, + value: 80 + years, + unit: "kg", + measuredAt: new Date(boundary.getTime() - years * 30 * DAY), + })), + ...[3, 2, 1].map((days) => ({ + userId: user.id, + type: "WEIGHT" as MeasurementType, + value: 79, + unit: "kg", + measuredAt: new Date(now.getTime() - days * DAY), + deletedAt: days === 2 ? now : null, + })), + ], + }); + await recomputeUserRollups(user.id, { granularities: ["DAY"] }); + // A type written after the fold, never bucketed: counted live. + await prisma.measurement.createMany({ + data: [5, 4].map((days) => ({ + userId: user.id, + type: "BLOOD_GLUCOSE" as MeasurementType, + value: 98, + unit: "mg/dL", + measuredAt: new Date(now.getTime() - days * DAY), + })), + }); + + let started = performance.now(); + const legacy = await legacyAllTime(user.id); + const legacyMs = performance.now() - started; + started = performance.now(); + const shipped = await readAllTimeCounts(prisma, user.id, TYPES, now); + const shippedMs = performance.now() - started; + console.log( + `all-time counts over ${legacy.get("PULSE")} pulse rows: live ${legacyMs.toFixed(0)} ms, rollups ${shippedMs.toFixed(0)} ms`, + ); + + expect(new Map(shipped.map((r) => [r.type, r.count]))).toEqual(legacy); + expect(legacy.get("PULSE")).toBeGreaterThan(500_000); + expect(legacy.get("WEIGHT")).toBe(5); + expect(legacy.get("BLOOD_GLUCOSE")).toBe(2); + }); +});