From 01cd755de65788318e2a3b87f6591900e4b79262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 24 Sep 2026 07:22:06 +0200 Subject: [PATCH 1/7] fix(medications): keep recent doses in the trailing history window The Verlauf ledger dropped doses the full history showed. Three windows ended too early or started too late: - The server clamped the window floor to the medication's creation instant and filtered recorded doses by the same floor, so a dose logged for earlier the same day on a medication added afterwards fell out of "Last 90 days". The creation floor now bounds minted slots only; recorded doses use the requested window. - A dose taken inside its window before the slot's own time is stored on the slot anchor, which still lay past the window end. The read horizon now reaches the latest recorded anchor within a day, without minting later upcoming slots. - The client pinned from/to at mount and reused them on every refetch, so a dose added after the tab opened stayed hidden until a reload. The window is now resolved when the request fires and only "from" travels; the query key carries the window length instead of instants. Refs #1028 --- .../[id]/dose-history/__tests__/route.test.ts | 156 ++++++++++++++++++ .../medications/[id]/dose-history/route.ts | 37 ++++- .../dose-history-ledger-window.test.tsx | 105 ++++++++++++ .../medications/dose-history-ledger.tsx | 25 +-- src/lib/query-keys/medications.ts | 4 +- 5 files changed, 301 insertions(+), 26 deletions(-) create mode 100644 src/components/medications/__tests__/dose-history-ledger-window.test.tsx diff --git a/src/app/api/medications/[id]/dose-history/__tests__/route.test.ts b/src/app/api/medications/[id]/dose-history/__tests__/route.test.ts index 4a5c8fed8..23e01838e 100644 --- a/src/app/api/medications/[id]/dose-history/__tests__/route.test.ts +++ b/src/app/api/medications/[id]/dose-history/__tests__/route.test.ts @@ -264,3 +264,159 @@ describe("GET /api/medications/[id]/dose-history", () => { expect(res.status).toBe(422); }); }); + +/** + * Issue #1028 — a dose logged for earlier today on a medication created later + * the same day vanished from the trailing-window ledger while the full + * history (which has no window) showed it. The window floor was clamped to + * the medication's creation instant and the same clamped floor filtered the + * intakes, so any dose recorded with a time before the moment the medication + * was added fell out of "Last 90 days". The creation floor exists to keep + * the ledger from minting phantom slots before the medication existed; it + * says nothing about which recorded doses belong in the window. + * + * Pinned in a positive and a negative UTC offset so a zone-dependent floor + * cannot hide behind one of them. + */ +describe("GET /api/medications/[id]/dose-history — intakes recorded before creation (#1028)", () => { + for (const tz of ["Asia/Kolkata", "America/Los_Angeles"]) { + it(`keeps a same-day dose logged before the medication was added (${tz})`, async () => { + const today = new Date("2026-09-22T12:00:00Z"); + const now = localHmAsUtc(today, tz, 16, 30); + const createdAt = localHmAsUtc(today, tz, 16, 20); + const takenAt = localHmAsUtc(today, tz, 14, 0); + + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(now); + try { + vi.mocked(getSession).mockResolvedValue({ + ...SESSION_OK, + user: { ...SESSION_OK.user, timezone: tz }, + } as never); + vi.mocked(prisma.medication.findUnique).mockResolvedValue({ + id: "med-1", + startsOn: null, + endsOn: null, + oneShot: false, + createdAt, + scheduleRevisions: [], + schedules: [ + { + id: "sched-1", + windowStart: "08:00", + windowEnd: "08:00", + daysOfWeek: null, + timesOfDay: ["08:00"], + reminderGraceMinutes: null, + rrule: null, + rollingIntervalDays: null, + scheduleType: "SCHEDULED", + cyclicOnWeeks: null, + cyclicOffWeeks: null, + }, + ], + } as never); + vi.mocked(prisma.medicationIntakeEvent.findMany).mockResolvedValue([ + { + id: "evt-earlier-today", + scheduledFor: takenAt, + takenAt, + skipped: false, + autoMissed: false, + attributionSource: "AUTO", + doseTaken: null, + source: "WEB", + }, + ] as never); + + // Exactly what the Verlauf tab sends: the trailing 90 days to now. + const from = new Date(now.getTime() - 90 * 86_400_000).toISOString(); + const to = now.toISOString(); + const res = await GET( + getReq( + `?from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}`, + ), + ROUTE_PARAMS, + ); + expect(res.status).toBe(200); + const json = await res.json(); + const ids = ( + json.data.rows as Array<{ intake: { id: string | null } | null }> + ).map((r) => r.intake?.id); + expect(ids).toContain("evt-earlier-today"); + } finally { + vi.useRealTimers(); + } + }); + } +}); + +describe("GET /api/medications/[id]/dose-history — early take (#1028 class)", () => { + it("keeps a dose taken inside its window before the slot's own time", async () => { + const tz = "Asia/Kolkata"; + const today = new Date("2026-09-22T12:00:00Z"); + const now = localHmAsUtc(today, tz, 19, 30); + const slot = localHmAsUtc(today, tz, 20, 0); + const takenAt = localHmAsUtc(today, tz, 19, 15); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(now); + try { + vi.mocked(getSession).mockResolvedValue({ + ...SESSION_OK, + user: { ...SESSION_OK.user, timezone: tz }, + } as never); + vi.mocked(prisma.medication.findUnique).mockResolvedValue({ + id: "med-1", + startsOn: null, + endsOn: null, + oneShot: false, + createdAt: new Date("2026-01-01T00:00:00Z"), + scheduleRevisions: [], + schedules: [ + { + id: "sched-1", + windowStart: "20:00", + windowEnd: "20:00", + daysOfWeek: null, + timesOfDay: ["20:00"], + reminderGraceMinutes: null, + rrule: null, + rollingIntervalDays: null, + scheduleType: "SCHEDULED", + cyclicOnWeeks: null, + cyclicOffWeeks: null, + }, + ], + } as never); + vi.mocked(prisma.medicationIntakeEvent.findMany).mockResolvedValue([ + { + id: "evt-early", + scheduledFor: slot, + takenAt, + skipped: false, + autoMissed: false, + attributionSource: "AUTO", + doseTaken: null, + source: "WEB", + }, + ] as never); + const from = new Date(now.getTime() - 90 * 86_400_000).toISOString(); + const res = await GET( + getReq(`?from=${encodeURIComponent(from)}`), + ROUTE_PARAMS, + ); + const json = await res.json(); + const rows = json.data.rows as Array<{ + status: string; + intake: { id: string | null } | null; + }>; + const early = rows.find((r) => r.intake?.id === "evt-early"); + expect(early?.status).toBe("taken_on_time"); + // The read horizon stretches only to the recorded anchor: no later + // slot is minted as upcoming. + expect(rows.filter((r) => r.status === "upcoming")).toHaveLength(0); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/src/app/api/medications/[id]/dose-history/route.ts b/src/app/api/medications/[id]/dose-history/route.ts index 11c60c9a0..11256b61f 100644 --- a/src/app/api/medications/[id]/dose-history/route.ts +++ b/src/app/api/medications/[id]/dose-history/route.ts @@ -142,12 +142,18 @@ export const GET = apiHandler( const requestedFrom = parsed.data.from ?? new Date(to.getTime() - 90 * DAY_MS); const spanFloor = new Date(to.getTime() - MAX_WINDOW_DAYS * DAY_MS); + // The window the caller asked for, capped only by the span limit. This is + // the floor for recorded doses: a dose logged with a time before the + // medication was added (created at 16:20, "taken at 14:00") is a real + // record and belongs in the window, exactly as the unwindowed full + // history shows it (#1028). + const intakeFrom = new Date( + Math.max(requestedFrom.getTime(), spanFloor.getTime()), + ); + // The floor for minted slots additionally stops at the medication's + // creation, so pre-existence days never mint phantom missed slots. const from = new Date( - Math.max( - requestedFrom.getTime(), - medication.createdAt.getTime(), - spanFloor.getTime(), - ), + Math.max(intakeFrom.getTime(), medication.createdAt.getTime()), ); const events = await prisma.medicationIntakeEvent.findMany({ @@ -182,6 +188,23 @@ export const GET = apiHandler( const lastIntakeAt = lastNonSkippedTakenAt(mapped); + // A dose may be recorded before its slot's own time: the dose window opens + // up to two hours early, and a skip can be logged ahead. Such a row is + // stored on the slot's anchor, which still lies after `to`, so a window + // that ends at `to` dropped a dose the user had just recorded until the + // slot's clock time passed (#1028). Extend the read horizon to the latest + // anchor of a recorded row (taken or skipped) within a day of `to`, and no + // further: pending reminder rows record nothing and never stretch it. + const horizon = new Date( + mapped.reduce((latest, e) => { + const t = e.scheduledFor.getTime(); + const recorded = e.skipped || (e.takenAt !== null && e.takenAt <= to); + return recorded && t > latest && t <= to.getTime() + DAY_MS + ? t + : latest; + }, to.getTime()), + ); + const bandMedication: BandMinterMedication = { id: medication.id, startsOn: medication.startsOn, @@ -223,7 +246,7 @@ export const GET = apiHandler( revisions: medication.scheduleRevisions ?? [], ctx, userTz, - range: { from, to }, + range: { from, to: horizon }, now, intakeInstants, }); @@ -241,7 +264,7 @@ export const GET = apiHandler( // partition, so filter on the stored anchor (a take snapped to an // out-of-window slot is excluded, matching the compliance read). const historyIntakes: HistoryIntake[] = mapped - .filter((e) => e.scheduledFor >= from && e.scheduledFor <= to) + .filter((e) => e.scheduledFor >= intakeFrom && e.scheduledFor <= horizon) .map((e) => ({ id: e.id, scheduledFor: e.scheduledFor, diff --git a/src/components/medications/__tests__/dose-history-ledger-window.test.tsx b/src/components/medications/__tests__/dose-history-ledger-window.test.tsx new file mode 100644 index 000000000..6f0884b52 --- /dev/null +++ b/src/components/medications/__tests__/dose-history-ledger-window.test.tsx @@ -0,0 +1,105 @@ +/** + * Issue #1028 — the Verlauf ledger's trailing window must be resolved when + * the request fires, not when the tab mounts. + * + * The ledger used to pin `from` and `to` in state at mount and reuse them for + * every fetch. A refetch after adding a dose therefore asked for rows up to + * the moment the tab opened, and a dose recorded afterwards stayed out of + * "Last 90 days" until a reload, while the unwindowed full history showed it. + * + * The test captures the query function the ledger hands to TanStack Query and + * runs it twice at two different clock readings: both requests must reach the + * present, which means no pinned `to` and a `from` that moves with the clock. + */ +import { describe, expect, it, vi, afterEach } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; + +const captured: { queryFn?: () => Promise; queryKey?: unknown } = {}; + +vi.mock("@tanstack/react-query", async (orig) => { + const actual = await orig(); + return { + ...actual, + useQuery: (opts: { + queryFn: () => Promise; + queryKey: unknown; + }) => { + captured.queryFn = opts.queryFn; + captured.queryKey = opts.queryKey; + return { data: undefined, isLoading: true, isError: false }; + }, + useQueryClient: () => ({}), + }; +}); + +vi.mock("@/lib/api/api-fetch", () => ({ + apiGet: vi.fn().mockResolvedValue({ rows: [] }), + apiPost: vi.fn(), + apiPut: vi.fn(), + apiDelete: vi.fn(), +})); + +vi.mock("@/hooks/use-auth", () => ({ + useAuth: () => ({ + user: { id: "u1", username: "t", role: "USER", timezone: "Asia/Kolkata" }, + isAuthenticated: true, + isLoading: false, + }), +})); + +vi.mock("@/hooks/use-record-capabilities", () => ({ + useRecordCapabilities: () => ({ canWriteDomain: () => true }), + useActiveRecordName: () => null, +})); + +import { I18nProvider } from "@/lib/i18n/context"; +import { DoseHistoryLedger } from "@/components/medications/dose-history-ledger"; +import { apiGet } from "@/lib/api/api-fetch"; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("DoseHistoryLedger request window (#1028)", () => { + it("resolves the window at fetch time so a later refetch reaches the present", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + const mountedAt = new Date("2026-09-22T10:00:00Z"); + vi.setSystemTime(mountedAt); + + renderToStaticMarkup( + + + , + ); + expect(captured.queryFn).toBeTypeOf("function"); + + await captured.queryFn!(); + // An hour later the user adds a dose; the invalidation refetches. + const later = new Date(mountedAt.getTime() + 3_600_000); + vi.setSystemTime(later); + await captured.queryFn!(); + + const urls = vi.mocked(apiGet).mock.calls.map((c) => String(c[0])); + expect(urls).toHaveLength(2); + for (const url of urls) { + const params = new URL(url, "http://x").searchParams; + // No pinned upper bound: the server's `to` is its own now. + expect(params.get("to")).toBeNull(); + } + const secondFrom = new URL(urls[1], "http://x").searchParams.get("from"); + expect(secondFrom).toBe( + new Date(later.getTime() - 90 * 86_400_000).toISOString(), + ); + // The cache key carries no instant, so it stays stable across renders. + expect(captured.queryKey).toEqual([ + "medications", + "med-1", + "dose-history", + 90, + ]); + }); +}); diff --git a/src/components/medications/dose-history-ledger.tsx b/src/components/medications/dose-history-ledger.tsx index d5ecabcb9..e80592a7f 100644 --- a/src/components/medications/dose-history-ledger.tsx +++ b/src/components/medications/dose-history-ledger.tsx @@ -200,27 +200,18 @@ export function DoseHistoryLedger({ slotAt: string; } | null>(null); - // Stable window captured once at mount (a lazy state initializer keeps the - // impure `Date.now()` out of render) so the query key + the `from`/`to` - // request stay byte-equal across renders rather than re-keying every paint. - const [{ fromIso, toIso }] = useState(() => { - const now = Date.now(); - return { - toIso: new Date(now).toISOString(), - fromIso: new Date(now - windowDays * DAY_MS).toISOString(), - }; - }); - - const queryKey = queryKeys.medicationDoseHistory( - medicationId, - fromIso, - toIso, - ); + // The window is resolved when the request fires, not when the ledger + // mounts. A window pinned at mount made every refetch (including the one + // after adding an entry) ask for rows up to the moment the tab opened, so a + // dose recorded afterwards stayed invisible until a reload (#1028). Only + // `from` travels: the server's `to` defaults to its own now. + const queryKey = queryKeys.medicationDoseHistory(medicationId, windowDays); const { data, isLoading, isError } = useQuery({ queryKey, queryFn: async () => { - const search = new URLSearchParams({ from: fromIso, to: toIso }); + const from = new Date(Date.now() - windowDays * DAY_MS).toISOString(); + const search = new URLSearchParams({ from }); return apiGet( `/api/medications/${medicationId}/dose-history?${search.toString()}`, ); diff --git a/src/lib/query-keys/medications.ts b/src/lib/query-keys/medications.ts index 979aefde5..b7f5b2f1a 100644 --- a/src/lib/query-keys/medications.ts +++ b/src/lib/query-keys/medications.ts @@ -83,8 +83,8 @@ export const medicationKeys = { * Übersicht headline % flip in the same paint, then reconciles on the * authoritative refetch. */ - medicationDoseHistory: (medicationId: string, from: string, to: string) => - ["medications", medicationId, "dose-history", from, to] as const, + medicationDoseHistory: (medicationId: string, windowDays: number) => + ["medications", medicationId, "dose-history", windowDays] as const, medicationIntakeDrugLevelChart: (medicationId: string) => ["medications", medicationId, "intake", "drug-level-chart"] as const, /** From 88cea09c7bd17fe17d8d73bceb8f09d5de691d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 24 Sep 2026 07:56:33 +0200 Subject: [PATCH 2/7] fix(measurements): cut chart day buckets at the user's midnight Switching a chart from 7 to 30 days moved steps, active energy, flights and distance totals onto neighbouring days. The 7-day range buckets raw rows by the user's calendar day; the 30-day range read the DAY rollup tier, whose buckets are UTC days, and the chart labelled each bucket's UTC-midnight start in the user's zone. East of UTC a sample shortly after local midnight counted on the previous day; west of UTC every bucket read one day early. The daily series (the chart read and the batched dashboard read) and the aggregate=daily|weekly|monthly list now fold live in the user's zone and return the instant of local midnight, so every range and every summed metric shares one day boundary. The fold runs in Postgres and returns one row per day; about 0.1 s for 90 days of per-minute samples. Windows longer than the DAY cap still step up to the coarser rollup tiers. Refs #1026 --- .../__tests__/range-aggregation-route.test.ts | 341 ++---------------- src/app/api/measurements/route.ts | 142 ++------ .../daily-series-read-tiered.test.ts | 117 ++---- src/lib/measurements/daily-series-read.ts | 327 +++++++---------- .../integration/daily-series-user-tz.test.ts | 127 +++++++ .../measurements-aggregate-daily.test.ts | 13 +- 6 files changed, 365 insertions(+), 702 deletions(-) create mode 100644 tests/integration/daily-series-user-tz.test.ts diff --git a/src/app/api/measurements/__tests__/range-aggregation-route.test.ts b/src/app/api/measurements/__tests__/range-aggregation-route.test.ts index 8f0245114..968963986 100644 --- a/src/app/api/measurements/__tests__/range-aggregation-route.test.ts +++ b/src/app/api/measurements/__tests__/range-aggregation-route.test.ts @@ -32,9 +32,6 @@ vi.mock("@/lib/db", () => ({ }, })); -// v1.4.39.2 — the rollup branch's coverage-mismatch fallback may call -// `recomputeUserRollups` inline. Mock it so tests can assert whether -// the inline fold fired without exercising the real populator. vi.mock("@/lib/rollups/measurement-rollups", async () => { const actual = await vi.importActual< typeof import("@/lib/rollups/measurement-rollups") @@ -66,7 +63,7 @@ vi.mock("next/headers", () => ({ import { GET } from "../route"; import { prisma } from "@/lib/db"; import { getSession } from "@/lib/auth/session"; -import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; const SESSION_OK = { session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, @@ -221,318 +218,50 @@ describe("GET /api/measurements — all-time semantics (SD-H1)", () => { expect(months.size).toBe(24); }); - it("source=rollup + aggregate=daily reads measurement_rollups without firing the heavy date_trunc query", async () => { - // v1.4.36 W1 — Insights trends row routes the three daily chart - // fetches through the persistent DAY buckets via `source=rollup`. - // The heavy `date_trunc` $queryRaw must NOT fire on the happy - // path where the rollup has rows for the requested window. - const buckets = Array.from({ length: 30 }, (_, i) => ({ - type: "WEIGHT", - bucketStart: new Date(Date.UTC(2026, 3, 16 + i, 0, 0, 0)), - mean: 81 + i * 0.05, - count: 5, - // v1.11.1 — single source per day → collapse is identity. - source: "APPLE_HEALTH", - })); - vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue( - buckets as never, - ); + it("source=rollup + aggregate=daily folds live in the user's zone, never the UTC-day rollup (#1026)", async () => { + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + timezone: "Asia/Kolkata", + sourcePriorityJson: null, + } as never); + // The resolver caches per user; drop what earlier cases left behind. + invalidateUserTimezone("user-1"); + vi.mocked(prisma.$queryRaw).mockResolvedValue([ + { + type: "ACTIVITY_STEPS", + bucket_start: new Date("2026-05-09T18:30:00.000Z"), + avg: 8123, + cnt: 96, + min_value: 0, + max_value: 400, + }, + ] as never); const res = await GET( getRequest( - "type=WEIGHT&from=2026-04-15T00:00:00Z&to=2026-05-15T00:00:00Z&aggregate=daily&source=rollup&limit=5000", + "type=ACTIVITY_STEPS&from=2026-04-15T00:00:00Z&to=2026-05-15T00:00:00Z&aggregate=daily&source=rollup&limit=5000", ), ); + expect(res.status).toBe(200); const json = (await res.json()) as { data: { - measurements: Array<{ type: string; value: number; count: number }>; - meta: { aggregate?: string }; + measurements: Array<{ + measuredAt: string; + value: number; + minValue?: number; + }>; }; }; - expect(prisma.measurementRollup.findMany).toHaveBeenCalledTimes(1); - expect(prisma.$queryRaw).not.toHaveBeenCalled(); - expect(json.data.measurements.length).toBe(30); - // First bucket reflects the rollup mean — round-trip-stable. - expect(json.data.measurements[0].type).toBe("WEIGHT"); - expect(json.data.measurements[0].value).toBeCloseTo(81, 5); - expect(json.data.measurements[0].count).toBe(5); - expect(json.data.meta.aggregate).toBe("daily"); - }); - - it("source=rollup cumulative path reads sum_value directly (v1.4.39 W-SUM)", async () => { - // ACTIVITY_STEPS rollup row with sum_value populated. The route - // must consume sumValue directly — NOT reconstruct from mean * - // count. Use a sum that is NOT mean × count so the test - // distinguishes the two paths. - const buckets = Array.from({ length: 3 }, (_, i) => ({ - type: "ACTIVITY_STEPS", - bucketStart: new Date(Date.UTC(2026, 4, 1 + i, 0, 0, 0)), - mean: 2000, // would yield 10000 if multiplied by count=5 - count: 5, - sumValue: 11000 + i * 250, // distinct from mean × count - source: "APPLE_HEALTH", - })); - vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue( - buckets as never, - ); - - const res = await GET( - getRequest( - "type=ACTIVITY_STEPS&from=2026-05-01T00:00:00Z&to=2026-05-05T00:00:00Z&aggregate=daily&source=rollup&limit=5000", - ), - ); - expect(res.status).toBe(200); - const json = (await res.json()) as { - data: { measurements: Array<{ value: number }> }; - }; - expect(prisma.$queryRaw).not.toHaveBeenCalled(); - expect(json.data.measurements.map((m) => m.value)).toEqual([ - 11000, 11250, 11500, - ]); - }); - - it("source=rollup cumulative path falls back to mean*count when sum_value is NULL (v1.4.39 W-SUM)", async () => { - // Pre-v1.4.39 row — boot-backfill hasn't converged yet. The - // route falls back to mean × count so the chart never paints - // a hole during the convergence window. - const buckets = [ - { - type: "ACTIVITY_STEPS", - bucketStart: new Date(Date.UTC(2026, 4, 1)), - mean: 2000, - count: 5, - sumValue: null, - source: "APPLE_HEALTH", - }, - ]; - vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue( - buckets as never, - ); - - const res = await GET( - getRequest( - "type=ACTIVITY_STEPS&from=2026-05-01T00:00:00Z&to=2026-05-05T00:00:00Z&aggregate=daily&source=rollup&limit=5000", - ), - ); - const json = (await res.json()) as { - data: { measurements: Array<{ value: number }> }; - }; - expect(json.data.measurements[0].value).toBe(10000); - }); - - it("falls back to live date_trunc when source=rollup returns zero buckets", async () => { - // Empty rollup ⇒ heavy aggregate runs so brand-new accounts still - // see a correct chart on their first render. - vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue([] as never); - const liveBuckets = Array.from({ length: 7 }, (_, i) => ({ - type: "WEIGHT", - bucket_start: new Date(Date.UTC(2026, 4, 8 + i, 0, 0, 0)), - avg: 81 + i * 0.05, - cnt: 2, - })); - vi.mocked(prisma.$queryRaw).mockResolvedValue(liveBuckets as never); - - const res = await GET( - getRequest( - "type=WEIGHT&from=2026-05-08T00:00:00Z&to=2026-05-15T00:00:00Z&aggregate=daily&source=rollup&limit=5000", - ), - ); - expect(res.status).toBe(200); - expect(prisma.measurementRollup.findMany).toHaveBeenCalledTimes(1); - expect(prisma.$queryRaw).toHaveBeenCalledTimes(1); - const json = (await res.json()) as { - data: { measurements: Array }; - }; - expect(json.data.measurements.length).toBe(7); - }); - - it("source=rollup folds the partition inline and re-reads when the rollup is under-converged (v1.4.39.2)", async () => { - // The maintainer's pre-fix production trace: 30-day BP_SYS window returned - // `total: 2` from the rollup branch even though `measurements` - // carried ~30 distinct days. The chart tripped its `< 3 daily - // points` empty-state because the route short-circuited on - // `rollupRows.length > 0` and never reconciled with the live - // table. The coverage-mismatch fallback runs an inline - // `recomputeUserRollups` for the (user, type, DAY, window) - // partition and re-reads so the request returns the converged - // rows. - const sparseRollup = [ - { - type: "BLOOD_PRESSURE_SYS", - bucketStart: new Date(Date.UTC(2026, 4, 1, 0, 0, 0)), - mean: 128, - count: 1, - sumValue: null, - source: "APPLE_HEALTH", - }, - { - type: "BLOOD_PRESSURE_SYS", - bucketStart: new Date(Date.UTC(2026, 4, 10, 0, 0, 0)), - mean: 132, - count: 1, - sumValue: null, - source: "APPLE_HEALTH", - }, - ]; - const convergedRollup = Array.from({ length: 28 }, (_, i) => ({ - type: "BLOOD_PRESSURE_SYS", - bucketStart: new Date(Date.UTC(2026, 4, 1 + i, 0, 0, 0)), - mean: 128 + (i % 5), - count: 2, - sumValue: null, - source: "APPLE_HEALTH", - })); - // First read returns the sparse rows; second (post-fold) read - // returns the converged rows. - vi.mocked(prisma.measurementRollup.findMany) - .mockResolvedValueOnce(sparseRollup as never) - .mockResolvedValueOnce(convergedRollup as never); - // Coverage probe — live `measurements` has 28 distinct days. - vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([ - { days: BigInt(28) }, - ] as never); - - const res = await GET( - getRequest( - "type=BLOOD_PRESSURE_SYS&from=2026-04-30T00:00:00Z&to=2026-05-30T00:00:00Z&aggregate=daily&source=rollup&limit=5000", - ), - ); - expect(res.status).toBe(200); - const json = (await res.json()) as { - data: { measurements: Array }; - }; - // The inline fold fired exactly once with the (type, DAY, window) - // bound so the partition write stays small. - expect(recomputeUserRollups).toHaveBeenCalledTimes(1); - expect(recomputeUserRollups).toHaveBeenCalledWith( - "user-1", - expect.objectContaining({ - types: ["BLOOD_PRESSURE_SYS"], - granularities: ["DAY"], - }), - ); - // The route re-read the rollup after the fold so the response - // carries the converged 28 buckets — well above the chart's - // `< 3 daily points` empty-state threshold. - expect(prisma.measurementRollup.findMany).toHaveBeenCalledTimes(2); - expect(json.data.measurements.length).toBe(28); - }); - - it("source=rollup skips the coverage probe on a covered window (v1.4.39.2 hot-path)", async () => { - // Happy case: 30-day window, 28 rollup rows present. The route - // must short-circuit on the first read and skip the probe so the - // covered-tenant hot path stays a single indexed read. - const buckets = Array.from({ length: 28 }, (_, i) => ({ - type: "WEIGHT", - bucketStart: new Date(Date.UTC(2026, 3, 16 + i, 0, 0, 0)), - mean: 81 + i * 0.05, - count: 2, - sumValue: null, - source: "APPLE_HEALTH", - })); - vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue( - buckets as never, - ); - - const res = await GET( - getRequest( - "type=WEIGHT&from=2026-04-15T00:00:00Z&to=2026-05-15T00:00:00Z&aggregate=daily&source=rollup&limit=5000", - ), - ); - expect(res.status).toBe(200); - expect(prisma.measurementRollup.findMany).toHaveBeenCalledTimes(1); - expect(recomputeUserRollups).not.toHaveBeenCalled(); - expect(prisma.$queryRaw).not.toHaveBeenCalled(); - }); - - it("source=rollup skips the coverage fallback on short windows (v1.4.39.2)", async () => { - // A 5-day window with 2 rollup rows is plausibly genuine sparsity - // for a user who measures BP every other day. The coverage - // fallback gates on `windowDays >= 7` so the probe + inline fold - // never fire on these short windows. - const sparseRollup = [ - { - type: "BLOOD_PRESSURE_SYS", - bucketStart: new Date(Date.UTC(2026, 4, 1, 0, 0, 0)), - mean: 128, - count: 1, - sumValue: null, - source: "APPLE_HEALTH", - }, - { - type: "BLOOD_PRESSURE_SYS", - bucketStart: new Date(Date.UTC(2026, 4, 3, 0, 0, 0)), - mean: 132, - count: 1, - sumValue: null, - source: "APPLE_HEALTH", - }, - ]; - vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue( - sparseRollup as never, - ); - - const res = await GET( - getRequest( - "type=BLOOD_PRESSURE_SYS&from=2026-05-01T00:00:00Z&to=2026-05-06T00:00:00Z&aggregate=daily&source=rollup&limit=5000", - ), - ); - expect(res.status).toBe(200); - expect(recomputeUserRollups).not.toHaveBeenCalled(); - expect(prisma.$queryRaw).not.toHaveBeenCalled(); - const json = (await res.json()) as { - data: { measurements: Array }; - }; - expect(json.data.measurements.length).toBe(2); - }); - - it("source=rollup keeps the sparse rollup when live `measurements` agrees (v1.4.39.2)", async () => { - // Genuine sparsity: the user measured BP twice in 30 days. The - // probe finds the same 2 distinct days in `measurements` so the - // route returns the 2 rollup rows without firing the inline - // fold. The chart's `< 3 daily points` empty-state is the right - // surface for this case — the data really is sparse. - const sparseRollup = [ - { - type: "BLOOD_PRESSURE_SYS", - bucketStart: new Date(Date.UTC(2026, 4, 1, 0, 0, 0)), - mean: 128, - count: 1, - sumValue: null, - source: "APPLE_HEALTH", - }, - { - type: "BLOOD_PRESSURE_SYS", - bucketStart: new Date(Date.UTC(2026, 4, 10, 0, 0, 0)), - mean: 132, - count: 1, - sumValue: null, - source: "APPLE_HEALTH", - }, - ]; - vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue( - sparseRollup as never, - ); - // Probe agrees — 2 distinct days in `measurements` too. - vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([ - { days: BigInt(2) }, - ] as never); - - const res = await GET( - getRequest( - "type=BLOOD_PRESSURE_SYS&from=2026-04-30T00:00:00Z&to=2026-05-30T00:00:00Z&aggregate=daily&source=rollup&limit=5000", - ), - ); - expect(res.status).toBe(200); - // Probe fired (suspicious-low rollup) but the inline fold did - // not (live and rollup agree). - expect(prisma.$queryRaw).toHaveBeenCalledTimes(1); - expect(recomputeUserRollups).not.toHaveBeenCalled(); - const json = (await res.json()) as { - data: { measurements: Array }; - }; - expect(json.data.measurements.length).toBe(2); + expect(prisma.measurementRollup.findMany).not.toHaveBeenCalled(); + // The zone travels as a bound parameter of the fold. + const call = vi.mocked(prisma.$queryRaw).mock.calls[0] as unknown[]; + expect(call.slice(1)).toContain("Asia/Kolkata"); + // Local midnight of 10 May in Kolkata; a SUM carries no spread band. + expect(json.data.measurements[0]).toMatchObject({ + measuredAt: "2026-05-09T18:30:00.000Z", + value: 8123, + }); + expect(json.data.measurements[0].minValue).toBeUndefined(); }); it("returns a weekly series when the all-time window is under two years", async () => { diff --git a/src/app/api/measurements/route.ts b/src/app/api/measurements/route.ts index ca5e59685..a1eb337a1 100644 --- a/src/app/api/measurements/route.ts +++ b/src/app/api/measurements/route.ts @@ -45,20 +45,21 @@ import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; import { runSafetyFloorCheck } from "@/lib/illness/safety-floor-check"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; -import { readDailySeries } from "@/lib/measurements/daily-series-read"; +import { + readDailySeries, + readLiveBuckets, +} from "@/lib/measurements/daily-series-read"; import { reconstructSleepSessions, pickMainNightAndNaps, } from "@/lib/analytics/sleep-night"; import { resolveUserTimezone } from "@/lib/tz/resolver"; -import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; import { NextRequest } from "next/server"; import type { MeasurementType, MeasurementSource, GlucoseContext, } from "@/generated/prisma/client"; -import { Prisma } from "@/generated/prisma/client"; export const GET = apiHandler(async (request: NextRequest) => { const { user } = await requireRecordAuth("read", "measurements"); @@ -393,36 +394,12 @@ export const GET = apiHandler(async (request: NextRequest) => { // The route's response is bounded by the `BUCKET_CAP` ceiling per // grain (monthly: 24, weekly: 105, daily: 365) so a multi-decade // account never paints an unbounded series. - // v1.4.36 W1 — rollup-table read for daily aggregates. - // - // The Insights trends row fires three parallel - // `GET /api/measurements?type=…&aggregate=daily&limit=5000` requests - // (BP_SYS / BP_DIA / WEIGHT). The legacy `date_trunc('day', …)` - // path scanned the measurements table each time — a full year window - // on the maintainer's account hit 3 × ~3 s on the v1.4.35 HAR. Reading from - // the persistent `measurement_rollups` DAY buckets drops each call - // to a small indexed read against the ~5 k-row rollup table. - // - // Gating: - // - `source=rollup` is an explicit client opt-in (`@/lib/validations/measurement` - // constrains the enum to `["rollup"]`). - // - We require `aggregate=daily` because the rollup populator only - // keeps DAY buckets on the synchronous write hook; the WEEK / - // MONTH paths still depend on the async pg-boss recomputes which - // may lag behind a recent write. - // - `type` is mandatory so the response shape mirrors the per-type - // ask the chart code makes. - // - When the rollup window returns zero rows we fall through to - // the live `date_trunc` path so a brand-new account whose - // populator hasn't caught up still sees a correct chart on its - // first render. + // `source=rollup` + `aggregate=daily` — the chart's series read. Windows up + // to the DAY cap fold live into the user's local days (#1026); wider + // windows step up to the WEEK / MONTH / YEAR rollup tier. `type` is + // mandatory so the response mirrors the per-type ask the chart makes. if (source === "rollup" && aggregate === "daily" && type && from && to) { - // v1.18.6 — the rollup-daily read (per-source collapse + coverage - // probe + inline fold + live-SQL fallback) moved to the shared - // `readDailySeries` reader so the batched series endpoint - // (`/api/measurements/series-batch`) reads through the SAME path - // instead of duplicating this SQL. Behaviour is unchanged: the rows - // returned here are byte-identical with the pre-extraction branch. + // Shared with the batched series endpoint, so both read the same rows. const cap = Math.min(limit, BUCKET_CAP.daily); const priorityJson = await loadUserSourcePriority(user.id); const measurements = await readDailySeries({ @@ -444,94 +421,29 @@ export const GET = apiHandler(async (request: NextRequest) => { }, }); } - // `readDailySeries` already falls back to the live `date_trunc` - // aggregate on a rollup-coverage miss, so an empty result here means - // the window genuinely holds no rows. Fall through to the shared - // live aggregate branch below to keep the response shape identical - // for the truly-empty case. + // An empty result means the window genuinely holds no rows. Fall + // through to the shared live aggregate branch below to keep the + // response shape identical for the truly-empty case. } if (aggregate && aggregate !== "raw" && from && to) { - const grain: AggregateGrain = aggregate; + const grain: Exclude = aggregate; const cap = Math.min(limit, BUCKET_CAP[grain]); - // Postgres `date_trunc` requires the unit argument to be a SQL - // literal — it cannot be supplied as a prepared-statement parameter. - // The previous `${truncUnit}` (bound) interpolation 500'd in - // production for every grain; the grain string is restricted to - // the `AggregateGrain` enum upstream via the Zod schema, so - // injecting the mapped unit via `Prisma.raw` is safe. The mapping - // also fixes the "weekly"/"monthly" passthrough — Postgres expects - // the singular forms `week`/`month`. - const TRUNC_UNIT: Record, string> = { - daily: "day", - weekly: "week", - monthly: "month", - }; - const truncUnit = TRUNC_UNIT[grain]; - const truncUnitLiteral = Prisma.raw(`'${truncUnit}'`); - // Cumulative HK types (steps, active energy, flights, distance, - // daylight) are partial-day increments — averaging across the day - // understates the daily total by the per-bucket sample count. - // SUM for those; AVG for spot metrics (BP, weight, pulse, BG, - // body fat, mood, sleep). See R-A finding 2. - const useSum = - type != null && CUMULATIVE_HK_TYPES.has(type as MeasurementType); - const aggregator = useSum - ? Prisma.raw(`SUM(m."value")::double precision`) - : Prisma.raw(`AVG(m."value")::double precision`); - // v1.11.1 — collapse overlapping sources to the ladder-canonical reading - // per (type, bucket) before aggregating, so this cold/uncovered fallback - // agrees with the warm rollup path. `canon` picks the winning source for - // each (type, bucket) via the per-user rank; the join keeps only its rows. + // Buckets are cut at the user's own calendar boundaries (#1026), the + // same day the raw list and the chart's short range use. Cumulative + // types SUM per bucket, every other type averages; overlapping sources + // collapse to the ladder-canonical one first. const priorityJson = await loadUserSourcePriority(user.id); - const rankRaw = Prisma.raw( - buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'), - ); - const typeFilter = type - ? Prisma.sql`AND m."type" = ${type}::measurement_type` - : Prisma.empty; - const buckets = await prisma.$queryRaw< - Array<{ type: string; bucket_start: Date; avg: number; cnt: number }> - >` - WITH canon AS ( - SELECT DISTINCT ON (m."type", date_trunc(${truncUnitLiteral}, m."measured_at")) - m."type" AS t, - date_trunc(${truncUnitLiteral}, m."measured_at") AS d, - m."source" AS canon - FROM measurements m - WHERE m."user_id" = ${user.id} - AND m."measured_at" >= ${from} - AND m."measured_at" <= ${to} - AND m."deleted_at" IS NULL - ${typeFilter} - ORDER BY m."type", date_trunc(${truncUnitLiteral}, m."measured_at"), (${rankRaw}), m."source" - ) - SELECT - m."type"::text AS type, - date_trunc(${truncUnitLiteral}, m."measured_at") AS bucket_start, - ${aggregator} AS avg, - COUNT(*)::int AS cnt - FROM measurements m - JOIN canon c - ON c.t = m."type" - AND c.d = date_trunc(${truncUnitLiteral}, m."measured_at") - AND c.canon = m."source" - WHERE m."user_id" = ${user.id} - AND m."measured_at" >= ${from} - AND m."measured_at" <= ${to} - AND m."deleted_at" IS NULL - ${typeFilter} - GROUP BY m."type", bucket_start - ORDER BY bucket_start ASC - LIMIT ${cap} - `; - - const measurements = buckets.map((b) => ({ - type: b.type, - value: Number(b.avg), - measuredAt: b.bucket_start.toISOString(), - count: Number(b.cnt), - })); + const measurements = await readLiveBuckets({ + userId: user.id, + type: (type as MeasurementType | undefined) ?? null, + from, + to, + cap, + priorityJson, + grain, + timeZone: await resolveUserTimezone(user.id), + }); annotate({ action: { name: "measurement.list" }, meta: { total: measurements.length, type, aggregate: grain }, diff --git a/src/lib/measurements/__tests__/daily-series-read-tiered.test.ts b/src/lib/measurements/__tests__/daily-series-read-tiered.test.ts index 4fd85bec3..9f646f59b 100644 --- a/src/lib/measurements/__tests__/daily-series-read-tiered.test.ts +++ b/src/lib/measurements/__tests__/daily-series-read-tiered.test.ts @@ -27,10 +27,6 @@ vi.mock("@/lib/rollups/measurement-read-wmy", () => ({ readTieredRollupSeries: mocks.readTieredRollupSeries, })); -vi.mock("@/lib/rollups/measurement-rollups", () => ({ - recomputeUserRollups: vi.fn(), -})); - import { readDailySeries } from "../daily-series-read"; const DAY_MS = 86_400_000; @@ -39,47 +35,25 @@ beforeEach(() => { mocks.findMany.mockReset(); mocks.queryRaw.mockReset(); mocks.readTieredRollupSeries.mockReset(); - // Default DAY-rollup read returns enough rows (≥ SUSPICIOUS_ROW_FLOOR) - // so the short-window path resolves without the coverage probe / live - // query firing. - mocks.findMany.mockResolvedValue([ - { + // The daily path folds live, one row per local day. + mocks.queryRaw.mockResolvedValue( + ["2026-06-01", "2026-06-02", "2026-06-03"].map((d, i) => ({ type: "WEIGHT", - source: "MANUAL", - bucketStart: new Date("2026-06-01T00:00:00.000Z"), - mean: 81, - count: 1, - sumValue: null, - minValue: 81, - maxValue: 81, - }, - { - type: "WEIGHT", - source: "MANUAL", - bucketStart: new Date("2026-06-02T00:00:00.000Z"), - mean: 82, - count: 1, - sumValue: null, - minValue: 82, - maxValue: 82, - }, - { - type: "WEIGHT", - source: "MANUAL", - bucketStart: new Date("2026-06-03T00:00:00.000Z"), - mean: 80, - count: 1, - sumValue: null, - minValue: 80, - maxValue: 80, - }, - ]); + bucket_start: new Date(`${d}T00:00:00.000Z`), + avg: 80 + i, + cnt: 1, + min_value: 80 + i, + max_value: 80 + i, + })), + ); }); afterEach(() => { vi.restoreAllMocks(); }); +const TZ = "Europe/Berlin"; + describe("readDailySeries — long-range tier step-up", () => { it("routes a multi-year window through the tiered reader and returns its whole-history rows", async () => { const to = new Date("2026-06-21T00:00:00.000Z"); @@ -109,6 +83,7 @@ describe("readDailySeries — long-range tier step-up", () => { from, to, priorityJson: null, + timeZone: TZ, }); expect(mocks.readTieredRollupSeries).toHaveBeenCalledTimes(1); @@ -119,8 +94,8 @@ describe("readDailySeries — long-range tier step-up", () => { // Whole-history coverage: the earliest 2017 bucket survives. expect(result[0].measuredAt).toBe("2017-01-01T00:00:00.000Z"); expect(result).toHaveLength(2); - // The DAY-rollup path was NOT consulted for the long window. - expect(mocks.findMany).not.toHaveBeenCalled(); + // The daily path was NOT consulted for the long window. + expect(mocks.queryRaw).not.toHaveBeenCalled(); }); it("falls through to the daily path on a tiered coverage miss (no silent empty)", async () => { @@ -134,11 +109,11 @@ describe("readDailySeries — long-range tier step-up", () => { from, to, priorityJson: null, + timeZone: TZ, }); expect(mocks.readTieredRollupSeries).toHaveBeenCalledTimes(1); - // Daily DAY-rollup read still ran as the fallback. - expect(mocks.findMany).toHaveBeenCalled(); + expect(mocks.queryRaw).toHaveBeenCalled(); expect(result).toHaveLength(3); }); @@ -146,56 +121,25 @@ describe("readDailySeries — long-range tier step-up", () => { const to = new Date("2026-06-21T00:00:00.000Z"); const from = new Date(to.getTime() - 90 * DAY_MS); - await readDailySeries({ - userId: "u", - type: "WEIGHT", - from, - to, - priorityJson: null, - }); - - expect(mocks.readTieredRollupSeries).not.toHaveBeenCalled(); - expect(mocks.findMany).toHaveBeenCalled(); - }); -}); - -describe("readDailySeries — rollup read throw falls back to live SQL (F-DB-2)", () => { - it("a rollup-table throw on a normal window degrades to the live date_trunc aggregate, not a 500", async () => { - const to = new Date("2026-06-21T00:00:00.000Z"); - const from = new Date(to.getTime() - 90 * DAY_MS); - - // The primary rollup read throws (statement_timeout / deadlock / reset). - mocks.findMany - .mockReset() - .mockRejectedValue(new Error("statement timeout")); - // Live fallback serves the tile. - mocks.queryRaw.mockResolvedValueOnce([ - { - type: "WEIGHT", - bucket_start: new Date("2026-06-10T00:00:00.000Z"), - avg: 80.5, - cnt: 2, - min_value: 79.8, - max_value: 81.2, - }, - ]); - const result = await readDailySeries({ userId: "u", type: "WEIGHT", from, to, priorityJson: null, + timeZone: TZ, }); - // The read did NOT throw — it fell through to live SQL and served a row. - expect(result).toHaveLength(1); - expect(result[0]).toMatchObject({ type: "WEIGHT", value: 80.5, count: 2 }); - // The live fallback must carry the min/max spread the rollup path emits — - // otherwise a rollup-cold account silently loses the chart's spread band - // for the same underlying data. - expect(result[0]).toMatchObject({ minValue: 79.8, maxValue: 81.2 }); - expect(mocks.queryRaw).toHaveBeenCalled(); + expect(mocks.readTieredRollupSeries).not.toHaveBeenCalled(); + // The daily series never reads the UTC-day DAY rollup (#1026). + expect(mocks.findMany).not.toHaveBeenCalled(); + expect(result[0]).toMatchObject({ + type: "WEIGHT", + value: 80, + count: 1, + minValue: 80, + maxValue: 80, + }); }); it("a tiered-reader throw on a long window falls through to the daily path", async () => { @@ -209,11 +153,10 @@ describe("readDailySeries — rollup read throw falls back to live SQL (F-DB-2)" from, to, priorityJson: null, + timeZone: TZ, }); - // The daily DAY-rollup read still ran as the fallback (the default findMany - // returns 3 rows), so the throw did not 500 the read. - expect(mocks.findMany).toHaveBeenCalled(); + expect(mocks.queryRaw).toHaveBeenCalled(); expect(result).toHaveLength(3); }); }); diff --git a/src/lib/measurements/daily-series-read.ts b/src/lib/measurements/daily-series-read.ts index f63e6a163..8c20993ab 100644 --- a/src/lib/measurements/daily-series-read.ts +++ b/src/lib/measurements/daily-series-read.ts @@ -1,29 +1,26 @@ /** - * v1.18.6 — shared rollup-backed daily-series reader. + * Shared daily-series reader for the chart surfaces. * - * Extracted from the `GET /api/measurements` rollup-daily branch so the - * single-type route AND the batched series endpoint - * (`GET /api/measurements/series-batch`) read through the SAME path - * instead of duplicating the rollup SQL + the coverage-fallback fold. - * - * `readDailySeries` reproduces the route's `source=rollup` + - * `aggregate=daily` contract byte-for-byte: collapse per-source rollup - * rows to the ladder-canonical reading per day, probe for an - * under-converged rollup window and fold inline when the live table - * carries more distinct days, and fall through to the live `date_trunc` - * aggregate when the rollup is empty for the window (brand-new accounts / - * write-hook race). The returned rows match the wire shape the chart - * client consumes (`{ type, value, measuredAt, count, minValue?, - * maxValue? }`). + * Serves the `GET /api/measurements?aggregate=daily&source=rollup` contract + * and the batched series endpoint (`GET /api/measurements/series-batch`) from + * one path, so both read the same numbers. Windows up to the DAY cap are + * folded live, one row per local calendar day of the user (#1026); longer + * windows step up to the WEEK / MONTH / YEAR rollup tier. The returned rows + * match the wire shape the chart client consumes (`{ type, value, + * measuredAt, count, minValue?, maxValue? }`), with `measuredAt` the instant + * of the bucket's local midnight. */ import { prisma } from "@/lib/db"; -import { BUCKET_CAP } from "@/lib/measurements/range-aggregation"; +import { + BUCKET_CAP, + type AggregateGrain, +} from "@/lib/measurements/range-aggregation"; import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; -import { collapseRollupRowsBySource } from "@/lib/rollups/measurement-read"; import { readTieredRollupSeries } from "@/lib/rollups/measurement-read-wmy"; -import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; import { annotate } from "@/lib/logging/context"; +import { isValidTimezone } from "@/lib/tz/format"; +import { resolveUserTimezone } from "@/lib/tz/resolver"; import type { MeasurementType } from "@/generated/prisma/client"; import { Prisma } from "@/generated/prisma/client"; @@ -37,12 +34,9 @@ export interface DailySeriesRow { maxValue?: number | null; } -const SUSPICIOUS_ROW_FLOOR = 3; -const SUSPICIOUS_WINDOW_DAYS = 7; - /** - * Read one type's daily series for `[from, to]` through the rollup tier, - * falling back to the live `date_trunc` aggregate on a coverage miss. + * Read one type's daily series for `[from, to]`, one row per local day of + * the user (or per tier bucket when the window exceeds the DAY cap). * * `priorityJson` is the user's source-priority ladder (load once per * request via `loadUserSourcePriority` and thread it across types so the @@ -56,6 +50,8 @@ export async function readDailySeries(opts: { to: Date; limit?: number; priorityJson: unknown; + /** IANA zone the days are cut in; defaults to the user's own zone. */ + timeZone?: string; }): Promise { const { userId, type, from, to, priorityJson } = opts; const cap = Math.min(opts.limit ?? BUCKET_CAP.daily, BUCKET_CAP.daily); @@ -106,10 +102,10 @@ export async function readDailySeries(opts: { return tiered.rows; } // Coverage miss at every tier — fall through to the daily path, which - // probes the live table and folds an under-converged window. The - // daily fallback still caps at `cap`; the annotate below records that - // the long-range read could not step up so a truncated daily slice is - // an explicit, logged outcome rather than a silent one. + // folds the live table. The daily fallback still caps at `cap`; the + // annotate below records that the long-range read could not step up so + // a truncated daily slice is an explicit, logged outcome rather than a + // silent one. annotate({ meta: { tiered_series_coverage_miss: true, @@ -119,139 +115,94 @@ export async function readDailySeries(opts: { }); } - const readRollup = async () => - collapseRollupRowsBySource( - await prisma.measurementRollup.findMany({ - where: { - userId, - type, - granularity: "DAY", - bucketStart: { gte: from, lte: to }, - }, - orderBy: { bucketStart: "asc" }, - select: { - type: true, - source: true, - bucketStart: true, - mean: true, - count: true, - sumValue: true, - minValue: true, - maxValue: true, - }, - }), - type, - priorityJson, - ).slice(0, cap); - - // F-DB-2 — the primary rollup read is wrapped: a transient rollup-table error - // (statement_timeout, deadlock, connection reset) must NOT 500 the tile when - // the live table can still serve it. Treat a throw as a coverage miss and fall - // straight through to the live `date_trunc` aggregate — the same posture the - // batched series reader already takes, so the single-type route and the batch - // route no longer diverge on a rollup hiccup. - let rollupRows: Awaited>; - try { - rollupRows = await readRollup(); - } catch (err) { - annotate({ - meta: { - rollup_read_threw: true, - rollup_read_error: err instanceof Error ? err.message : String(err), - type, - }, - }); - return readLiveDaily({ userId, type, from, to, cap, priorityJson }); - } - - // Coverage-mismatch probe + inline fold (identical to the route). - const windowMs = to.getTime() - from.getTime(); - const windowDays = Math.ceil(windowMs / 86_400_000); - let effectiveRows = rollupRows; - let coverageFallbackFired = false; - if ( - rollupRows.length > 0 && - rollupRows.length < SUSPICIOUS_ROW_FLOOR && - windowDays >= SUSPICIOUS_WINDOW_DAYS - ) { - const liveDayCountRows = await prisma.$queryRaw>` - SELECT COUNT(DISTINCT date_trunc('day', m."measured_at"))::bigint AS days - FROM measurements m - WHERE m."user_id" = ${userId} - AND m."type" = ${type}::measurement_type - AND m."measured_at" >= ${from} - AND m."measured_at" <= ${to} - AND m."deleted_at" IS NULL - `; - const liveDays = Number(liveDayCountRows[0]?.days ?? BigInt(0)); - if (liveDays > rollupRows.length) { - coverageFallbackFired = true; - try { - await recomputeUserRollups(userId, { - types: [type], - granularities: ["DAY"], - from, - to, - }); - effectiveRows = await readRollup(); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - annotate({ - meta: { - rollup_coverage_fallback_failed: true, - rollup_coverage_fallback_error: message, - type, - }, - }); - effectiveRows = []; - } - } - } - - if (effectiveRows.length > 0) { - const useSum = CUMULATIVE_HK_TYPES.has(type); - annotate({ - action: { name: "measurement.list" }, - meta: { - total: effectiveRows.length, - type, - aggregate: "daily", - source: "rollup", - coverage_fallback: coverageFallbackFired, - }, - }); - return effectiveRows.map((r) => ({ - type: r.type, - value: useSum ? (r.sumValue ?? r.mean * r.count) : r.mean, - measuredAt: r.bucketStart.toISOString(), - count: r.count, - minValue: useSum ? undefined : r.minValue, - maxValue: useSum ? undefined : r.maxValue, - })); - } - - // Live `date_trunc` fallback — rollup empty for the window. - return readLiveDaily({ userId, type, from, to, cap, priorityJson }); + // The daily series is cut at the user's own midnight (#1026). The DAY + // rollup tier cannot serve it: its buckets are UTC days, so east of UTC a + // sample shortly after local midnight counted on the previous day, and + // every bucket's UTC-midnight start read as the previous local day west of + // UTC. The 7-day chart buckets raw rows by the local day, so switching + // ranges moved daily totals between neighbouring days. The live fold is + // one indexed pass per window (about 0.1 s for 90 days of per-minute + // samples) and returns at most one row per day. + const timeZone = opts.timeZone ?? (await resolveUserTimezone(userId)); + const rows = await readLiveBuckets({ + userId, + type, + from, + to, + cap, + priorityJson, + grain: "daily", + timeZone, + }); + annotate({ + action: { name: "measurement.list" }, + meta: { total: rows.length, type, aggregate: "daily", source: "live" }, + }); + return rows; } -/** Live `date_trunc('day', …)` aggregate — the rollup-empty fallback. */ -async function readLiveDaily(opts: { +/** SQL `date_trunc` unit per aggregate grain (a closed map, never input). */ +const TRUNC_UNIT: Record, string> = { + daily: "day", + weekly: "week", + monthly: "month", +}; + +/** + * Live aggregate of the measurements table into buckets cut at the user's + * own calendar boundaries. + * + * Each sample is converted to wall-clock time in `timeZone` before + * truncation, so a bucket is the user's local day (or week / month), and the + * returned `measuredAt` is the instant of that local midnight. The chart + * labels a row by formatting `measuredAt` in the same zone, which lands on + * the bucket's own day for every offset, including half-hour and negative + * ones, and across a DST change. + * + * Per bucket the overlapping sources collapse to the ladder-canonical one + * before the fold (the same pick the rollup reader makes), cumulative types + * SUM and every other type averages. The result set is at most `cap` rows, + * however dense the underlying stream: the fold runs in Postgres. + * + * `type` narrows to one type; `null` folds every type in the window (the + * multi-type `GET /api/measurements?aggregate=` form). + */ +export async function readLiveBuckets(opts: { userId: string; - type: MeasurementType; + type: MeasurementType | null; from: Date; to: Date; cap: number; priorityJson: unknown; + grain: Exclude; + timeZone: string; }): Promise { - const { userId, type, from, to, cap, priorityJson } = opts; - const truncUnitLiteral = Prisma.raw(`'day'`); - const useSum = CUMULATIVE_HK_TYPES.has(type); - const aggregator = useSum - ? Prisma.raw(`SUM(m."value")::double precision`) - : Prisma.raw(`AVG(m."value")::double precision`); + const { userId, type, from, to, cap, priorityJson, grain } = opts; + // A zone Postgres does not know would 500 the read; the resolver already + // validates stored zones, this guards a caller passing a raw string. + const timeZone = isValidTimezone(opts.timeZone) ? opts.timeZone : "UTC"; + // `date_trunc` needs its unit as a literal; the unit comes from the closed + // `TRUNC_UNIT` map keyed by the Zod-constrained grain, never from input. + const unit = Prisma.raw(`'${TRUNC_UNIT[grain]}'`); + // Cumulative types SUM per bucket, every other type averages. The type list + // is the closed code constant, spliced as literals; `s."type"` is a GROUP BY + // column, so choosing the aggregate per group is legal. + const cumulativeList = [...CUMULATIVE_HK_TYPES] + .map((t) => `'${t}'`) + .join(","); + const aggregator = Prisma.raw( + `(CASE WHEN s."type"::text IN (${cumulativeList}) THEN SUM(s."value") ELSE AVG(s."value") END)::double precision`, + ); const rankRaw = Prisma.raw( buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'), ); + const typeFilter = type + ? Prisma.sql`AND m."type" = ${type}::measurement_type` + : Prisma.empty; + // measured_at is a UTC wall-clock timestamp: pin it to UTC, re-read it in + // the user's zone and truncate there. The bucket is computed once in `src` + // and referenced by name afterwards, so the zone parameter never has to + // match itself across SELECT and GROUP BY. const buckets = await prisma.$queryRaw< Array<{ type: string; @@ -262,58 +213,54 @@ async function readLiveDaily(opts: { max_value: number | null; }> >` - WITH canon AS ( - SELECT DISTINCT ON (m."type", date_trunc(${truncUnitLiteral}, m."measured_at")) - m."type" AS t, - date_trunc(${truncUnitLiteral}, m."measured_at") AS d, - m."source" AS canon + WITH src AS ( + SELECT + m."type", + m."source", + m."value", + date_trunc(${unit}, (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}) AS d FROM measurements m WHERE m."user_id" = ${userId} AND m."measured_at" >= ${from} AND m."measured_at" <= ${to} AND m."deleted_at" IS NULL - AND m."type" = ${type}::measurement_type - ORDER BY m."type", date_trunc(${truncUnitLiteral}, m."measured_at"), (${rankRaw}), m."source" + ${typeFilter} + ), + canon AS ( + SELECT DISTINCT ON (m."type", m.d) + m."type" AS t, + m.d AS d, + m."source" AS canon + FROM src m + ORDER BY m."type", m.d, (${rankRaw}), m."source" ) SELECT - m."type"::text AS type, - date_trunc(${truncUnitLiteral}, m."measured_at") AS bucket_start, + s."type"::text AS type, + s.d AT TIME ZONE ${timeZone} AS bucket_start, ${aggregator} AS avg, COUNT(*)::int AS cnt, - MIN(m."value") AS min_value, - MAX(m."value") AS max_value - FROM measurements m + MIN(s."value") AS min_value, + MAX(s."value") AS max_value + FROM src s JOIN canon c - ON c.t = m."type" - AND c.d = date_trunc(${truncUnitLiteral}, m."measured_at") - AND c.canon = m."source" - WHERE m."user_id" = ${userId} - AND m."measured_at" >= ${from} - AND m."measured_at" <= ${to} - AND m."deleted_at" IS NULL - AND m."type" = ${type}::measurement_type - GROUP BY m."type", bucket_start - ORDER BY bucket_start ASC + ON c.t = s."type" + AND c.d = s.d + AND c.canon = s."source" + GROUP BY s."type", s.d + ORDER BY s.d ASC LIMIT ${cap} `; - annotate({ - action: { name: "measurement.list" }, - meta: { total: buckets.length, type, aggregate: "daily" }, + return buckets.map((b) => { + const isSum = CUMULATIVE_HK_TYPES.has(b.type as MeasurementType); + return { + type: b.type, + value: Number(b.avg), + measuredAt: b.bucket_start.toISOString(), + count: Number(b.cnt), + // A SUM has no meaningful intra-day spread; the chart draws the band + // for averaged types only. + minValue: isSum || b.min_value === null ? undefined : Number(b.min_value), + maxValue: isSum || b.max_value === null ? undefined : Number(b.max_value), + }; }); - return buckets.map((b) => ({ - type: b.type, - value: Number(b.avg), - measuredAt: b.bucket_start.toISOString(), - count: Number(b.cnt), - // The rollup path emits these; the live fallback used to omit them, so a - // rollup-cold account silently lost the min/max spread band on the chart - // for the same underlying data — contradicting this file's own - // byte-for-byte parity claim. - minValue: b.min_value === null ? undefined : Number(b.min_value), - maxValue: b.max_value === null ? undefined : Number(b.max_value), - // The rollup path emits these; the live fallback used to omit them, so a - // rollup-cold account silently lost the min/max spread band on the chart - // for the same underlying data — contradicting this file's own - // byte-for-byte parity claim. - })); } diff --git a/tests/integration/daily-series-user-tz.test.ts b/tests/integration/daily-series-user-tz.test.ts new file mode 100644 index 000000000..e96270b81 --- /dev/null +++ b/tests/integration/daily-series-user-tz.test.ts @@ -0,0 +1,127 @@ +/** + * Issue #1026 — a daily total must land on the same calendar day whichever + * chart range reads it. + * + * The 7-day chart pulls raw rows and buckets them by the user's own calendar + * day. The 30-day chart reads the daily series from the rollup tier, whose + * DAY buckets are cut at UTC midnight, and the chart then labels each bucket's + * UTC-midnight start in the user's zone. East of UTC a sample shortly after + * local midnight was counted on the previous day; west of UTC every bucket + * was labelled one day early. Steps, active energy, flights and distance are + * summed per day, so the drift showed up as whole totals moving between + * neighbouring days when the range changed. + * + * This file seeds cumulative samples right next to local midnight for a + * positive, a half-hour and a negative offset, warms the rollup tier the way + * the live instance has it, and asserts that the daily series (both the + * rollup-backed read and the live aggregate) buckets by the user's day. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { + readDailySeries, + readLiveBuckets, +} from "@/lib/measurements/daily-series-read"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; +import { userDayKey } from "@/lib/tz/format"; +import { localHmAsUtc } from "@/lib/tz/local-day"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); +}); + +const DAY = new Date("2026-09-10T12:00:00Z"); +const NEXT_DAY = new Date("2026-09-11T12:00:00Z"); + +async function seed(tz: string): Promise { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { + username: `tz-${tz.replace(/\W/g, "-").toLowerCase()}`, + email: `tz-${tz.replace(/\W/g, "-").toLowerCase()}@example.test`, + role: "USER", + timezone: tz, + }, + }); + invalidateUserTimezone(user.id); + const rows = [ + // Just after local midnight: belongs to DAY. + { at: localHmAsUtc(DAY, tz, 0, 20), value: 100 }, + // Just before the next local midnight: still DAY. + { at: localHmAsUtc(DAY, tz, 23, 40), value: 1000 }, + // A daily total stamped at the start of the next local day. + { at: localHmAsUtc(NEXT_DAY, tz, 0, 0), value: 7 }, + { at: localHmAsUtc(NEXT_DAY, tz, 12, 0), value: 3 }, + ]; + await prisma.measurement.createMany({ + data: rows.map((r) => ({ + userId: user.id, + type: "ACTIVITY_STEPS" as const, + value: r.value, + unit: "count", + measuredAt: r.at, + source: "APPLE_HEALTH" as const, + })), + }); + return user.id; +} + +function byLocalDay( + rows: Array<{ measuredAt: string; value: number }>, + tz: string, +): Record { + const out: Record = {}; + for (const r of rows) { + const key = userDayKey(new Date(r.measuredAt), tz); + out[key] = (out[key] ?? 0) + r.value; + } + return out; +} + +const EXPECTED = { "2026-09-10": 1100, "2026-09-11": 10 }; + +describe("daily series buckets by the user's calendar day (#1026)", () => { + for (const tz of ["Europe/Berlin", "Asia/Kolkata", "America/New_York"]) { + it(`chart daily read agrees with the local day (${tz})`, async () => { + const userId = await seed(tz); + const from = new Date("2026-09-05T00:00:00Z"); + const to = new Date("2026-09-15T00:00:00Z"); + // Warm the rollup tier exactly as the write hook / backfill does. + await recomputeUserRollups(userId, { + types: ["ACTIVITY_STEPS"], + granularities: ["DAY"], + from, + to, + }); + const rows = await readDailySeries({ + userId, + type: "ACTIVITY_STEPS", + from, + to, + priorityJson: null, + }); + expect(byLocalDay(rows, tz)).toEqual(EXPECTED); + }); + + it(`live daily aggregate agrees with the local day (${tz})`, async () => { + const userId = await seed(tz); + const rows = await readLiveBuckets({ + userId, + type: "ACTIVITY_STEPS", + from: new Date("2026-09-05T00:00:00Z"), + to: new Date("2026-09-15T00:00:00Z"), + cap: 366, + priorityJson: null, + grain: "daily", + timeZone: tz, + }); + expect(byLocalDay(rows, tz)).toEqual(EXPECTED); + }); + } +}); diff --git a/tests/integration/measurements-aggregate-daily.test.ts b/tests/integration/measurements-aggregate-daily.test.ts index a4b88631e..baeb54aad 100644 --- a/tests/integration/measurements-aggregate-daily.test.ts +++ b/tests/integration/measurements-aggregate-daily.test.ts @@ -14,6 +14,7 @@ */ import { NextRequest } from "next/server"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { userDayKey } from "@/lib/tz/format"; import { cookieJar } from "./mock-next-headers"; import { getPrismaClient, truncateAllTables } from "./setup"; @@ -128,11 +129,15 @@ describe("GET /api/measurements?aggregate=… (real Postgres)", () => { expect(json.data.meta.aggregate).toBe("daily"); expect(json.data.measurements).toHaveLength(2); - const may1 = json.data.measurements.find((m) => - m.measuredAt.startsWith("2026-05-01"), + // Buckets are the account's own calendar days (Europe/Berlin, the + // column default), keyed by the instant of local midnight (#1026). + const may1 = json.data.measurements.find( + (m) => + userDayKey(new Date(m.measuredAt), "Europe/Berlin") === "2026-05-01", ); - const may2 = json.data.measurements.find((m) => - m.measuredAt.startsWith("2026-05-02"), + const may2 = json.data.measurements.find( + (m) => + userDayKey(new Date(m.measuredAt), "Europe/Berlin") === "2026-05-02", ); expect(may1).toBeDefined(); expect(may1!.count).toBe(2); From 5a0b6ee1538729df65fc135d32acf2ce0581dad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 24 Sep 2026 07:56:39 +0200 Subject: [PATCH 3/7] fix(insights): fold dense measurement reads per day in SQL A single account syncing heart rate from a watch took the worker down within half an hour. The status cards' graded series read the last 91 days of raw rows, bounded in time but not in rows, so a stream sampled every minute or faster materialised six or seven figures of objects on every refresh and the read ran into the statement timeout. Add readDayAggregates, which folds one type into per-day sum, count, min and max in Postgres (optionally split into age segments and filtered to the metric's plausibility domain), and move the readers that folded raw rows into days in memory onto it: the graded series (recent, weekly and the cold-tier fallback, for every type), the vitals day-mean fallback behind the personal band, the tiered context's recent days and the plan review. Recent, weekly and monthly buckets are identical to the raw fold; the cold-tier yearly slope now runs over day means. On 91 days of per-minute pulse the graded series now allocates 1.6 MB where it allocated 62 MB, while the raw read alone holds 22 MB. A structural guard freezes the remaining untaken measurement reads in the worker and insight directories: each must name a sparse literal type or carry a listed reason. Refs #1023 --- .../dense-measurement-read-guard.test.ts | 177 +++++++++++++ .../insights/cards/__tests__/route.test.ts | 8 + .../__tests__/blood-pressure-status.test.ts | 8 + src/lib/insights/__tests__/bmi-status.test.ts | 8 + .../__tests__/graded-series-rollups.test.ts | 224 +++++++++-------- .../insights/__tests__/metric-status.test.ts | 8 + .../insights/__tests__/payload-budget.test.ts | 8 + .../__tests__/pulse-status-timeout.test.ts | 8 + .../insights/__tests__/pulse-status.test.ts | 8 + .../insights/__tests__/weight-status.test.ts | 8 + .../__tests__/baseline-plausibility.test.ts | 55 +++- .../derived/__tests__/baseline.test.ts | 8 + .../__tests__/coincident-deviation.test.ts | 8 + .../__tests__/dispatch-window-days.test.ts | 8 + .../derived/__tests__/dispatch.test.ts | 8 + .../derived/__tests__/readiness.test.ts | 8 + .../derived/__tests__/trajectory.test.ts | 8 + src/lib/insights/derived/baseline.ts | 65 ++--- src/lib/insights/graded-series.ts | 237 ++++++++++++------ src/lib/jobs/coach-plan-review.ts | 34 +-- .../__tests__/fake-day-aggregates.ts | 73 ++++++ src/lib/measurements/day-aggregates.ts | 122 +++++++++ src/lib/measurements/value-domain.ts | 14 +- .../rollups/__tests__/tiered-context.test.ts | 8 + src/lib/rollups/tiered-context.ts | 40 ++- tests/integration/dense-read-bounded.test.ts | 209 +++++++++++++++ 26 files changed, 1083 insertions(+), 287 deletions(-) create mode 100644 src/__tests__/dense-measurement-read-guard.test.ts create mode 100644 src/lib/measurements/__tests__/fake-day-aggregates.ts create mode 100644 src/lib/measurements/day-aggregates.ts create mode 100644 tests/integration/dense-read-bounded.test.ts diff --git a/src/__tests__/dense-measurement-read-guard.test.ts b/src/__tests__/dense-measurement-read-guard.test.ts new file mode 100644 index 000000000..d669c99d4 --- /dev/null +++ b/src/__tests__/dense-measurement-read-guard.test.ts @@ -0,0 +1,177 @@ +/** + * Structural guard: background and insight readers must not materialise a + * dense measurement stream. + * + * #1023: a single account syncing heart rate from a watch took the worker + * down within half an hour. The status-card read was bounded in time + * (`measuredAt >= now − 91 days`) but not in rows, and a stream sampled every + * minute or faster turns 91 days into six or seven figures of objects. The + * read also ran into the statement timeout. The cure was to fold per day in + * SQL (`readDayAggregates`, `readLiveBuckets`), so the reader's size tracks + * the number of days, not the sampling rate. + * + * What this guard pins, in the directories that run on the worker or feed an + * insight: every `measurement.findMany` without a `take` reads a type that is + * named as a literal and is not one a device streams. A read whose type is a + * variable could be handed a dense type by the next caller, so it must be + * listed below with the reason it is safe. The list is frozen both ways: a new + * unlisted read fails, and an entry whose read is gone fails too. + * + * It is a tripwire, not a proof: the matcher reads source text, and a read + * written some other way (a raw SELECT of rows, a helper in another + * directory) passes it. The floor below fails the guard if the matcher stops + * finding reads at all. + */ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { walkSourceFiles } from "./helpers/source-files"; +import { + CUMULATIVE_HK_TYPES, + HIGH_FREQUENCY_MEAN_TYPES, +} from "@/lib/measurements/apple-health-mapping"; + +const SRC = join(process.cwd(), "src"); + +const SCOPED_DIRS = [ + "lib/insights/", + "lib/jobs/", + "lib/rollups/", + "lib/ai/", + "lib/analytics/", + "lib/dashboard/", + "lib/daily/", +]; + +/** Types a device writes at sampling rate rather than a few times a day. */ +const DENSE_TYPES: ReadonlySet = new Set([ + "PULSE", + "BLOOD_GLUCOSE", + "HEART_RATE_VARIABILITY", + "OXYGEN_SATURATION", + ...CUMULATIVE_HK_TYPES, + ...HIGH_FREQUENCY_MEAN_TYPES, +]); + +/** + * Untaken reads whose type is not a sparse literal, keyed + * `::`, with why each stays bounded. + */ +const ALLOWED: Record = { + "lib/insights/comprehensive-generate.ts::(shorthand)": + 'Only reached for SLEEP_DURATION (guarded by `type === "SLEEP_DURATION"`); a few stage rows per night.', + "lib/insights/derived/vascular-age.ts::VASCULAR_AGE_TYPE": + "A constant for a type written at most a few times a day.", + "lib/insights/derived/fitness-age.ts::VO2_MAX_TYPE": + "A constant for VO2 max, written at most daily.", + "lib/insights/derived/six-minute-walk.ts::SIX_MINUTE_WALK_TYPE": + "A constant for six-minute-walk distance, written at most daily.", + "lib/insights/derived/wellness-scores.ts::measurementType": + "Daily computed / provider scores (recovery, strain, readiness), one row per day.", + "lib/jobs/reaction-line.ts::{ in: [...types]": + "Exact-instant match (`measuredAt: row.occurredAt`), not a window.", + "lib/jobs/step-consolidation-repair.ts::STEP_TYPE": + "Repair job over one user's step rows for a bounded day range; runs once per repair, not per status refresh.", + 'lib/ai/coach/snapshot.ts::"BLOOD_GLUCOSE" as never': + "Clinical CGM metrics (time in range, GMI, CV) need individual readings; 30 days at a sensor's fixed rate (at most one a minute) stays below 45 000 rows.", + 'lib/dashboard/snapshot.ts::"BLOOD_GLUCOSE"': + "Same 30-day clinical glucose panel as the coach snapshot, grouped by meal context.", + "lib/ai/coach/cycle-snapshot.ts::{ in: PHASE_CROSSTAB_METRIC_TYPES": + "Cycle-phase crosstab over 365 days. Its dense members (steps, CGM glucose) are the known remaining case; steps are drained to one row per day nightly, CGM is not. Listed so it cannot grow unseen.", + "lib/analytics/score/reader.ts::{ in: types": + "Health-score inputs: steps (drained to one row per day nightly), sleep, waist, weight, blood pressure, fasting glucose only.", +}; + +interface UntakenRead { + file: string; + line: number; + typeExpr: string; +} + +function scopedFiles(): string[] { + return walkSourceFiles(SRC, { floor: 3000 }) + .filter((p) => SCOPED_DIRS.some((d) => p.startsWith(d))) + .filter((p) => !p.includes("__tests__")) + .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx")) + .sort(); +} + +function untakenReads(): { all: number; untaken: UntakenRead[] } { + let all = 0; + const untaken: UntakenRead[] = []; + for (const file of scopedFiles()) { + const src = readFileSync(join(SRC, file), "utf8"); + // Whitespace-tolerant: a call split across lines still matches. + const re = /\.measurement\s*\.\s*findMany\s*\(/g; + for (let m = re.exec(src); m; m = re.exec(src)) { + all += 1; + let i = m.index + m[0].length; + let depth = 1; + while (depth > 0 && i < src.length) { + if (src[i] === "(") depth += 1; + else if (src[i] === ")") depth -= 1; + i += 1; + } + const call = src.slice(m.index, i); + if (/\btake\b/.test(call)) continue; + const typed = /\btype\s*:\s*([^,\n}]+)/.exec(call); + const typeExpr = typed + ? typed[1].trim() + : /\btype\s*,/.test(call) + ? "(shorthand)" + : "(none)"; + untaken.push({ + file, + line: src.slice(0, m.index).split("\n").length, + typeExpr, + }); + } + } + return { all, untaken }; +} + +function sparseLiteral(typeExpr: string): boolean { + const lit = /^"([A-Z_0-9]+)"/.exec(typeExpr); + return lit !== null && !DENSE_TYPES.has(lit[1]); +} + +describe("dense measurement reads stay bounded (#1023)", () => { + const { all, untaken } = untakenReads(); + + it("finds the reads it is meant to police", () => { + // Guard against a matcher that silently matches nothing. + expect(all).toBeGreaterThanOrEqual(40); + expect(untaken.length).toBeGreaterThanOrEqual(15); + }); + + it("every untaken read names a sparse literal type or is listed with its reason", () => { + const offenders = untaken + .filter((r) => !sparseLiteral(r.typeExpr)) + .filter((r) => !(`${r.file}::${r.typeExpr}` in ALLOWED)) + .map((r) => `${r.file}:${r.line} type ${r.typeExpr}`); + expect(offenders).toEqual([]); + }); + + it("every listed exception still exists", () => { + const present = new Set(untaken.map((r) => `${r.file}::${r.typeExpr}`)); + expect(Object.keys(ALLOWED).filter((k) => !present.has(k))).toEqual([]); + }); + + it("the readers fixed for #1023 fold in SQL rather than reading rows", () => { + for (const file of [ + "lib/insights/graded-series.ts", + "lib/rollups/tiered-context.ts", + "lib/jobs/coach-plan-review.ts", + "lib/measurements/daily-series-read.ts", + ]) { + const src = readFileSync(join(SRC, file), "utf8"); + expect(src, file).not.toMatch(/\.measurement\s*\.\s*findMany\s*\(/); + } + const baseline = readFileSync( + join(SRC, "lib/insights/derived/baseline.ts"), + "utf8", + ); + expect(baseline).not.toMatch(/\.measurement\s*\.\s*findMany\s*\(/); + }); +}); diff --git a/src/app/api/insights/cards/__tests__/route.test.ts b/src/app/api/insights/cards/__tests__/route.test.ts index fbd8d1a08..d3252833b 100644 --- a/src/app/api/insights/cards/__tests__/route.test.ts +++ b/src/app/api/insights/cards/__tests__/route.test.ts @@ -1,6 +1,14 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; +// Day-mean reads fold per-day aggregates in SQL; the fake folds the mocked +// `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: vi.fn() }, diff --git a/src/lib/insights/__tests__/blood-pressure-status.test.ts b/src/lib/insights/__tests__/blood-pressure-status.test.ts index 437c065da..ce3d9f7a0 100644 --- a/src/lib/insights/__tests__/blood-pressure-status.test.ts +++ b/src/lib/insights/__tests__/blood-pressure-status.test.ts @@ -1,5 +1,13 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; +// The graded series folds per-day aggregates in SQL; the fake folds the +// mocked `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: vi.fn() }, diff --git a/src/lib/insights/__tests__/bmi-status.test.ts b/src/lib/insights/__tests__/bmi-status.test.ts index d0cc8da06..9db3e07a5 100644 --- a/src/lib/insights/__tests__/bmi-status.test.ts +++ b/src/lib/insights/__tests__/bmi-status.test.ts @@ -1,5 +1,13 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; +// The graded series folds per-day aggregates in SQL; the fake folds the +// mocked `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: vi.fn() }, diff --git a/src/lib/insights/__tests__/graded-series-rollups.test.ts b/src/lib/insights/__tests__/graded-series-rollups.test.ts index a6af77a54..1f386cfb8 100644 --- a/src/lib/insights/__tests__/graded-series-rollups.test.ts +++ b/src/lib/insights/__tests__/graded-series-rollups.test.ts @@ -1,7 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; const findMany = vi.fn(); -const queryRaw = vi.fn(); vi.mock("@/lib/db", () => ({ prisma: { measurement: { @@ -10,12 +9,14 @@ vi.mock("@/lib/db", () => ({ // loadUserSourcePriority lazy-loads the source-priority blob; null // (default findUnique) falls back to the default ladders. user: { findUnique: vi.fn() }, - // v1.28.25 — dense types (BLOOD_GLUCOSE / PULSE) day-bucket the - // cold-tier fallback in SQL. - $queryRaw: (...args: unknown[]) => queryRaw(...args), }, })); +const readDayAggregates = vi.fn(); +vi.mock("@/lib/measurements/day-aggregates", () => ({ + readDayAggregates: (...args: unknown[]) => readDayAggregates(...args), +})); + const readBestGranularityRollups = vi.fn(); const rollupRow = (bucketStart: Date, mean: number) => ({ bucketStart, @@ -31,147 +32,160 @@ const rollupRow = (bucketStart: Date, mean: number) => ({ vi.mock("@/lib/rollups/measurement-read-wmy", () => ({ readBestGranularityRollups: (...args: unknown[]) => readBestGranularityRollups(...args), - // aggregateWmyBuckets is no longer imported by graded-series, but keep the - // module shape intact for any transitive consumer. aggregateWmyBuckets: vi.fn(), })); -import { buildGradedSeriesWithRollups } from "../graded-series"; +import { + buildGradedSeriesFromDayAggregates, + buildGradedSeriesFromPoints, + buildGradedSeriesWithRollups, +} from "../graded-series"; +import { foldDayAggregates } from "@/lib/measurements/__tests__/fake-day-aggregates"; +import type { ReadDayAggregatesOptions } from "@/lib/measurements/day-aggregates"; const dayMs = 24 * 60 * 60 * 1000; -/** Daily readings spanning `days` back from `now`. */ -function dailyRows(days: number, now: Date) { +/** Readings spanning `days` back from `now`, `perDay` per day. */ +function points(days: number, now: Date, perDay = 1) { const rows: Array<{ measuredAt: Date; value: number }> = []; for (let i = 0; i < days; i++) { - rows.push({ measuredAt: new Date(now.getTime() - i * dayMs), value: 80 }); + for (let k = 0; k < perDay; k++) { + rows.push({ + measuredAt: new Date(now.getTime() - i * dayMs - k * 3_600_000), + value: 60 + ((i * 7 + k * 3) % 41), + }); + } } - return rows; + return rows.reverse(); +} + +/** Serve `readDayAggregates` from an in-memory row set. */ +function serveFrom(rows: Array<{ measuredAt: Date; value: number }>) { + readDayAggregates.mockImplementation(async (opts: ReadDayAggregatesOptions) => + foldDayAggregates(rows, opts), + ); } afterEach(() => { vi.clearAllMocks(); }); -describe("buildGradedSeriesWithRollups — rollup-miss fallback", () => { +describe("buildGradedSeriesWithRollups — bounded reads (#1023)", () => { const now = new Date("2026-05-31T12:00:00Z"); + it("never materialises raw rows: the recent read is a per-day aggregate", async () => { + serveFrom(points(80, now, 24)); + readBestGranularityRollups.mockResolvedValue({ + granularity: "MONTH", + rows: [rollupRow(new Date("2025-09-01T00:00:00Z"), 79)], + }); + + await buildGradedSeriesWithRollups("u1", "PULSE", now); + + expect(findMany).not.toHaveBeenCalled(); + const opts = readDayAggregates.mock.calls[0][0] as ReadDayAggregatesOptions; + expect(opts).toMatchObject({ userId: "u1", type: "PULSE" }); + expect(opts.until).toEqual(now); + expect(opts.timeZone).toBe("Europe/Berlin"); + expect(now.getTime() - opts.since.getTime()).toBe(91 * dayMs); + // The same age partition the parity test below proves exact: a reading + // exactly 21 / 91 / 451 days old belongs to the older slice. + expect(opts.segmentStarts).toEqual( + [21, 91, 451].map((d) => new Date(now.getTime() - d * dayMs + 1)), + ); + }); + it("folds monthly/yearly from the bounded fallback when the tier has no coverage", async () => { - // The bounded recent read (~90 d) returns only recent points; the - // fallback read returns 2+ years so the fold has monthly/yearly. - findMany - // 1st call — bounded recent read (gte: since) - .mockResolvedValueOnce(dailyRows(80, now)) - // 2nd call — bounded fallback read (gte: 1095-day horizon) - .mockResolvedValueOnce(dailyRows(800, now)); - - // Tier miss: no MONTH / YEAR coverage at all. + serveFrom(points(800, now)); readBestGranularityRollups.mockResolvedValue(null); const series = await buildGradedSeriesWithRollups("u1", "WEIGHT", now); - // The bug: before the fix monthly/yearly were derived from the - // bounded ~90-day read and so were always empty even when years of - // raw history existed. After the fix they fold from the fallback. - expect(series.monthly.length).toBeGreaterThan(0); - // 800 days of history puts at least one year into the yearly tail. - expect(series.yearly.length).toBeGreaterThan(0); - // The fallback read must have been issued... - expect(findMany).toHaveBeenCalledTimes(2); - // ...and bounded (v1.28.25): it must carry a `measuredAt.gte` floor - // ~1095 days back, never an unwindowed full-history walk. - const fallbackArgs = findMany.mock.calls[1][0] as { - where: { measuredAt?: { gte?: Date } }; - }; - const gte = fallbackArgs.where.measuredAt?.gte; - expect(gte).toBeInstanceOf(Date); - expect(gte!.getTime()).toBe(now.getTime() - 1095 * dayMs); - }); - - it("day-buckets the dense-type fallback in SQL instead of a raw walk (v1.28.25)", async () => { - // BLOOD_GLUCOSE on a cold tier (heavy import before the rollup - // backfill warms) used to findMany the entire CGM history. The - // fallback now runs a single day-bucket aggregate; only the bounded - // recent read touches findMany. - findMany.mockResolvedValueOnce(dailyRows(80, now)); // bounded recent - const bucketRows: Array<{ bucket_start: Date; mean: number }> = []; - for (let i = 0; i < 800; i++) { - bucketRows.push({ - bucket_start: new Date(now.getTime() - i * dayMs), - mean: 100, - }); - } - queryRaw.mockResolvedValueOnce(bucketRows.reverse()); - - readBestGranularityRollups.mockResolvedValue(null); - - const series = await buildGradedSeriesWithRollups( - "u1", - "BLOOD_GLUCOSE", - now, - ); - expect(series.monthly.length).toBeGreaterThan(0); expect(series.yearly.length).toBeGreaterThan(0); - // Raw findMany ran once (the bounded recent read) — the fallback - // itself went through the SQL aggregate. - expect(findMany).toHaveBeenCalledTimes(1); - expect(queryRaw).toHaveBeenCalledTimes(1); + expect(readDayAggregates).toHaveBeenCalledTimes(2); + const fallback = readDayAggregates.mock + .calls[1][0] as ReadDayAggregatesOptions; + // Bounded to the yearly router's horizon, never an unwindowed walk. + expect(fallback.since.getTime()).toBe(now.getTime() - 1095 * dayMs); + expect(findMany).not.toHaveBeenCalled(); }); - it("reads monthly/yearly from the tier when it has coverage and skips the full read", async () => { - findMany.mockResolvedValueOnce(dailyRows(80, now)); - - // MONTH coverage for the 1-year window, YEAR coverage for the 3-year. + it("reads monthly/yearly from the tier when it has coverage and skips the fallback", async () => { + serveFrom(points(80, now)); readBestGranularityRollups.mockImplementation( - async (_u: string, _t: string, windowDays: number) => { - if (windowDays === 365) { - return { - granularity: "MONTH", - rows: [ - rollupRow(new Date("2025-09-01T00:00:00Z"), 79), - rollupRow(new Date("2025-10-01T00:00:00Z"), 80), - ], - }; - } - return { - granularity: "YEAR", - rows: [rollupRow(new Date("2023-01-01T00:00:00Z"), 82)], - }; - }, + async (_u: string, _t: string, windowDays: number) => + windowDays === 365 + ? { + granularity: "MONTH", + rows: [ + rollupRow(new Date("2025-09-01T00:00:00Z"), 79), + rollupRow(new Date("2025-10-01T00:00:00Z"), 80), + ], + } + : { + granularity: "YEAR", + rows: [rollupRow(new Date("2023-01-01T00:00:00Z"), 82)], + }, ); const series = await buildGradedSeriesWithRollups("u1", "WEIGHT", now); expect(series.monthly.length).toBeGreaterThan(0); expect(series.yearly.length).toBeGreaterThan(0); - // Only the bounded recent read — no full-history fallback when the - // tier covers both coarse slices. - expect(findMany).toHaveBeenCalledTimes(1); + expect(readDayAggregates).toHaveBeenCalledTimes(1); }); - it("falls back to a full read for only the slice the tier misses", async () => { - findMany - .mockResolvedValueOnce(dailyRows(80, now)) // bounded recent - .mockResolvedValueOnce(dailyRows(800, now)); // full-history fallback - - // MONTH covered, YEAR missing. + it("falls back only for the slice the tier misses", async () => { + serveFrom(points(800, now)); readBestGranularityRollups.mockImplementation( - async (_u: string, _t: string, windowDays: number) => { - if (windowDays === 365) { - return { - granularity: "MONTH", - rows: [rollupRow(new Date("2025-10-01T00:00:00Z"), 80)], - }; - } - return null; - }, + async (_u: string, _t: string, windowDays: number) => + windowDays === 365 + ? { + granularity: "MONTH", + rows: [rollupRow(new Date("2025-10-01T00:00:00Z"), 80)], + } + : null, ); const series = await buildGradedSeriesWithRollups("u1", "WEIGHT", now); - expect(series.monthly.length).toBeGreaterThan(0); + expect(series.monthly).toHaveLength(1); expect(series.yearly.length).toBeGreaterThan(0); - expect(findMany).toHaveBeenCalledTimes(2); + expect(readDayAggregates).toHaveBeenCalledTimes(2); + }); +}); + +describe("buildGradedSeriesFromDayAggregates — parity with the raw fold", () => { + const now = new Date("2026-05-31T12:00:00Z"); + const segmentStarts = [21, 91, 451].map( + (d) => new Date(now.getTime() - d * dayMs + 1), + ); + + it("matches the raw-row fold for recent / weekly / monthly and yearly stats", () => { + // Several readings per day, including readings exactly on the segment + // edges, so the partition itself is under test. + const rows = [ + ...points(900, now, 5), + { measuredAt: new Date(now.getTime() - 21 * dayMs), value: 99 }, + { measuredAt: new Date(now.getTime() - 91 * dayMs), value: 41 }, + ]; + const raw = buildGradedSeriesFromPoints(rows, now); + const agg = buildGradedSeriesFromDayAggregates( + foldDayAggregates(rows, { + since: new Date(0), + until: now, + timeZone: "Europe/Berlin", + segmentStarts, + }), + ); + expect(agg.recent).toEqual(raw.recent); + expect(agg.weekly).toEqual(raw.weekly); + expect(agg.monthly).toEqual(raw.monthly); + // The yearly slope runs over day means rather than single readings; + // every other yearly figure is exact. + expect(agg.yearly.map(({ slope: _s, ...b }) => b)).toEqual( + raw.yearly.map(({ slope: _s, ...b }) => b), + ); }); }); diff --git a/src/lib/insights/__tests__/metric-status.test.ts b/src/lib/insights/__tests__/metric-status.test.ts index d930ac00f..89f68e479 100644 --- a/src/lib/insights/__tests__/metric-status.test.ts +++ b/src/lib/insights/__tests__/metric-status.test.ts @@ -1,5 +1,13 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; +// The graded series folds per-day aggregates in SQL; the fake folds the +// mocked `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: vi.fn() }, diff --git a/src/lib/insights/__tests__/payload-budget.test.ts b/src/lib/insights/__tests__/payload-budget.test.ts index 11de84479..36c84eb29 100644 --- a/src/lib/insights/__tests__/payload-budget.test.ts +++ b/src/lib/insights/__tests__/payload-budget.test.ts @@ -12,6 +12,14 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; * `runStatusCompletion` is stubbed to capture it. */ +// The graded series folds per-day aggregates in SQL; the fake folds the +// mocked `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: vi.fn() }, diff --git a/src/lib/insights/__tests__/pulse-status-timeout.test.ts b/src/lib/insights/__tests__/pulse-status-timeout.test.ts index 508cff3b2..7ecb3c025 100644 --- a/src/lib/insights/__tests__/pulse-status-timeout.test.ts +++ b/src/lib/insights/__tests__/pulse-status-timeout.test.ts @@ -19,6 +19,14 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; * generation on every navigation while a provider is degraded. */ +// The graded series folds per-day aggregates in SQL; the fake folds the +// mocked `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: vi.fn() }, diff --git a/src/lib/insights/__tests__/pulse-status.test.ts b/src/lib/insights/__tests__/pulse-status.test.ts index 1aed06d5c..ddf2632b2 100644 --- a/src/lib/insights/__tests__/pulse-status.test.ts +++ b/src/lib/insights/__tests__/pulse-status.test.ts @@ -1,5 +1,13 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; +// The graded series folds per-day aggregates in SQL; the fake folds the +// mocked `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: vi.fn() }, diff --git a/src/lib/insights/__tests__/weight-status.test.ts b/src/lib/insights/__tests__/weight-status.test.ts index ec4e98a37..5f715f219 100644 --- a/src/lib/insights/__tests__/weight-status.test.ts +++ b/src/lib/insights/__tests__/weight-status.test.ts @@ -1,5 +1,13 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; +// The graded series folds per-day aggregates in SQL; the fake folds the +// mocked `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { auditLog: { findFirst: vi.fn(), create: vi.fn() }, diff --git a/src/lib/insights/derived/__tests__/baseline-plausibility.test.ts b/src/lib/insights/derived/__tests__/baseline-plausibility.test.ts index 7a83f5b61..37a894773 100644 --- a/src/lib/insights/derived/__tests__/baseline-plausibility.test.ts +++ b/src/lib/insights/derived/__tests__/baseline-plausibility.test.ts @@ -12,9 +12,27 @@ * The numbers below are shaped like the real thing: a resting-pulse series in * the 60s and 70s, one impossible sample dropped into it. */ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; -import { buildBaselineBand, dayMeansFromRows } from "../baseline"; +import type { ReadDayAggregatesOptions } from "@/lib/measurements/day-aggregates"; + +// The live day-mean read folds in SQL; the fake folds `rowsForRead` with the +// same day / range rules, so these tests see the filter the read asks for. +let rowsForRead: { value: number; measuredAt: Date }[] = []; +const seenOptions: ReadDayAggregatesOptions[] = []; +vi.mock("@/lib/db", () => ({ prisma: {} })); +vi.mock("@/lib/measurements/day-aggregates", async () => { + const { foldDayAggregates } = + await import("@/lib/measurements/__tests__/fake-day-aggregates"); + return { + readDayAggregates: async (opts: ReadDayAggregatesOptions) => { + seenOptions.push(opts); + return foldDayAggregates(rowsForRead, opts); + }, + }; +}); + +import { buildBaselineBand, readDayMeanSeries } from "../baseline"; import { latestDayMeanFromRows } from "../coincident-deviation"; import { isPlausibleMetricValue } from "@/lib/measurements/value-domain"; @@ -47,28 +65,49 @@ describe("metric plausibility domain", () => { }); }); +/** The live (rollup-miss) day-mean read over `rows`. */ +async function dayMeansFromRows( + rows: { value: number; measuredAt: Date }[], + type: "PULSE", +) { + rowsForRead = rows; + const { points } = await readDayMeanSeries( + "u1", + type, + 3650, + new Date("2026-09-01T00:00:00Z"), + new Map(), + ); + return points; +} + describe("per-day means behind the personal band", () => { - it("keeps the day mean on the real readings when one sample is impossible", () => { + it("asks the read for the metric's own plausibility domain", async () => { + await dayMeansFromRows(pulseDay("2026-08-01", [66]), "PULSE"); + expect(seenOptions.at(-1)?.valueRange).toEqual({ min: 20, max: 300 }); + }); + + it("keeps the day mean on the real readings when one sample is impossible", async () => { const rows = pulseDay("2026-08-01", [66, 70, 111287531.01, 68]); - const [point] = dayMeansFromRows(rows, "PULSE"); + const [point] = await dayMeansFromRows(rows, "PULSE"); expect(point.day).toBe("2026-08-01"); expect(point.mean).toBeCloseTo(68, 6); }); - it("drops a day whose every reading is impossible rather than inventing one", () => { + it("drops a day whose every reading is impossible rather than inventing one", async () => { const rows = [ ...pulseDay("2026-08-01", [66, 70, 68]), ...pulseDay("2026-08-02", [36016.75, 111287531.01]), ]; - const points = dayMeansFromRows(rows, "PULSE"); + const points = await dayMeansFromRows(rows, "PULSE"); expect(points.map((p) => p.day)).toEqual(["2026-08-01"]); }); - it("holds the band inside the metric's own domain across a poisoned month", () => { + it("holds the band inside the metric's own domain across a poisoned month", async () => { const rows: { value: number; measuredAt: Date }[] = []; for (let d = 1; d <= 28; d += 1) { const day = `2026-08-${String(d).padStart(2, "0")}`; @@ -81,7 +120,7 @@ describe("per-day means behind the personal band", () => { rows.push(...pulseDay("2026-08-26", [98211.5])); const band = buildBaselineBand( - dayMeansFromRows(rows, "PULSE").map((p) => p.mean), + (await dayMeansFromRows(rows, "PULSE")).map((p) => p.mean), "PULSE", ); diff --git a/src/lib/insights/derived/__tests__/baseline.test.ts b/src/lib/insights/derived/__tests__/baseline.test.ts index a4742a89a..dd30f888f 100644 --- a/src/lib/insights/derived/__tests__/baseline.test.ts +++ b/src/lib/insights/derived/__tests__/baseline.test.ts @@ -1,5 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; +// Day-mean reads fold per-day aggregates in SQL; the fake folds the mocked +// `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { measurement: { findMany: vi.fn() } }, })); diff --git a/src/lib/insights/derived/__tests__/coincident-deviation.test.ts b/src/lib/insights/derived/__tests__/coincident-deviation.test.ts index c101ef457..8c0fc800f 100644 --- a/src/lib/insights/derived/__tests__/coincident-deviation.test.ts +++ b/src/lib/insights/derived/__tests__/coincident-deviation.test.ts @@ -1,5 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; +// Day-mean reads fold per-day aggregates in SQL; the fake folds the mocked +// `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { measurement: { findMany: vi.fn() } }, })); diff --git a/src/lib/insights/derived/__tests__/dispatch-window-days.test.ts b/src/lib/insights/derived/__tests__/dispatch-window-days.test.ts index 1a6618097..c99ca3301 100644 --- a/src/lib/insights/derived/__tests__/dispatch-window-days.test.ts +++ b/src/lib/insights/derived/__tests__/dispatch-window-days.test.ts @@ -17,6 +17,14 @@ */ import { describe, it, expect, vi, beforeEach } from "vitest"; +// Day-mean reads fold per-day aggregates in SQL; the fake folds the mocked +// `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { measurement: { diff --git a/src/lib/insights/derived/__tests__/dispatch.test.ts b/src/lib/insights/derived/__tests__/dispatch.test.ts index 267fc592c..02458a7ed 100644 --- a/src/lib/insights/derived/__tests__/dispatch.test.ts +++ b/src/lib/insights/derived/__tests__/dispatch.test.ts @@ -1,5 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; +// Day-mean reads fold per-day aggregates in SQL; the fake folds the mocked +// `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { measurement: { findMany: vi.fn().mockResolvedValue([]) }, diff --git a/src/lib/insights/derived/__tests__/readiness.test.ts b/src/lib/insights/derived/__tests__/readiness.test.ts index 0bb22411e..5371acb2f 100644 --- a/src/lib/insights/derived/__tests__/readiness.test.ts +++ b/src/lib/insights/derived/__tests__/readiness.test.ts @@ -1,5 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; +// Day-mean reads fold per-day aggregates in SQL; the fake folds the mocked +// `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { measurement: { findMany: vi.fn() }, diff --git a/src/lib/insights/derived/__tests__/trajectory.test.ts b/src/lib/insights/derived/__tests__/trajectory.test.ts index 705cfb00a..37c361161 100644 --- a/src/lib/insights/derived/__tests__/trajectory.test.ts +++ b/src/lib/insights/derived/__tests__/trajectory.test.ts @@ -1,5 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; +// Day-mean reads fold per-day aggregates in SQL; the fake folds the mocked +// `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("@/lib/db", () => ({ prisma: { measurement: { findMany: vi.fn() } }, })); diff --git a/src/lib/insights/derived/baseline.ts b/src/lib/insights/derived/baseline.ts index 881ae202d..d6319880f 100644 --- a/src/lib/insights/derived/baseline.ts +++ b/src/lib/insights/derived/baseline.ts @@ -29,12 +29,13 @@ * composed-bucket baseline matches the raw-DAY baseline within tolerance. */ import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; -import { prisma } from "@/lib/db"; +import { readDayAggregates } from "@/lib/measurements/day-aggregates"; import { getAgeFromDateOfBirth } from "@/lib/analytics/pulse-targets"; import { toProfileSex } from "@/lib/profile/sex"; import { clampDerivedLowerBound, isPlausibleMetricValue, + plausibleMetricRange, } from "@/lib/measurements/value-domain"; import type { ProfileSex } from "@/lib/profile/sex"; import { @@ -212,39 +213,6 @@ export function buildBaselineBand( }; } -/** - * Group raw readings into per-day means, keeping only the values that sit - * inside the metric's own plausibility domain. - * - * Pure, and exported because this is the arithmetic the band rests on: a - * single stored value the application itself declares impossible drags the - * day's mean with it, the median then carries it into the band, and the - * personal "usual range" reads in the tens of thousands. Filtering here rather - * than at the edge of the card keeps the number wrong in exactly one place — - * nowhere — instead of right on one surface and wrong on the next. - * - * A day whose every reading is implausible produces no point at all. That is - * the honest outcome: the day contributed no measurement, and an absent day is - * something the coverage floor already knows how to report. - */ -export function dayMeansFromRows( - rows: readonly { value: number; measuredAt: Date }[], - type: MeasurementType, -): DayMeanPoint[] { - const byDay = new Map(); - for (const row of rows) { - if (!isPlausibleMetricValue(type, row.value)) continue; - const day = row.measuredAt.toISOString().slice(0, 10); - const acc = byDay.get(day) ?? { sum: 0, count: 0 }; - acc.sum += row.value; - acc.count += 1; - byDay.set(day, acc); - } - return [...byDay.entries()] - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - .map(([day, acc]) => ({ day, mean: acc.sum / acc.count })); -} - // ── reads ──────────────────────────────────────────────────────────── /** @@ -292,23 +260,22 @@ export async function readDayMeanSeries( // the spread is still DAY-native. } - // Per-type live fallback — bounded raw read, grouped into per-day - // means. Honest provenance: source = "live". + // Per-type live fallback — per-day means folded in SQL, so the read is + // one row per day however densely the metric is sampled (#1023). Only + // values inside the metric's own plausibility domain count: a single + // stored value the application declares impossible would drag the day's + // mean with it and the median would carry it into the band. A day whose + // every reading is implausible produces no point at all. Days are UTC + // days, matching the DAY rollup path above. const since = new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000); - const rows = await prisma.measurement.findMany({ - where: { - userId, - type, - deletedAt: null, - measuredAt: { gte: since }, - }, - orderBy: { measuredAt: "asc" }, - select: { value: true, measuredAt: true }, + const days = await readDayAggregates({ + userId, + type, + since, + timeZone: "UTC", + valueRange: plausibleMetricRange(type), }); - if (rows.length === 0) { - return { points: [], source: "none" }; - } - const points = dayMeansFromRows(rows, type); + const points = days.map((d) => ({ day: d.day, mean: d.sum / d.n })); if (points.length === 0) { return { points: [], source: "none" }; } diff --git a/src/lib/insights/graded-series.ts b/src/lib/insights/graded-series.ts index 0053d6d5d..cf31d08f9 100644 --- a/src/lib/insights/graded-series.ts +++ b/src/lib/insights/graded-series.ts @@ -48,12 +48,15 @@ */ import type { MeasurementType } from "@/generated/prisma/client"; -import { prisma } from "@/lib/db"; import { readBestGranularityRollups, type RollupBucketRow, } from "@/lib/rollups/measurement-read-wmy"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { + readDayAggregates, + type DayAggregateRow, +} from "@/lib/measurements/day-aggregates"; import { toBerlinYmd } from "@/lib/tz/resolver"; const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -83,18 +86,6 @@ const MONTHLY_WINDOW_MS = WEEKLY_WINDOW_MS + MONTHLY_MONTHS * 30 * MS_PER_DAY; */ const COLD_FALLBACK_WINDOW_DAYS = 1095; -/** - * Sample-dense types whose cold-tier fallback is day-bucketed in SQL - * (one AVG row per day) instead of read raw — same dense set the series - * route caps (CGM glucose ~288 rows/day, PULSE per-sample / hourly). - * The fold's output shape is unchanged; on the cold path only, a dense - * type's monthly/yearly min/max/n derive from day means rather than - * individual samples — the warm rollup tier restores sample-exact - * figures as soon as the backfill lands. - */ -const DENSE_FALLBACK_TYPES: ReadonlySet = - new Set(["BLOOD_GLUCOSE", "PULSE"]); - export interface RecentDayBucket { /** Berlin YYYY-MM-DD. */ date: string; @@ -171,6 +162,11 @@ function berlinYearKey(date: Date): string { */ function berlinIsoWeekKey(date: Date): string { const { year, month, day } = toBerlinYmd(date); + return isoWeekKeyOfYmd(year, month, day); +} + +/** ISO week key of a calendar date given as `YYYY`, `MM`, `DD`. */ +function isoWeekKeyOfYmd(year: string, month: string, day: string): string { const localMidnight = new Date( Date.UTC(Number(year), Number(month) - 1, Number(day)), ); @@ -309,6 +305,133 @@ export function buildGradedSeriesFromPoints( return { recent, weekly, monthly, yearly }; } +function mergeInto( + map: Map, + key: string, + row: DayAggregateRow, +): void { + const existing = map.get(key); + if (existing) { + existing.sum += row.sum; + existing.n += row.n; + if (row.min < existing.min) existing.min = row.min; + if (row.max > existing.max) existing.max = row.max; + } else { + map.set(key, { sum: row.sum, min: row.min, max: row.max, n: row.n }); + } +} + +/** Segment boundaries matching `buildGradedSeriesFromPoints`' age split. */ +function gradedSegmentStarts(now: Date): Date[] { + const nowMs = now.getTime(); + return [ + new Date(nowMs - RECENT_WINDOW_MS), + new Date(nowMs - WEEKLY_WINDOW_MS), + new Date(nowMs - MONTHLY_WINDOW_MS), + ]; +} + +/** + * Read the graded series' per-day aggregates for one type, folded in SQL. + * + * Replaces the raw `findMany` over the window (#1023): a watch that streams + * heart rate every few seconds put a million rows into that read, which is + * what exhausted the worker's heap. The rows here are one per Berlin day per + * age segment, and their sum / count / min / max are exact, so the recent, + * weekly and monthly slices come out identical to folding the raw rows. + * + * The segment boundaries use the same edges as `buildGradedSeriesFromPoints`: + * a row exactly `RECENT_WINDOW_MS` old is weekly there, so it is segment 1 + * here too. + */ +async function readGradedDayAggregates( + userId: string, + type: MeasurementType, + since: Date, + now: Date, +): Promise { + return readDayAggregates({ + userId, + type, + since, + // Future-dated readings were skipped by the in-memory fold. + until: now, + timeZone: "Europe/Berlin", + // `readDayAggregates` puts a row at segment k when it is older than k + // starts (`measured_at < start`). The in-memory fold treats an age of + // exactly the window as the older bucket, so shift each start by 1 ms. + segmentStarts: gradedSegmentStarts(now).map( + (d) => new Date(d.getTime() + 1), + ), + }); +} + +/** + * The graded series folded from per-day aggregates (segment = age class). + * + * recent / weekly / monthly and every bucket's min / max / mean / n match + * the raw-row fold exactly. The yearly slope is taken over the day means in + * date order rather than over individual readings: a direction signal either + * way, and the only form that stays bounded on a dense stream. + */ +export function buildGradedSeriesFromDayAggregates( + rows: readonly DayAggregateRow[], +): GradedSeries { + const recentAgg = new Map(); + const weeklyAgg = new Map(); + const monthlyAgg = new Map(); + const yearlyAgg = new Map(); + const yearlyValues = new Map(); + + const ordered = [...rows].sort((a, b) => + a.day < b.day ? -1 : a.day > b.day ? 1 : b.segment - a.segment, + ); + for (const row of ordered) { + const [year, month, day] = row.day.split("-"); + if (row.segment === 0) { + mergeInto(recentAgg, row.day, row); + } else if (row.segment === 1) { + mergeInto(weeklyAgg, isoWeekKeyOfYmd(year, month, day), row); + } else if (row.segment === 2) { + mergeInto(monthlyAgg, `${year}-${month}`, row); + } else { + mergeInto(yearlyAgg, year, row); + const list = yearlyValues.get(year); + const dayMean = row.sum / row.n; + if (list) list.push(dayMean); + else yearlyValues.set(year, [dayMean]); + } + } + + const toBuckets = (map: Map) => + Array.from(map.entries()) + .map(([key, a]) => ({ + key, + min: round(a.min), + max: round(a.max), + mean: round(a.sum / a.n), + n: a.n, + })) + .sort((x, y) => x.key.localeCompare(y.key)); + + return { + recent: toBuckets(recentAgg).map(({ key, ...b }) => ({ date: key, ...b })), + weekly: toBuckets(weeklyAgg).map(({ key, ...b }) => ({ + weekISO: key, + ...b, + })), + monthly: toBuckets(monthlyAgg).map(({ key, ...b }) => ({ + month: key, + ...b, + })), + yearly: toBuckets(yearlyAgg).map(({ key, ...b }) => ({ + year: key, + ...b, + slope: leastSquaresSlope(yearlyValues.get(key) ?? []), + })), + }; +} + /** * Project a rollup bucket row to a monthly/yearly graded bucket. The * tier already carries min/max/mean/slope per bucket — no JS folding. @@ -335,24 +458,25 @@ function rollupYearly(rows: RollupBucketRow[]): YearlyBucket[] { } /** - * Build the graded series with the recent / weekly slices folded from a - * bounded raw read and the monthly / yearly slices read from the + * Build the graded series with the recent / weekly slices folded from + * per-day aggregates and the monthly / yearly slices read from the * pre-aggregated rollup tier. * - * The recent / weekly read is capped to the recent + weekly horizon - * (last ~90 days), so the common warm-tier path never pulls the full - * multi-year history into memory. The rollup router picks MONTH for the - * ~1-year window and YEAR for the multi-year tail. + * The recent / weekly read covers the recent + weekly horizon (last ~90 + * days) and is folded in SQL to one row per day, so its size depends on + * the number of days, never on how densely a device samples (#1023). The + * rollup router picks MONTH for the ~1-year window and YEAR for the + * multi-year tail. * * Coverage-miss fallback: when the tier has no MONTH (resp. YEAR) * coverage for a metric — a fresh account the boot-backfill has not yet * caught up on — the bounded ~90-day read can NOT supply the monthly / * yearly slices (every row in it folds into recent / weekly), so naively * reusing it would ship empty monthly / yearly arrays even when years of - * raw history exist. On that miss this falls back to an in-memory fold - * over a bounded window (1095 days — the yearly router's own horizon; - * sample-dense types are additionally day-bucketed in SQL) so the coarse - * slices are populated from the raw rows rather than left falsely empty. + * raw history exist. On that miss this falls back to the same per-day + * aggregate over a bounded window (1095 days — the yearly router's own + * horizon) so the coarse slices are populated from the stored rows rather + * than left falsely empty. * The fallback read only happens when the tier is cold, which is exactly * the case the write-amplified tier was meant to avoid — so the hot path * stays bounded and the cold path stays correct. @@ -363,15 +487,8 @@ export async function buildGradedSeriesWithRollups( now: Date, ): Promise { const since = new Date(now.getTime() - WEEKLY_WINDOW_MS); - const rawRecent = await prisma.measurement.findMany({ - where: { userId, type, deletedAt: null, measuredAt: { gte: since } }, - orderBy: { measuredAt: "asc" }, - select: { measuredAt: true, value: true }, - }); - - const recentGraded = buildGradedSeriesFromPoints( - rawRecent.map((r) => ({ measuredAt: r.measuredAt, value: r.value })), - now, + const recentGraded = buildGradedSeriesFromDayAggregates( + await readGradedDayAggregates(userId, type, since, now), ); // v1.11.2 — load the source-priority blob once and thread it into both @@ -420,57 +537,19 @@ export async function buildGradedSeriesWithRollups( } // Tier coverage miss for one or both coarse slices: the bounded - // ~90-day read can't supply them, so fold a BOUNDED history window in - // memory and take whichever slices the tier left empty. The fallback - // read is the exception (cold-tier accounts), never the warm-tier - // norm. v1.28.25 — the read is bounded to the same 1095-day horizon - // the yearly rollup router serves (it was previously unwindowed), and - // the sample-dense types are day-bucketed in SQL so a heavy import - // ahead of the rollup backfill can't drag six-figure row sets into JS. + // ~90-day read can't supply them, so fold a BOUNDED history window and + // take whichever slices the tier left empty. The fallback read is the + // exception (cold-tier accounts), never the warm-tier norm. It is + // bounded to the same 1095-day horizon the yearly rollup router serves, + // and folded per day in SQL for every type, so a heavy import ahead of + // the rollup backfill can't drag six-figure row sets into JS. if (!monthlyCovered || !yearlyCovered) { const fallbackSince = new Date( now.getTime() - COLD_FALLBACK_WINDOW_DAYS * MS_PER_DAY, ); - let fallbackPoints: Point[]; - if (DENSE_FALLBACK_TYPES.has(type)) { - // Parameter-bound day-bucket aggregate, mirroring the rollup tier's - // `date_trunc(...) GROUP BY` (measurement-rollups.ts) — at most - // ~1095 rows regardless of sample density. - const bucketRows = await prisma.$queryRaw< - Array<{ bucket_start: Date; mean: number }> - >` - SELECT - date_trunc('day', m."measured_at") AS bucket_start, - AVG(m."value")::double precision AS mean - FROM measurements m - WHERE m."user_id" = ${userId} - AND m."type" = ${type}::"measurement_type" - AND m."measured_at" >= ${fallbackSince} - AND m."deleted_at" IS NULL - GROUP BY date_trunc('day', m."measured_at") - ORDER BY bucket_start ASC - `; - fallbackPoints = bucketRows.map((r) => ({ - measuredAt: r.bucket_start, - value: r.mean, - })); - } else { - const rows = await prisma.measurement.findMany({ - where: { - userId, - type, - deletedAt: null, - measuredAt: { gte: fallbackSince }, - }, - orderBy: { measuredAt: "asc" }, - select: { measuredAt: true, value: true }, - }); - fallbackPoints = rows.map((r) => ({ - measuredAt: r.measuredAt, - value: r.value, - })); - } - const fullGraded = buildGradedSeriesFromPoints(fallbackPoints, now); + const fullGraded = buildGradedSeriesFromDayAggregates( + await readGradedDayAggregates(userId, type, fallbackSince, now), + ); if (!monthlyCovered) monthly = fullGraded.monthly; if (!yearlyCovered) yearly = fullGraded.yearly; } diff --git a/src/lib/jobs/coach-plan-review.ts b/src/lib/jobs/coach-plan-review.ts index 1b753f1a1..acf500618 100644 --- a/src/lib/jobs/coach-plan-review.ts +++ b/src/lib/jobs/coach-plan-review.ts @@ -24,6 +24,7 @@ */ import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; import { encryptToBytes } from "@/lib/ai/coach/bytes-codec"; +import { readDayAggregates } from "@/lib/measurements/day-aggregates"; export const COACH_PLAN_REVIEW_QUEUE = "coach-plan-review"; // Daily at 05:25 Europe/Berlin — just after the 05:20 reminder sweep, so the @@ -49,7 +50,7 @@ export interface CoachPlanReviewSummary { type ReviewPrisma = Pick< PrismaClient, - "coachReminder" | "coachPlan" | "measurement" + "coachReminder" | "coachPlan" | "$queryRaw" >; /** Metric valence — whether a higher reading is the better outcome. */ @@ -282,31 +283,22 @@ export async function runCoachPlanReviewTick( const since = new Date( plan.createdAt.getTime() - BEFORE_WINDOW_DAYS * MS_PER_DAY, ); - const rows = await prisma.measurement.findMany({ - where: { - userId: plan.userId, - type: resolved.type, - deletedAt: null, - measuredAt: { gte: since, lte: now }, - }, - orderBy: { measuredAt: "asc" }, - select: { value: true, measuredAt: true }, + // Day means folded in SQL (UTC days), one row per day however + // densely the metric is sampled (#1023). + const byDay = await readDayAggregates({ + userId: plan.userId, + type: resolved.type, + since, + until: now, + timeZone: "UTC", + db: prisma, }); - // Day-mean the raw rows, then split at the experiment start. - const byDay = new Map(); - for (const row of rows) { - const day = row.measuredAt.toISOString().slice(0, 10); - const agg = byDay.get(day) ?? { sum: 0, count: 0 }; - agg.sum += row.value; - agg.count += 1; - byDay.set(day, agg); - } const startKey = plan.createdAt.toISOString().slice(0, 10); const before: number[] = []; const after: number[] = []; const all: number[] = []; - for (const [day, agg] of byDay) { - const dayMean = agg.sum / agg.count; + for (const { day, sum, n } of byDay) { + const dayMean = sum / n; all.push(dayMean); if (day < startKey) before.push(dayMean); else after.push(dayMean); diff --git a/src/lib/measurements/__tests__/fake-day-aggregates.ts b/src/lib/measurements/__tests__/fake-day-aggregates.ts new file mode 100644 index 000000000..311e8350c --- /dev/null +++ b/src/lib/measurements/__tests__/fake-day-aggregates.ts @@ -0,0 +1,73 @@ +/** + * In-memory stand-in for `readDayAggregates`, for unit tests that mock + * `@/lib/db`. + * + * It reads the rows through the mocked `prisma.measurement.findMany` (so a + * test's existing row fixture keeps feeding the reader) and folds them with + * the same day / segment / range rules the SQL applies. The integration test + * `tests/integration/dense-read-bounded.test.ts` pins that this fold and the + * SQL agree on real Postgres. + */ +import { prisma } from "@/lib/db"; +import type { + DayAggregateRow, + ReadDayAggregatesOptions, +} from "@/lib/measurements/day-aggregates"; +import { userDayKey } from "@/lib/tz/format"; + +export function foldDayAggregates( + rows: ReadonlyArray<{ measuredAt: Date; value: number }>, + opts: Omit, +): DayAggregateRow[] { + const starts = opts.segmentStarts ?? []; + const byKey = new Map(); + for (const r of rows) { + const t = r.measuredAt.getTime(); + if (t < opts.since.getTime()) continue; + if (opts.until && t > opts.until.getTime()) continue; + if ( + opts.valueRange && + (r.value < opts.valueRange.min || r.value > opts.valueRange.max) + ) { + continue; + } + const day = userDayKey(r.measuredAt, opts.timeZone); + const segment = starts.filter((s) => t < s.getTime()).length; + const key = `${day}|${segment}`; + const acc = byKey.get(key); + if (acc) { + acc.n += 1; + acc.sum += r.value; + if (r.value < acc.min) acc.min = r.value; + if (r.value > acc.max) acc.max = r.value; + } else { + byKey.set(key, { + day, + segment, + n: 1, + sum: r.value, + min: r.value, + max: r.value, + }); + } + } + return [...byKey.values()].sort((a, b) => + a.day < b.day ? -1 : a.day > b.day ? 1 : b.segment - a.segment, + ); +} + +export async function fakeReadDayAggregates( + opts: ReadDayAggregatesOptions, +): Promise { + const rows = ((await prisma.measurement.findMany({ + where: { + userId: opts.userId, + type: opts.type, + deletedAt: null, + measuredAt: { gte: opts.since, ...(opts.until && { lte: opts.until }) }, + }, + orderBy: { measuredAt: "asc" }, + select: { measuredAt: true, value: true }, + })) ?? []) as Array<{ measuredAt: Date; value: number }>; + return foldDayAggregates(rows, opts); +} diff --git a/src/lib/measurements/day-aggregates.ts b/src/lib/measurements/day-aggregates.ts new file mode 100644 index 000000000..083b59b78 --- /dev/null +++ b/src/lib/measurements/day-aggregates.ts @@ -0,0 +1,122 @@ +/** + * Per-day aggregates of one measurement type, folded in Postgres. + * + * Several background readers (the status cards' graded series, the vitals + * baseline, the tiered context, the plan review) used to `findMany` every raw + * row in a time window and then fold them into days in JavaScript. The window + * bounded the time span but not the row count, so an account whose watch + * streams heart rate every few seconds materialised hundreds of thousands to + * millions of objects per read. In the worker that was enough to exhaust the + * heap, and the read itself outran the statement timeout. + * + * This reader returns at most one row per (day, segment), whatever the sample + * density: the fold runs in SQL and only the aggregates cross the wire. `sum`, + * `n`, `min` and `max` are exact, so every consumer that averaged, counted or + * took extremes over the raw rows gets the same numbers from these rows. + * + * Days are cut in `timeZone`. `segmentStarts` optionally splits the window at + * instants (newest first): a row's segment is the number of starts it lies + * before, so a caller that treated "the last 21 days" and "the 70 days before + * that" differently keeps exactly the same per-row partition. + */ +import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import { Prisma } from "@/generated/prisma/client"; +import { prisma } from "@/lib/db"; +import { isValidTimezone } from "@/lib/tz/format"; + +export interface DayAggregateRow { + /** Calendar day in the requested zone, `YYYY-MM-DD`. */ + day: string; + /** Segment index: 0 for rows at or after `segmentStarts[0]`, and so on. */ + segment: number; + n: number; + sum: number; + min: number; + max: number; +} + +export interface ReadDayAggregatesOptions { + userId: string; + type: MeasurementType; + /** Inclusive lower bound on `measuredAt`. */ + since: Date; + /** Inclusive upper bound on `measuredAt`; omit for no upper bound. */ + until?: Date; + /** IANA zone the day boundary is cut in. */ + timeZone: string; + /** Rows outside `[min, max]` are dropped before the fold. */ + valueRange?: { min: number; max: number }; + /** Segment boundaries, newest first. */ + segmentStarts?: readonly Date[]; + /** Client to read through; defaults to the shared one. */ + db?: Pick; +} + +export async function readDayAggregates( + opts: ReadDayAggregatesOptions, +): Promise { + const timeZone = isValidTimezone(opts.timeZone) ? opts.timeZone : "UTC"; + const until = opts.until + ? Prisma.sql`AND m."measured_at" <= ${opts.until}` + : Prisma.empty; + const range = opts.valueRange + ? Prisma.sql`AND m."value" >= ${opts.valueRange.min} AND m."value" <= ${opts.valueRange.max}` + : Prisma.empty; + const starts = opts.segmentStarts ?? []; + const segment = + starts.length === 0 + ? Prisma.sql`0` + : Prisma.sql`(${Prisma.join( + starts.map( + (s) => + Prisma.sql`(CASE WHEN m."measured_at" < ${s} THEN 1 ELSE 0 END)`, + ), + " + ", + )})`; + + // The day and segment are computed once per row in `src` and grouped by + // name, so the bound zone / boundary parameters never have to match + // themselves across SELECT and GROUP BY. + const rows = await (opts.db ?? prisma).$queryRaw< + Array<{ + day: string; + segment: number; + n: number; + sum: number; + min: number; + max: number; + }> + >` + WITH src AS ( + SELECT + to_char((m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}, 'YYYY-MM-DD') AS day, + ${segment} AS segment, + m."value" AS value + FROM measurements m + WHERE m."user_id" = ${opts.userId} + AND m."type" = ${opts.type}::measurement_type + AND m."deleted_at" IS NULL + AND m."measured_at" >= ${opts.since} + ${until} + ${range} + ) + SELECT + day, + segment::int AS segment, + COUNT(*)::int AS n, + SUM(value)::double precision AS sum, + MIN(value)::double precision AS min, + MAX(value)::double precision AS max + FROM src + GROUP BY day, segment + ORDER BY day ASC, segment DESC + `; + return rows.map((r) => ({ + day: r.day, + segment: Number(r.segment), + n: Number(r.n), + sum: Number(r.sum), + min: Number(r.min), + max: Number(r.max), + })); +} diff --git a/src/lib/measurements/value-domain.ts b/src/lib/measurements/value-domain.ts index fdfad09c4..d6e485827 100644 --- a/src/lib/measurements/value-domain.ts +++ b/src/lib/measurements/value-domain.ts @@ -105,12 +105,22 @@ export function isPlausibleMetricValue( value: number, ): boolean { if (!Number.isFinite(value)) return false; - if (!type) return true; - const range = VALUE_RANGES[type]; + const range = plausibleMetricRange(type); if (range === undefined) return true; return value >= range.min && value <= range.max; } +/** + * The plausibility domain of a metric, for a reader that filters in SQL + * rather than row by row (`readDayAggregates`' `valueRange`). `undefined` + * when the metric declares none, which admits every value. + */ +export function plausibleMetricRange( + type: string | null | undefined, +): { min: number; max: number } | undefined { + return type ? VALUE_RANGES[type] : undefined; +} + /** * True when EVERY named metric forbids negative values (and at least one was * named). Used by the chart, which can paint several series in one axis: a diff --git a/src/lib/rollups/__tests__/tiered-context.test.ts b/src/lib/rollups/__tests__/tiered-context.test.ts index 4998abc97..64610312b 100644 --- a/src/lib/rollups/__tests__/tiered-context.test.ts +++ b/src/lib/rollups/__tests__/tiered-context.test.ts @@ -23,6 +23,14 @@ const mocks = vi.hoisted(() => ({ queryRaw: vi.fn(), })); +// Day-mean reads fold per-day aggregates in SQL; the fake folds the mocked +// `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadDayAggregates, +})); + vi.mock("../measurement-rollups", () => ({ readRollupBuckets: mocks.readRollupBuckets, ensureUserRollupsFresh: mocks.ensureUserRollupsFresh, diff --git a/src/lib/rollups/tiered-context.ts b/src/lib/rollups/tiered-context.ts index 1546f3460..fdf43b62c 100644 --- a/src/lib/rollups/tiered-context.ts +++ b/src/lib/rollups/tiered-context.ts @@ -33,12 +33,13 @@ import type { MeasurementType, RollupGranularity, } from "@/generated/prisma/client"; -import { isNearUtc, userDayKey } from "@/lib/tz/format"; +import { isNearUtc } from "@/lib/tz/format"; import { ensureUserRollupsFresh, readRollupBuckets, } from "./measurement-rollups"; import { probeRollupCoverage } from "./measurement-coverage"; +import { readDayAggregates } from "@/lib/measurements/day-aggregates"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -112,7 +113,7 @@ export interface TieredSeries { export interface BuildTieredSeriesOptions { /** * The user's IANA timezone. REQUIRED — the raw 0–14d band keys its - * daily fold on the user's local calendar day (`userDayKey`) so a + * daily fold on the user's local calendar day so a * late-evening reading lands on the right day across DST boundaries * and for non-UTC users. The coarser DAY/WEEK/MONTH/YEAR bands read * straight from the rollup table, whose `bucketStart` is UTC-anchored @@ -244,8 +245,8 @@ function extractAnomalies( } /** - * Read the raw 0–14d daily points for one type, folding same-day rows. - * Days are keyed on the user's local calendar via `userDayKey(tz)`, so a + * Read the 0–14d daily points for one type, folding same-day rows. + * Days are keyed on the user's local calendar in `tz`, so a * 23:30-local reading folds onto its local day rather than the UTC day — * correct across DST transitions and for non-UTC users. */ @@ -256,26 +257,19 @@ async function readRecentDaily( tz: string, ): Promise { const since = new Date(now - TIERED_BANDS.rawDays * DAY_MS); - const rows = await prisma.measurement.findMany({ - where: { userId, type, measuredAt: { gte: since }, deletedAt: null }, - select: { value: true, measuredAt: true }, - orderBy: { measuredAt: "asc" }, + // Folded per local day in SQL: one row per day however densely the type + // is sampled, never the raw stream (#1023). + const days = await readDayAggregates({ + userId, + type, + since, + timeZone: tz, }); - const byDay = new Map(); - for (const r of rows) { - const key = userDayKey(r.measuredAt, tz); - const entry = byDay.get(key) ?? { sum: 0, count: 0 }; - entry.sum += r.value; - entry.count += 1; - byDay.set(key, entry); - } - return [...byDay.entries()] - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - .map(([date, { sum, count }]) => ({ - date, - value: round(sum / count), - count, - })); + return days.map((d) => ({ + date: d.day, + value: round(d.sum / d.n), + count: d.n, + })); } /** diff --git a/tests/integration/dense-read-bounded.test.ts b/tests/integration/dense-read-bounded.test.ts new file mode 100644 index 000000000..009a566a6 --- /dev/null +++ b/tests/integration/dense-read-bounded.test.ts @@ -0,0 +1,209 @@ +/** + * #1023 — a status card over a dense heart-rate stream, measured. + * + * A single account syncing heart rate from a watch killed the worker within + * half an hour: the pulse status card's recent-window read was bounded in + * time (91 days) but not in rows, so every refresh materialised the whole + * stream as objects, and the read itself ran past the statement timeout. + * The read now folds per day in Postgres. + * + * The fixture is one reading a minute for 91 days, 131 040 rows, seeded in + * one `INSERT … generate_series`. Each half of the memory test reads the + * same window: the per-day fold the card now uses, and the raw `findMany` it + * used to issue. Without the second half the budget could be any number and + * the first half would still pass. Every reading of the heap is taken after a + * forced collection, so what is compared is what each read HOLDS; the + * transient figure (taken before collecting) is reported alongside. + * + * The parity case pins that the SQL fold and the in-memory fold the unit + * tests use agree on real Postgres: same days in a non-UTC zone, same + * segment edges, same plausibility filter. + */ +import v8 from "node:v8"; +import vm from "node:vm"; + +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { buildGradedSeriesWithRollups } from "@/lib/insights/graded-series"; +import { readDayAggregates } from "@/lib/measurements/day-aggregates"; +import { foldDayAggregates } from "@/lib/measurements/__tests__/fake-day-aggregates"; +import { localHmAsUtc } from "@/lib/tz/local-day"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const DENSE_USER = "dense-pulse-owner"; +const PARITY_USER = "dense-parity-owner"; +const DAYS = 91; +const ROWS = DAYS * 24 * 60; +const NOW = new Date("2026-09-20T18:00:00.000Z"); + +const forceGc = ((): (() => void) => { + v8.setFlagsFromString("--expose-gc"); + const gc = vm.runInNewContext("gc") as () => void; + v8.setFlagsFromString("--no-expose-gc"); + return gc; +})(); + +function liveHeapBytes(): number { + forceGc(); + forceGc(); + return process.memoryUsage().heapUsed; +} + +const mb = (bytes: number): string => (bytes / 1024 / 1024).toFixed(1); + +function report(line: string): void { + process.stderr.write(`[dense-read] ${line}\n`); +} + +/** Run `read`, returning what it holds, its transient peak and its time. */ +async function measure(read: () => Promise): Promise<{ + result: T; + heldBytes: number; + transientBytes: number; + ms: number; +}> { + const before = liveHeapBytes(); + const t0 = performance.now(); + const result = await read(); + const ms = performance.now() - t0; + const transientBytes = process.memoryUsage().heapUsed - before; + const heldBytes = liveHeapBytes() - before; + return { result, heldBytes, transientBytes, ms }; +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { + id: DENSE_USER, + username: DENSE_USER, + timezone: "Europe/Berlin", + }, + }); + await prisma.$executeRawUnsafe( + `INSERT INTO measurements ( + id, user_id, type, value, unit, source, measured_at, + created_at, updated_at, sync_version) + SELECT + 'dp' || lpad(g::text, 8, '0'), + $1, + 'PULSE'::measurement_type, + 55 + (g % 50), + 'bpm', + 'GOOGLE_HEALTH'::measurement_source, + ($2::timestamptz AT TIME ZONE 'UTC') - (g * interval '1 minute'), + now(), now(), 1 + FROM generate_series(1, $3::int) g`, + DENSE_USER, + NOW.toISOString(), + ROWS, + ); + await prisma.$executeRawUnsafe(`ANALYZE measurements`); +}, 300_000); + +afterAll(async () => { + await truncateAllTables(prisma); +}); + +describe("pulse status over a dense stream (#1023)", () => { + it("builds the graded series without materialising the stream", async () => { + const graded = await measure(() => + buildGradedSeriesWithRollups(DENSE_USER, "PULSE", NOW), + ); + const series = graded.result; + // Every reading in the 91-day window is accounted for, one bucket per day + // or week, not one object per reading. The oldest reading sits exactly 91 + // days back, which the graded split files under the older slice. + const counted = [...series.recent, ...series.weekly].reduce( + (sum, b) => sum + b.n, + 0, + ); + expect(counted).toBe(ROWS - 1); + expect(series.recent.length).toBeLessThanOrEqual(22); + expect(series.weekly.length).toBeLessThanOrEqual(12); + + const raw = await measure(() => + prisma.measurement.findMany({ + where: { + userId: DENSE_USER, + type: "PULSE", + deletedAt: null, + measuredAt: { gte: new Date(NOW.getTime() - DAYS * 86_400_000) }, + }, + orderBy: { measuredAt: "asc" }, + select: { measuredAt: true, value: true }, + }), + ); + expect(raw.result.length).toBe(ROWS); + + report( + `graded series (per-day fold, cold tier): held ${mb(graded.heldBytes)} MB, ` + + `transient ${mb(graded.transientBytes)} MB, ${graded.ms.toFixed(0)} ms`, + ); + report( + `raw 91-day findMany (the old read): held ${mb(raw.heldBytes)} MB, ` + + `transient ${mb(raw.transientBytes)} MB, ${raw.ms.toFixed(0)} ms, ` + + `${raw.result.length} rows`, + ); + + // The fold holds a few kilobytes; the raw read holds the stream. The + // transient figure is the one that killed the worker: the old builder + // held little once it returned but allocated the whole stream on the + // way (62 MB on this fixture, against 1.6 MB for the fold). + expect(graded.heldBytes).toBeLessThan(2 * 1024 * 1024); + expect(graded.transientBytes).toBeLessThan(12 * 1024 * 1024); + expect(raw.heldBytes).toBeGreaterThan(20 * Math.max(graded.heldBytes, 1)); + }, 120_000); + + it("the SQL fold agrees with the in-memory fold on real Postgres", async () => { + const tz = "Asia/Kolkata"; + await prisma.user.create({ + data: { id: PARITY_USER, username: PARITY_USER, timezone: tz }, + }); + const day = new Date("2026-09-10T12:00:00Z"); + const edge = new Date(NOW.getTime() - 21 * 86_400_000); + const rows = [ + { at: localHmAsUtc(day, tz, 0, 5), value: 61 }, + { at: localHmAsUtc(day, tz, 23, 55), value: 99 }, + { at: localHmAsUtc(day, tz, 12, 0), value: 111_287_531 }, // impossible + { at: edge, value: 70 }, + { at: new Date(edge.getTime() + 1), value: 72 }, + { at: new Date(NOW.getTime() - 200 * 86_400_000), value: 58 }, + ]; + await prisma.measurement.createMany({ + data: rows.map((r, i) => ({ + id: `parity-${i}`, + userId: PARITY_USER, + type: "PULSE" as const, + value: r.value, + unit: "bpm", + source: "MANUAL" as const, + measuredAt: r.at, + })), + }); + const opts = { + since: new Date(NOW.getTime() - 400 * 86_400_000), + until: NOW, + timeZone: tz, + valueRange: { min: 20, max: 300 }, + segmentStarts: [edge, new Date(NOW.getTime() - 91 * 86_400_000)], + }; + const sql = await readDayAggregates({ + userId: PARITY_USER, + type: "PULSE", + ...opts, + }); + const memory = foldDayAggregates( + rows.map((r) => ({ measuredAt: r.at, value: r.value })), + opts, + ); + expect(sql).toEqual(memory); + // Sanity: the impossible value was dropped and the edge row split off. + expect(sql.reduce((n, r) => n + r.n, 0)).toBe(5); + }); +}); From 2f5d58dad2de16cfb77b23c455e9642f718b7152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 24 Sep 2026 08:00:57 +0200 Subject: [PATCH 4/7] fix(charts): read the 7-day range as daily rows On a per-minute heart-rate stream the 7-day chart showed only the first half of the week. The range read raw rows oldest-first under a 5000-row limit, and seven days of per-minute samples are 10 080 rows, so the newest days never arrived. The chart folds every range into days anyway, so the 7-day range now asks for one row per local day like the longer ranges. The whole week arrives whatever the sampling rate, and the 7-day and 30-day ranges share one day boundary and one source pick per day. The 7-day range still draws no spread band, so its picture is unchanged. Refs #1026 --- .../__tests__/health-chart-range.test.tsx | 9 +- src/components/charts/chart-series-request.ts | 27 +++++ src/components/charts/health-chart.tsx | 67 +++-------- tests/integration/chart-range-dense.test.ts | 106 ++++++++++++++++++ 4 files changed, 155 insertions(+), 54 deletions(-) create mode 100644 src/components/charts/chart-series-request.ts create mode 100644 tests/integration/chart-range-dense.test.ts diff --git a/src/components/charts/__tests__/health-chart-range.test.tsx b/src/components/charts/__tests__/health-chart-range.test.tsx index 9cbed264c..3db9cb2f6 100644 --- a/src/components/charts/__tests__/health-chart-range.test.tsx +++ b/src/components/charts/__tests__/health-chart-range.test.tsx @@ -93,9 +93,10 @@ describe(" — bounded range fetch", () => { expect(measurementsUrl!).toMatch(/aggregate=daily/); }); - it("omits aggregate=daily on windows of 7 days or fewer", async () => { - // v1.4.29 C3 — short windows keep raw fetching so the user can - // see hour-by-hour detail on the 7-day view. + it("asks for daily rows on the 7-day window too", async () => { + // The chart folds every range into days. The 7-day window used to read + // raw rows oldest-first under the 5000-row cap, which on a per-minute + // stream cut off the newest half of the week. const { I18nProvider } = await import("@/lib/i18n/context"); const { HealthChart } = await import("../health-chart"); @@ -116,7 +117,7 @@ describe(" — bounded range fetch", () => { u.includes("/api/measurements"), ); expect(measurementsUrl).toBeDefined(); - expect(measurementsUrl!).not.toMatch(/aggregate=daily/); + expect(measurementsUrl!).toMatch(/aggregate=daily/); }); it("re-keys the query when the range window changes", async () => { diff --git a/src/components/charts/chart-series-request.ts b/src/components/charts/chart-series-request.ts new file mode 100644 index 000000000..3656f6956 --- /dev/null +++ b/src/components/charts/chart-series-request.ts @@ -0,0 +1,27 @@ +/** + * Query string for one type's chart series (`GET /api/measurements`). + * + * Every range asks for one row per local calendar day. The 7-day range used + * to ask for raw rows oldest-first under a 5 000-row limit; a per-minute + * heart-rate stream holds 10 080 rows in seven days, so the answer stopped + * half-way through the week and the newest days were silently missing. The + * chart folds rows into days either way, so the daily answer draws the same + * points while its size depends only on the number of days. + * + * Pure so the request the chart makes can be tested against the real route. + */ +export function chartSeriesParams( + type: string, + window: { from: string; to: string }, +): URLSearchParams { + const params = new URLSearchParams(); + params.set("type", type); + params.set("sortBy", "measuredAt"); + params.set("sortDir", "asc"); + params.set("from", window.from); + params.set("to", window.to); + params.set("limit", "5000"); + params.set("aggregate", "daily"); + params.set("source", "rollup"); + return params; +} diff --git a/src/components/charts/health-chart.tsx b/src/components/charts/health-chart.tsx index 6948494cf..23f75dc76 100644 --- a/src/components/charts/health-chart.tsx +++ b/src/components/charts/health-chart.tsx @@ -1,5 +1,6 @@ "use client"; +import { chartSeriesParams } from "@/components/charts/chart-series-request"; import { keepPreviousData, useQuery } from "@tanstack/react-query"; import { useAuth } from "@/hooks/use-auth"; import { useRecordCapabilities } from "@/hooks/use-record-capabilities"; @@ -327,8 +328,8 @@ interface MeasurementApiRow { value: number; // v1.8.5 — per-day min / max emitted by the rollup daily-aggregate // path (non-cumulative metrics only). Drive the Apple-Health-style - // range band shaded around the mean line. Absent on the raw-row path - // (windows ≤ 7 days) and on cumulative metrics. + // range band shaded around the mean line. Not drawn on the 7-day range + // and absent on cumulative metrics. // v1.18.6 — `null` accepted alongside `undefined` so the batched // series endpoint's row shape (which carries `number | null`) feeds // `preloadedSeries` directly. The bucketing loop already guards each @@ -340,8 +341,7 @@ interface MeasurementApiRow { // the underlying raw-row count per bucket. Used downstream to gate // the "more days needed" empty-state copy on the actual measurement // count rather than the number of distinct calendar days. Optional - // because the short-window raw-row path does not populate it (each - // row already represents one measurement, count = 1). + // because the per-night sleep series does not carry it. count?: number; } @@ -764,12 +764,11 @@ export function HealthChart({ }; }, [rangePoints, effectiveCompareBaseline]); - // v1.18.6 — honour the dashboard's batched slice ONLY for the - // daily-aggregate window (`windowDays > 7`, the rollup path) and ONLY - // when EVERY non-sleep type the chart renders is present in the slice. - // SLEEP_DURATION is never batched (per-night `/series`), so a chart - // that includes it always self-fetches. A range-tab change to a ≤7-day - // window (raw rows) drops batched coverage and self-fetches. + // v1.18.6 — honour the dashboard's batched slice ONLY for windows over + // 7 days and ONLY when EVERY non-sleep type the chart renders is present + // in the slice. SLEEP_DURATION is never batched (per-night `/series`), + // so a chart that includes it always self-fetches. The 7-day tab + // self-fetches its own window. // // v1.19.0 — also require that the requested window FITS within the // batch's actual coverage. The dashboard batches only a ~30-day slice; @@ -916,41 +915,7 @@ export function HealthChart({ await fetchSleepNights(); return; } - const typeParams = new URLSearchParams(); - typeParams.set("type", type); - typeParams.set("sortBy", "measuredAt"); - typeParams.set("sortDir", "asc"); - typeParams.set("from", fetchWindow.from); - typeParams.set("to", fetchWindow.to); - typeParams.set("limit", "5000"); - // v1.4.29 C3 — windows over 7 days ask the server to bucket - // daily. Caps the chart's per-type payload at ~365 rows - // instead of ~5 000 for high-density types (pulse), and drops - // Recharts paint cost ~50× on continuous-monitoring accounts. - // Short windows keep raw fetching so hour-by-hour detail - // stays visible. - // - // v1.4.36 W1 — also route the daily aggregate through the - // persistent rollup buckets via `source=rollup`. The route - // reads from `measurement_rollups` instead of running a live - // `date_trunc` scan over the raw measurements table; the - // three parallel daily fetches on the Insights trends row - // (BP_SYS / BP_DIA / WEIGHT) drop from ~3 s each to a small - // indexed read against the ~5 k-row rollup table. The server - // falls back to live SQL when the rollup is empty for the - // requested window so brand-new accounts still see a chart - // on their first render. - // - // v1.19.2 — the `aggregate=daily` ask still holds, but for windows - // wider than the DAY bucket cap the server steps the rollup tier - // up to WEEK / MONTH and returns whole-history coverage at that - // tier. Each returned row is one coarse bucket; the daily fold + - // `bucketTimeSeries` below downsample it to the visible range, so - // the request contract is unchanged. - if (fetchWindow.windowDays > 7) { - typeParams.set("aggregate", "daily"); - typeParams.set("source", "rollup"); - } + const typeParams = chartSeriesParams(type, fetchWindow); // v1.18.6 — when the dashboard handed this chart a batched slice // for the type, fold those rows instead of a per-type fetch. The @@ -1011,8 +976,11 @@ export function HealthChart({ // same scale to the spread so the band tracks the line. The // BMI view drops the range tuple in its `select` callback (it // has never rendered the band), so the raw cache always - // carries the spread. + // carries the spread. The 7-day range never drew the band (it + // used to read raw rows, which carry no spread) and still does + // not, so its picture is unchanged now that it reads days too. if ( + fetchWindow.windowDays > 7 && typeof measurement.minValue === "number" && typeof measurement.maxValue === "number" ) { @@ -1044,10 +1012,9 @@ export function HealthChart({ for (const [type, stats] of Object.entries(bucket.values)) { // v1.4.29.1 — cumulative HealthKit types (steps, active energy, // distance, flights, daylight) must reduce with sum, not the - // per-sample average. The server already does this when the - // chart fetches with `aggregate=daily` (windows over 7 days); - // the 7-day path still pulls raw rows and aggregates here, - // so the same distinction must hold client-side. + // per-sample average. The server already returns one row per + // day; a day that arrives as several rows (the coarser tiers + // of the "All" range) must still add up rather than average. const isCumulative = CUMULATIVE_HK_TYPES.has( type as MeasurementType, ); diff --git a/tests/integration/chart-range-dense.test.ts b/tests/integration/chart-range-dense.test.ts new file mode 100644 index 000000000..5e36f45cd --- /dev/null +++ b/tests/integration/chart-range-dense.test.ts @@ -0,0 +1,106 @@ +/** + * The chart's own request, against the real route, on a dense stream. + * + * The 7-day range used to ask for raw rows oldest-first with a 5 000-row + * limit. Per-minute heart rate puts 10 080 rows in seven days, so the answer + * stopped about three and a half days in and the chart silently lost the + * most recent days. Every range now asks for one row per local day, so the + * whole window arrives whatever the sampling rate. + */ +import { NextRequest } from "next/server"; +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { cookieJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import { chartSeriesParams } from "@/components/charts/chart-series-request"; +import { userDayKey } from "@/lib/tz/format"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const USER = "chart-range-dense-user"; +const TZ = "Europe/Berlin"; +const TO = new Date("2026-09-20T18:00:00.000Z"); +const MINUTES = 7 * 24 * 60; + +beforeAll(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + cookieJar.clear(); + await prisma.user.create({ + data: { id: USER, username: USER, timezone: TZ }, + }); + invalidateUserTimezone(USER); + const session = await prisma.session.create({ + data: { userId: USER, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.set("healthlog_session", session.id); + await prisma.$executeRawUnsafe( + `INSERT INTO measurements ( + id, user_id, type, value, unit, source, measured_at, + created_at, updated_at, sync_version) + SELECT 'cr' || lpad(g::text, 6, '0'), $1, 'PULSE'::measurement_type, + 60 + (g % 30), 'bpm', 'GOOGLE_HEALTH'::measurement_source, + ($2::timestamptz AT TIME ZONE 'UTC') - (g * interval '1 minute'), + now(), now(), 1 + FROM generate_series(0, $3::int - 1) g`, + USER, + TO.toISOString(), + MINUTES, + ); +}, 120_000); + +describe("7-day chart range on per-minute pulse", () => { + it("returns every day of the window, the newest included", async () => { + const from = new Date(TO.getTime() - 7 * 86_400_000); + const params = chartSeriesParams("PULSE", { + from: from.toISOString(), + to: TO.toISOString(), + }); + const { GET } = await import("@/app/api/measurements/route"); + const res = await GET( + new NextRequest(`http://localhost/api/measurements?${params}`), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { + data: { measurements: Array<{ measuredAt: string }> }; + }; + const days = new Set( + json.data.measurements.map((m) => userDayKey(new Date(m.measuredAt), TZ)), + ); + // 20 Sep 20:00 Berlin back to 13 Sep 20:00: eight calendar days. + expect([...days].sort()).toEqual([ + "2026-09-13", + "2026-09-14", + "2026-09-15", + "2026-09-16", + "2026-09-17", + "2026-09-18", + "2026-09-19", + "2026-09-20", + ]); + }); +}); From c41df33dd361426716a8da184cc44ce19729c2a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 24 Sep 2026 08:06:47 +0200 Subject: [PATCH 5/7] fix(cycle): read the phase crosstab as per-source day aggregates The coach's cycle snapshot read a year of raw readings for the phase crosstab, including steps, heart-rate variability and CGM glucose. On a CGM account that is over 100 000 objects per snapshot. The crosstab only uses each day's sum and count after picking one source and device per day, and the pick depends only on which sources and devices are present. readSourceDayAggregates returns one row per type, day, source and device, and the day map now carries each row's reading count, so the day values are the ones the raw readings give. The 30-day clinical glucose reads stay raw: time in range, SD, J-index and the risk indices are functions of individual readings, and a sensor's rate bounds the window below 45 000 rows. Refs #1023 --- .../dense-measurement-read-guard.test.ts | 6 +- .../ai/coach/__tests__/cycle-snapshot.test.ts | 8 ++ src/lib/ai/coach/cycle-snapshot.ts | 31 +++-- src/lib/insights/mood-aggregates.ts | 6 + src/lib/insights/mood-crosstab.ts | 3 +- .../__tests__/fake-day-aggregates.ts | 59 +++++++++ src/lib/measurements/day-aggregates.ts | 85 ++++++++++++- .../cycle-crosstab-day-aggregates.test.ts | 114 ++++++++++++++++++ 8 files changed, 289 insertions(+), 23 deletions(-) create mode 100644 tests/integration/cycle-crosstab-day-aggregates.test.ts diff --git a/src/__tests__/dense-measurement-read-guard.test.ts b/src/__tests__/dense-measurement-read-guard.test.ts index d669c99d4..8414051a4 100644 --- a/src/__tests__/dense-measurement-read-guard.test.ts +++ b/src/__tests__/dense-measurement-read-guard.test.ts @@ -74,11 +74,9 @@ const ALLOWED: Record = { "lib/jobs/step-consolidation-repair.ts::STEP_TYPE": "Repair job over one user's step rows for a bounded day range; runs once per repair, not per status refresh.", 'lib/ai/coach/snapshot.ts::"BLOOD_GLUCOSE" as never': - "Clinical CGM metrics (time in range, GMI, CV) need individual readings; 30 days at a sensor's fixed rate (at most one a minute) stays below 45 000 rows.", + "Clinical CGM metrics (time in range, SD / CV, J-index, LBGI / HBGI, reading span) are functions of the individual readings, which a per-day aggregate cannot reproduce. 30 days at a sensor's fixed rate (at most one a minute) stays below 45 000 rows.", 'lib/dashboard/snapshot.ts::"BLOOD_GLUCOSE"': - "Same 30-day clinical glucose panel as the coach snapshot, grouped by meal context.", - "lib/ai/coach/cycle-snapshot.ts::{ in: PHASE_CROSSTAB_METRIC_TYPES": - "Cycle-phase crosstab over 365 days. Its dense members (steps, CGM glucose) are the known remaining case; steps are drained to one row per day nightly, CGM is not. Listed so it cannot grow unseen.", + "Same 30-day clinical glucose panel as the coach snapshot (per-reading metrics), grouped by meal context.", "lib/analytics/score/reader.ts::{ in: types": "Health-score inputs: steps (drained to one row per day nightly), sleep, waist, weight, blood pressure, fasting glucose only.", }; diff --git a/src/lib/ai/coach/__tests__/cycle-snapshot.test.ts b/src/lib/ai/coach/__tests__/cycle-snapshot.test.ts index 7ac766bf2..25869b76d 100644 --- a/src/lib/ai/coach/__tests__/cycle-snapshot.test.ts +++ b/src/lib/ai/coach/__tests__/cycle-snapshot.test.ts @@ -10,6 +10,14 @@ vi.mock("@/lib/db", () => ({ }, })); +// The crosstab reads per-source day aggregates; the fake groups the mocked +// `measurement.findMany` rows with the same rules. +vi.mock("@/lib/measurements/day-aggregates", async () => ({ + readSourceDayAggregates: ( + await import("@/lib/measurements/__tests__/fake-day-aggregates") + ).fakeReadSourceDayAggregates, +})); + import { buildCycleSnapshotBlock } from "../cycle-snapshot"; import { prisma } from "@/lib/db"; diff --git a/src/lib/ai/coach/cycle-snapshot.ts b/src/lib/ai/coach/cycle-snapshot.ts index c4f0b4cad..297496213 100644 --- a/src/lib/ai/coach/cycle-snapshot.ts +++ b/src/lib/ai/coach/cycle-snapshot.ts @@ -40,6 +40,7 @@ import { } from "@/lib/cycle/phase-crosstab"; import type { CrossMetricMeasurement } from "@/lib/insights/mood-aggregates"; import { addDays } from "@/lib/cycle/day-math"; +import { readSourceDayAggregates } from "@/lib/measurements/day-aggregates"; import { DEFAULT_TIMEZONE, moodDateKey } from "@/lib/mood/date-key"; /** Trailing window the phase contrast walks (days). Mirrors the read route. */ @@ -195,21 +196,16 @@ export async function buildCycleSnapshotBlock( orderBy: { measuredAt: "asc" }, select: { measuredAt: true, value: true }, }), - prisma.measurement.findMany({ - where: { - userId, - deletedAt: null, - type: { in: PHASE_CROSSTAB_METRIC_TYPES }, - measuredAt: { gte: new Date(Date.parse(`${from}T00:00:00Z`)) }, - }, - orderBy: { measuredAt: "asc" }, - select: { - type: true, - value: true, - measuredAt: true, - source: true, - deviceType: true, - }, + // A year of steps, heart-rate variability and CGM glucose is far too + // many readings to hold as objects (#1023). The crosstab only needs + // each day's sum and count per source and device, which is what the + // per-source picker decides on, so the fold runs in SQL. Days are the + // Berlin days `metricDayMap` keys on. + readSourceDayAggregates({ + userId, + types: PHASE_CROSSTAB_METRIC_TYPES, + since: new Date(Date.parse(`${from}T00:00:00Z`)), + timeZone: "Europe/Berlin", }), prisma.user.findUnique({ where: { id: userId }, @@ -285,8 +281,9 @@ export async function buildCycleSnapshotBlock( ); const measurements: CrossMetricMeasurement[] = measurementRows.map((m) => ({ type: m.type, - value: m.value, - measuredAt: m.measuredAt, + value: m.sum, + count: m.n, + measuredAt: m.firstAt, source: m.source, deviceType: m.deviceType, })); diff --git a/src/lib/insights/mood-aggregates.ts b/src/lib/insights/mood-aggregates.ts index ed2c0a633..a6e634cd9 100644 --- a/src/lib/insights/mood-aggregates.ts +++ b/src/lib/insights/mood-aggregates.ts @@ -254,6 +254,12 @@ export interface CrossMetricMeasurement { measuredAt: Date; source?: MeasurementSource | null; deviceType?: string | null; + /** + * Readings this row stands for. Absent on a raw reading (one). A row + * pre-aggregated per day, source and device carries its readings' sum in + * `value` and their number here, so the day's mean stays exact. + */ + count?: number; } /** diff --git a/src/lib/insights/mood-crosstab.ts b/src/lib/insights/mood-crosstab.ts index d349febb6..181754cad 100644 --- a/src/lib/insights/mood-crosstab.ts +++ b/src/lib/insights/mood-crosstab.ts @@ -231,6 +231,7 @@ export function metricDayMap( deviceType: m.deviceType ?? null, type: type as MeasurementType, value: m.value, + count: m.count, })), metricKey, userPriorityJson, @@ -243,7 +244,7 @@ export function metricDayMap( const key = berlinDayKey(m.measuredAt); const cur = byDay.get(key) ?? { sum: 0, count: 0 }; cur.sum += m.value; - cur.count += 1; + cur.count += m.count ?? 1; byDay.set(key, cur); } const out = new Map(); diff --git a/src/lib/measurements/__tests__/fake-day-aggregates.ts b/src/lib/measurements/__tests__/fake-day-aggregates.ts index 311e8350c..096727fb9 100644 --- a/src/lib/measurements/__tests__/fake-day-aggregates.ts +++ b/src/lib/measurements/__tests__/fake-day-aggregates.ts @@ -9,9 +9,14 @@ * SQL agree on real Postgres. */ import { prisma } from "@/lib/db"; +import type { + MeasurementSource, + MeasurementType, +} from "@/generated/prisma/client"; import type { DayAggregateRow, ReadDayAggregatesOptions, + SourceDayAggregateRow, } from "@/lib/measurements/day-aggregates"; import { userDayKey } from "@/lib/tz/format"; @@ -71,3 +76,57 @@ export async function fakeReadDayAggregates( })) ?? []) as Array<{ measuredAt: Date; value: number }>; return foldDayAggregates(rows, opts); } + +/** + * In-memory stand-in for `readSourceDayAggregates`: reads the mocked + * `measurement.findMany` rows and groups them per type, day, source and + * device with the SQL's rules. + */ +export async function fakeReadSourceDayAggregates(opts: { + userId: string; + types: readonly MeasurementType[]; + since: Date; + timeZone: string; +}): Promise { + const rows = ((await prisma.measurement.findMany({ + where: { + userId: opts.userId, + deletedAt: null, + type: { in: [...opts.types] }, + measuredAt: { gte: opts.since }, + }, + orderBy: { measuredAt: "asc" }, + })) ?? []) as Array<{ + type: MeasurementType; + value: number; + measuredAt: Date; + source?: MeasurementSource | null; + deviceType?: string | null; + }>; + const groups = new Map(); + for (const r of rows) { + if (r.measuredAt.getTime() < opts.since.getTime()) continue; + if (!opts.types.includes(r.type)) continue; + const day = userDayKey(r.measuredAt, opts.timeZone); + const source = (r.source ?? "MANUAL") as MeasurementSource; + const deviceType = r.deviceType ?? null; + const key = `${r.type}|${day}|${source}|${deviceType}`; + const g = groups.get(key); + if (g) { + g.n += 1; + g.sum += r.value; + if (r.measuredAt < g.firstAt) g.firstAt = r.measuredAt; + } else { + groups.set(key, { + type: r.type, + day, + source, + deviceType, + n: 1, + sum: r.value, + firstAt: r.measuredAt, + }); + } + } + return [...groups.values()]; +} diff --git a/src/lib/measurements/day-aggregates.ts b/src/lib/measurements/day-aggregates.ts index 083b59b78..48d2966b1 100644 --- a/src/lib/measurements/day-aggregates.ts +++ b/src/lib/measurements/day-aggregates.ts @@ -19,7 +19,11 @@ * before, so a caller that treated "the last 21 days" and "the 70 days before * that" differently keeps exactly the same per-row partition. */ -import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; +import type { + MeasurementSource, + MeasurementType, + PrismaClient, +} from "@/generated/prisma/client"; import { Prisma } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import { isValidTimezone } from "@/lib/tz/format"; @@ -120,3 +124,82 @@ export async function readDayAggregates( max: Number(r.max), })); } + +export interface SourceDayAggregateRow { + type: MeasurementType; + /** Calendar day in the requested zone, `YYYY-MM-DD`. */ + day: string; + source: MeasurementSource; + deviceType: string | null; + n: number; + sum: number; + /** Earliest reading of the group; lies inside `day` in the zone. */ + firstAt: Date; +} + +/** + * Per-day aggregates of several types, kept apart per source and device. + * + * For readers that pick one source per day (`pickCanonicalSourceRows`) and + * then add up or average that day. The picker decides from which sources and + * device types are present on a day, never from individual values, so it + * makes the same pick over these groups as over the raw rows, and the day's + * sum and count follow exactly. Size: one row per type, day, source and + * device, however densely a type is sampled. + */ +export async function readSourceDayAggregates(opts: { + userId: string; + types: readonly MeasurementType[]; + since: Date; + timeZone: string; + db?: Pick; +}): Promise { + if (opts.types.length === 0) return []; + const timeZone = isValidTimezone(opts.timeZone) ? opts.timeZone : "UTC"; + const types = Prisma.join( + opts.types.map((t) => Prisma.sql`${t}::measurement_type`), + ); + const rows = await (opts.db ?? prisma).$queryRaw< + Array<{ + type: string; + day: string; + source: string; + device_type: string | null; + n: number; + sum: number; + first_at: Date; + }> + >` + WITH src AS ( + SELECT + m."type"::text AS type, + to_char((m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}, 'YYYY-MM-DD') AS day, + m."source"::text AS source, + m."device_type" AS device_type, + m."value" AS value, + m."measured_at" AS measured_at + FROM measurements m + WHERE m."user_id" = ${opts.userId} + AND m."type" IN (${types}) + AND m."deleted_at" IS NULL + AND m."measured_at" >= ${opts.since} + ) + SELECT + type, day, source, device_type, + COUNT(*)::int AS n, + SUM(value)::double precision AS sum, + MIN(measured_at) AS first_at + FROM src + GROUP BY type, day, source, device_type + ORDER BY day ASC, type ASC, source ASC + `; + return rows.map((r) => ({ + type: r.type as MeasurementType, + day: r.day, + source: r.source as MeasurementSource, + deviceType: r.device_type, + n: Number(r.n), + sum: Number(r.sum), + firstAt: r.first_at, + })); +} diff --git a/tests/integration/cycle-crosstab-day-aggregates.test.ts b/tests/integration/cycle-crosstab-day-aggregates.test.ts new file mode 100644 index 000000000..cdbc2df2e --- /dev/null +++ b/tests/integration/cycle-crosstab-day-aggregates.test.ts @@ -0,0 +1,114 @@ +/** + * The cycle-phase crosstab reads per-day, per-source aggregates instead of a + * year of raw readings (#1023). This pins that the day values it tests are + * the ones the raw readings give: for every crosstab metric, `metricDayMap` + * over the aggregates equals `metricDayMap` over the raw rows, on a fixture + * with two sources on the same days, two device types within one source, + * dense CGM glucose, summed steps and readings next to Berlin midnight. + */ +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { readSourceDayAggregates } from "@/lib/measurements/day-aggregates"; +import { metricDayMap } from "@/lib/insights/mood-crosstab"; +import { PHASE_CROSSTAB_METRIC_TYPES } from "@/lib/cycle/phase-crosstab"; +import type { CrossMetricMeasurement } from "@/lib/insights/mood-aggregates"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const USER = "cycle-crosstab-agg"; +const SINCE = new Date("2026-06-01T00:00:00Z"); + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ data: { id: USER, username: USER } }); + // CGM glucose every 5 minutes for 60 days from two sources. + await prisma.$executeRawUnsafe( + `INSERT INTO measurements (id, user_id, type, value, unit, source, measured_at, created_at, updated_at, sync_version) + SELECT 'cg' || s || g, $1, 'BLOOD_GLUCOSE'::measurement_type, 80 + (g % 60), 'mg/dL', + (CASE WHEN s = 0 THEN 'APPLE_HEALTH' ELSE 'NIGHTSCOUT' END)::measurement_source, + timestamp '2026-06-02 00:00:00' + (g * interval '5 minutes') + (s * interval '1 second'), + now(), now(), 1 + FROM generate_series(0, 60 * 288 - 1) g, generate_series(0, 1) s`, + USER, + ); + // Steps every 15 minutes from a watch and a phone, plus Google totals on + // some days; resting heart rate once a day near Berlin midnight. + await prisma.$executeRawUnsafe( + `INSERT INTO measurements (id, user_id, type, value, unit, source, device_type, measured_at, created_at, updated_at, sync_version) + SELECT 'st' || d || g, $1, 'ACTIVITY_STEPS'::measurement_type, 10 + (g % 7), 'count', + 'APPLE_HEALTH'::measurement_source, CASE WHEN d = 0 THEN 'watch' ELSE 'phone' END, + timestamp '2026-06-02 00:00:00' + (g * interval '15 minutes') + (d * interval '2 seconds'), + now(), now(), 1 + FROM generate_series(0, 60 * 96 - 1) g, generate_series(0, 1) d + WHERE d = 0 OR g % 3 = 0`, + USER, + ); + await prisma.$executeRawUnsafe( + `INSERT INTO measurements (id, user_id, type, value, unit, source, measured_at, created_at, updated_at, sync_version) + SELECT 'gs' || g, $1, 'ACTIVITY_STEPS'::measurement_type, 5000 + g, 'count', + 'GOOGLE_HEALTH'::measurement_source, + timestamp '2026-06-02 12:00:00' + (g * interval '1 day'), now(), now(), 1 + FROM generate_series(0, 59, 4) g`, + USER, + ); + await prisma.$executeRawUnsafe( + `INSERT INTO measurements (id, user_id, type, value, unit, source, measured_at, created_at, updated_at, sync_version) + SELECT 'rh' || g, $1, 'RESTING_HEART_RATE'::measurement_type, 55 + (g % 9), 'bpm', + 'WITHINGS'::measurement_source, + timestamp '2026-06-02 22:30:00' + (g * interval '1 day'), now(), now(), 1 + FROM generate_series(0, 59) g`, + USER, + ); +}, 120_000); + +describe("cycle crosstab day values (#1023)", () => { + it("per-source day aggregates give the raw rows' day values for every metric", async () => { + const raw = await prisma.measurement.findMany({ + where: { + userId: USER, + deletedAt: null, + type: { in: PHASE_CROSSTAB_METRIC_TYPES }, + measuredAt: { gte: SINCE }, + }, + select: { + type: true, + value: true, + measuredAt: true, + source: true, + deviceType: true, + }, + }); + const agg = await readSourceDayAggregates({ + userId: USER, + types: PHASE_CROSSTAB_METRIC_TYPES, + since: SINCE, + timeZone: "Europe/Berlin", + }); + const aggRows: CrossMetricMeasurement[] = agg.map((a) => ({ + type: a.type, + value: a.sum, + count: a.n, + measuredAt: a.firstAt, + source: a.source, + deviceType: a.deviceType, + })); + + expect(raw.length).toBeGreaterThan(40_000); + expect(aggRows.length).toBeLessThan(500); + + for (const type of PHASE_CROSSTAB_METRIC_TYPES) { + const expected = metricDayMap(raw, type, null); + const actual = metricDayMap(aggRows, type, null); + expect([...actual.keys()].sort(), type).toEqual( + [...expected.keys()].sort(), + ); + for (const [day, value] of expected) { + expect(actual.get(day), `${type} ${day}`).toBeCloseTo(value, 9); + } + } + }, 60_000); +}); From 3170a622d15471c375b026a90532c36a2803ae25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 24 Sep 2026 08:12:23 +0200 Subject: [PATCH 6/7] fix(medications): report the dose-history window doses were read over The dose-history response reported the slot window: `from` clamped to the medication's creation, `to` the request's end. Recorded doses are now read from the requested start and, for a dose recorded ahead of its slot, up to that slot, so the response reported a narrower window than the rows it carried. `from` and `to` now report the window recorded doses were read over. Both fields keep their shape; the OpenAPI descriptions say what they mean. Refs #1028 --- docs/api/openapi.yaml | 17 +++++++++++------ .../[id]/dose-history/__tests__/route.test.ts | 6 ++++++ .../api/medications/[id]/dose-history/route.ts | 8 ++++++-- src/lib/openapi/routes/medications/schemas.ts | 18 ++++++++++++++---- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 22eb9ba0b..c6ff8a5ec 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -5961,21 +5961,23 @@ paths: - in: query name: from schema: - description: Window start (inclusive). Defaults to 90 days before `to`; clamped to the medication's `createdAt` and a - 366-day span floor. + description: Window start (inclusive). Defaults to 90 days before `to`; clamped to a 366-day span floor. Recorded doses + are returned from this instant; expected slots are not minted before the medication's `createdAt`. type: string format: date-time pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ - description: Window start (inclusive). Defaults to 90 days before `to`; clamped to the medication's `createdAt` and a - 366-day span floor. + description: Window start (inclusive). Defaults to 90 days before `to`; clamped to a 366-day span floor. Recorded doses + are returned from this instant; expected slots are not minted before the medication's `createdAt`. - in: query name: to schema: - description: Window end (inclusive). Defaults to now. Must be ≥ `from`. + description: Window end (inclusive). Defaults to now. Must be ≥ `from`. A dose recorded ahead of its slot (the slot's + time within a day after `to`) is still returned. type: string format: date-time pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ - description: Window end (inclusive). Defaults to now. Must be ≥ `from`. + description: Window end (inclusive). Defaults to now. Must be ≥ `from`. A dose recorded ahead of its slot (the slot's + time within a day after `to`) is still returned. responses: "200": description: Dose-history ledger. @@ -43671,10 +43673,13 @@ components: type: string format: date-time pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + description: "Start of the window recorded doses were read over: the requested `from` after the span clamp." to: type: string format: date-time pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ + description: "End of the window recorded doses were read over: the requested `to`, or the slot time of a dose recorded + ahead of its slot when that lies later (at most one day after `to`)." family: type: string enum: diff --git a/src/app/api/medications/[id]/dose-history/__tests__/route.test.ts b/src/app/api/medications/[id]/dose-history/__tests__/route.test.ts index 23e01838e..1cccc72ad 100644 --- a/src/app/api/medications/[id]/dose-history/__tests__/route.test.ts +++ b/src/app/api/medications/[id]/dose-history/__tests__/route.test.ts @@ -344,6 +344,9 @@ describe("GET /api/medications/[id]/dose-history — intakes recorded before cre json.data.rows as Array<{ intake: { id: string | null } | null }> ).map((r) => r.intake?.id); expect(ids).toContain("evt-earlier-today"); + // The response reports the window the recorded doses were read + // over: the requested start, not the medication's creation. + expect(json.data.from).toBe(from); } finally { vi.useRealTimers(); } @@ -410,6 +413,9 @@ describe("GET /api/medications/[id]/dose-history — early take (#1028 class)", status: string; intake: { id: string | null } | null; }>; + // `to` reports how far recorded doses were read: up to the early + // dose's slot, past the request's own now. + expect(json.data.to).toBe(slot.toISOString()); const early = rows.find((r) => r.intake?.id === "evt-early"); expect(early?.status).toBe("taken_on_time"); // The read horizon stretches only to the recorded anchor: no later diff --git a/src/app/api/medications/[id]/dose-history/route.ts b/src/app/api/medications/[id]/dose-history/route.ts index 11256b61f..8b1dae088 100644 --- a/src/app/api/medications/[id]/dose-history/route.ts +++ b/src/app/api/medications/[id]/dose-history/route.ts @@ -313,9 +313,13 @@ export const GET = apiHandler( meta: { row_count: serialized.length, family }, }); + // The window the recorded doses were read over (#1028): from the + // requested start (slots additionally start at the medication's + // creation) to the read horizon, which reaches past `to` only for a dose + // recorded ahead of its slot. return apiSuccess({ - from: from.toISOString(), - to: to.toISOString(), + from: intakeFrom.toISOString(), + to: horizon.toISOString(), family, hasExpectedSlots: bands.length > 0, rows: serialized, diff --git a/src/lib/openapi/routes/medications/schemas.ts b/src/lib/openapi/routes/medications/schemas.ts index 02b1a3908..47a2bd6e3 100644 --- a/src/lib/openapi/routes/medications/schemas.ts +++ b/src/lib/openapi/routes/medications/schemas.ts @@ -911,12 +911,14 @@ export const doseHistoryQuery = z.object({ .datetime({ offset: true }) .optional() .describe( - "Window start (inclusive). Defaults to 90 days before `to`; clamped to the medication's `createdAt` and a 366-day span floor.", + "Window start (inclusive). Defaults to 90 days before `to`; clamped to a 366-day span floor. Recorded doses are returned from this instant; expected slots are not minted before the medication's `createdAt`.", ), to: z.iso .datetime({ offset: true }) .optional() - .describe("Window end (inclusive). Defaults to now. Must be ≥ `from`."), + .describe( + "Window end (inclusive). Defaults to now. Must be ≥ `from`. A dose recorded ahead of its slot (the slot's time within a day after `to`) is still returned.", + ), }); const doseHistoryRow = z @@ -982,8 +984,16 @@ const doseHistoryRow = z export const doseHistoryResponse = z .object({ - from: z.iso.datetime({ offset: true }), - to: z.iso.datetime({ offset: true }), + from: z.iso + .datetime({ offset: true }) + .describe( + "Start of the window recorded doses were read over: the requested `from` after the span clamp.", + ), + to: z.iso + .datetime({ offset: true }) + .describe( + "End of the window recorded doses were read over: the requested `to`, or the slot time of a dose recorded ahead of its slot when that lies later (at most one day after `to`).", + ), family: z .enum(["daily", "weekly", "one_shot", "none"]) .describe("Cadence family the window's slots were minted under."), From 16ef75e729cb915c0becf0c0e8641322bddbbfa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 24 Sep 2026 08:15:16 +0200 Subject: [PATCH 7/7] fix(charts): cut days in the account's zone when a mount passes none Several chart mounts (the medications page, the coach chat, the recovery section) pass no zone, and the chart then fell back to a fixed Europe/Berlin, so those charts cut and labelled days in Berlin for every user whatever their profile said. The chart now falls back to the signed-in account's zone, and only before that has loaded to the shared DEFAULT_TIMEZONE the server's own resolver ends on. /api/auth/me reports the zone the server resolves: the stored one, or the instance default when the stored one is unusable, so the client cuts days where the server does. Refs #1026 --- src/app/api/auth/me/route.ts | 11 +++- .../__tests__/health-chart-timezone.test.tsx | 53 +++++++++++++++++++ src/components/charts/health-chart.tsx | 15 ++++-- tests/integration/timezone-per-user.test.ts | 31 +++++++++++ 4 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 src/components/charts/__tests__/health-chart-timezone.test.tsx diff --git a/src/app/api/auth/me/route.ts b/src/app/api/auth/me/route.ts index 52dcfc7cf..1bd2b7f13 100644 --- a/src/app/api/auth/me/route.ts +++ b/src/app/api/auth/me/route.ts @@ -49,6 +49,10 @@ import { syncMfaEnrollCookie } from "@/lib/auth/mfa-enrollment"; import { buildAvatarUrl } from "@/lib/avatar"; import { decrypt } from "@/lib/crypto"; import { prisma } from "@/lib/db"; +import { + isValidTimezone, + resolveServerDefaultTimezone, +} from "@/lib/tz/resolver"; import { isCycleEnabled } from "@/lib/cycle/gate"; import { resolveModuleMap, @@ -238,7 +242,12 @@ export const GET = apiHandler(async () => { heightCm: user.heightCm, dateOfBirth: user.dateOfBirth, gender: user.gender, - timezone: user.timezone, + // The zone the server cuts this account's days in: the stored zone, or + // the instance default when the stored one is unusable. Clients bucket + // by this value, so it must be the resolved one, not the raw column. + timezone: isValidTimezone(user.timezone) + ? user.timezone + : await resolveServerDefaultTimezone(), onboardingCompletedAt: user.onboardingCompletedAt, onboardingTourCompleted: user.onboardingTourCompleted, // v1.39 (C1) — the needs-based setup flow, published additively: diff --git a/src/components/charts/__tests__/health-chart-timezone.test.tsx b/src/components/charts/__tests__/health-chart-timezone.test.tsx new file mode 100644 index 000000000..4dd513ae7 --- /dev/null +++ b/src/components/charts/__tests__/health-chart-timezone.test.tsx @@ -0,0 +1,53 @@ +import { describe, it, expect, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; + +/** + * A chart mount that passes no `userTimezone` must still cut days in the + * signed-in user's zone. Several mounts (the medications page, the coach + * chat, the recovery section) pass none, and the component used to fall + * back to a fixed Europe/Berlin, so a user in Asia/Kolkata saw those charts + * bucketed and labelled in Berlin days while every other chart used their + * own. The zone now comes from the account, which the server resolves. + */ + +let lastKey: unknown[] | undefined; + +vi.mock("@tanstack/react-query", () => ({ + keepPreviousData: (previous: unknown) => previous, + useQuery: ({ queryKey }: { queryKey: unknown[] }) => { + lastKey = queryKey; + return { data: [], isLoading: false }; + }, + useQueryClient: () => ({ + cancelQueries: () => Promise.resolve(), + getQueryData: () => undefined, + setQueryData: () => undefined, + invalidateQueries: () => Promise.resolve(), + }), + useMutation: () => ({ mutate: () => undefined, isPending: false }), +})); + +vi.mock("@/hooks/use-auth", () => ({ + useAuth: () => ({ + isAuthenticated: true, + user: { id: "u1", timezone: "Asia/Kolkata" }, + isLoading: false, + }), +})); + +describe(" timezone", () => { + it("uses the account's zone when the mount passes none", async () => { + const { I18nProvider } = await import("@/lib/i18n/context"); + const { HealthChart } = await import("../health-chart"); + + renderToStaticMarkup( + + + , + ); + + expect(lastKey).toBeDefined(); + expect(lastKey).toContain("Asia/Kolkata"); + expect(lastKey).not.toContain("Europe/Berlin"); + }); +}); diff --git a/src/components/charts/health-chart.tsx b/src/components/charts/health-chart.tsx index 23f75dc76..d2d766ce8 100644 --- a/src/components/charts/health-chart.tsx +++ b/src/components/charts/health-chart.tsx @@ -1,6 +1,7 @@ "use client"; import { chartSeriesParams } from "@/components/charts/chart-series-request"; +import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; import { keepPreviousData, useQuery } from "@tanstack/react-query"; import { useAuth } from "@/hooks/use-auth"; import { useRecordCapabilities } from "@/hooks/use-record-capabilities"; @@ -214,10 +215,11 @@ interface HealthChartProps { /** * v1.4.25 W7b — per-user display timezone. When passed (mount sites * thread `useAuth().user?.timezone`), x-axis tick labels and the - * per-day bucket keys both render in the user's zone instead of the - * legacy Europe/Berlin pin. Defaults to "Europe/Berlin" so older - * callers that haven't yet adopted the prop keep their previous - * behaviour bit-for-bit. + * per-day bucket keys both render in the user's zone. When a mount + * passes none, the chart uses the signed-in account's zone (which + * `/api/auth/me` reports already resolved against the server default), + * and only before that has loaded the shared `DEFAULT_TIMEZONE` the + * server itself falls back to. */ userTimezone?: string; /** @@ -614,7 +616,7 @@ export function HealthChart({ chartKey, annotations, verticalMarkers, - userTimezone = "Europe/Berlin", + userTimezone: userTimezoneProp, valueScale = 1, valueOffset = 0, onVisibleStats, @@ -625,6 +627,9 @@ export function HealthChart({ showDataTable = false, }: HealthChartProps) { const { isAuthenticated, user } = useAuth(); + // A mount without the prop used to pin Europe/Berlin, so those charts + // cut days in Berlin for every user whatever their profile said. + const userTimezone = userTimezoneProp || user?.timezone || DEFAULT_TIMEZONE; const { t, locale } = useTranslations(); const fmt = useFormatters(); // Issue #490 — x-axis tick + tooltip date labels. Every timestamp this diff --git a/tests/integration/timezone-per-user.test.ts b/tests/integration/timezone-per-user.test.ts index 117fb1742..c064e7f9c 100644 --- a/tests/integration/timezone-per-user.test.ts +++ b/tests/integration/timezone-per-user.test.ts @@ -28,6 +28,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; process.env.ENCRYPTION_KEY ??= "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +import { invalidateServerDefaultTimezone } from "@/lib/tz/resolver"; import { cookieJar, headerJar } from "./mock-next-headers"; import { getPrismaClient, truncateAllTables } from "./setup"; @@ -166,6 +167,36 @@ describe("per-user timezone — Pacific/Auckland end-to-end", () => { expect(body.data?.timezone).toBe("Pacific/Auckland"); }); + // A stored zone the server cannot use falls back to the instance + // default for every server-side day cut. `/api/auth/me` reports that + // same resolved zone, so a client (the charts' day buckets among them) + // never cuts days in a different zone than the server did. + it("GET /api/auth/me reports the resolved zone when the stored one is unusable", async () => { + const user = await seedAucklandUser(); + const prisma = getPrismaClient(); + await prisma.user.update({ + where: { id: user.id }, + data: { timezone: "" }, + }); + await prisma.appSettings.upsert({ + where: { id: "singleton" }, + create: { id: "singleton", defaultUserTimezone: "Asia/Kolkata" }, + update: { defaultUserTimezone: "Asia/Kolkata" }, + }); + invalidateServerDefaultTimezone(); + try { + const { GET } = await import("@/app/api/auth/me/route"); + const res = await GET(); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { timezone?: string } | null; + }; + expect(body.data?.timezone).toBe("Asia/Kolkata"); + } finally { + invalidateServerDefaultTimezone(); + } + }); + it("PUT /api/auth/me/timezone rejects an invalid IANA zone with 422", async () => { await seedAucklandUser();