diff --git a/src/app/api/measurements/__tests__/group-by-day.test.ts b/src/app/api/measurements/__tests__/group-by-day.test.ts index a37bb293c..db8f542f9 100644 --- a/src/app/api/measurements/__tests__/group-by-day.test.ts +++ b/src/app/api/measurements/__tests__/group-by-day.test.ts @@ -2,22 +2,29 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { NextRequest } from "next/server"; /** - * v1.4.37 W7c — GET /api/measurements gains two new branches for - * cumulative HK types (steps, active energy, distance, flights, - * daylight): + * GET /api/measurements has two branches for cumulative HK types (steps, + * active energy, distance, flights, daylight): * * 1. `?type=ACTIVITY_STEPS&groupBy=day` → one synthesised row per - * user-TZ day with `value` = SUM and `sampleCount` = number of - * per-sample rows behind the bucket. + * user-TZ day, built by `dayGroupedListResponse`. What the rows hold + * (whole days, the source ladder, paging, the day cut in the user's + * zone) is pinned against Postgres in + * `tests/integration/measurements-day-grouped-paging.test.ts`; this + * suite pins only which branch a request reaches. * 2. `?type=ACTIVITY_STEPS&dayKey=YYYY-MM-DD` → raw per-sample rows * for that single calendar day in the user's IANA timezone. * - * Spot metrics (BP, weight, pulse, etc.) keep the legacy - * `findMany`-only behaviour even when these params are present — - * the schema accepts the params but the route gates each branch on + * Spot metrics (BP, weight, pulse, etc.) keep the plain list even when + * these params are present: the route gates each branch on * `CUMULATIVE_HK_TYPES`. */ +vi.mock("@/lib/measurements/list/day-grouped", () => ({ + dayGroupedListResponse: vi.fn(async () => + Response.json({ data: { measurements: [], meta: {} }, error: null }), + ), +})); + vi.mock("@/lib/db", () => ({ prisma: { measurement: { @@ -32,7 +39,7 @@ vi.mock("@/lib/db", () => ({ // blob via `loadUserSourcePriority` (prisma.user.findUnique). `null` // resolves to the default ladders. user: { findUnique: vi.fn() }, - // v1.4.43 W6 — validation-failed paths write an audit breadcrumb. + // v1.4.43 — validation-failed paths write an audit breadcrumb. auditLog: { create: vi.fn().mockResolvedValue({}) }, $queryRaw: vi.fn(), }, @@ -55,6 +62,7 @@ vi.mock("next/headers", () => ({ import { GET } from "../route"; import { auditLog } from "@/lib/auth/audit"; +import { dayGroupedListResponse } from "@/lib/measurements/list/day-grouped"; import { prisma } from "@/lib/db"; import { getSession } from "@/lib/auth/session"; @@ -74,71 +82,10 @@ function getRequest(query: string): NextRequest { }); } -interface Group { - day_key: string; - source: string; - device_type: string | null; - unit: string; - total: number; - n: number; -} - -/** One SQL fold row: a day's readings of one source, device and unit. */ -function group( - day_key: string, - total: number, - n: number, - over: Partial = {}, -): Group { - return { - day_key, - source: "APPLE_HEALTH", - device_type: null, - unit: "steps", - total, - n, - ...over, - }; -} - -/** - * The day-grouped branch answers from SQL alone: one query picks the page's - * days, each row carrying the distinct-day total from a window count; a - * second folds those days into source groups. A page past the end has no row - * to carry the total, so only then is it counted on its own. Answer each - * query by its text. - */ -function mockDayPage( - days: string[], - total = days.length, - groups: Group[] = [], -) { - vi.mocked(prisma.$queryRaw).mockImplementation((( - strings: TemplateStringsArray, - ) => { - const text = strings.join(""); - if (text.includes("COUNT(DISTINCT")) return Promise.resolve([{ total }]); - if (text.includes("COUNT(*) OVER ()")) { - return Promise.resolve(days.map((day_key) => ({ day_key, total }))); - } - if (text.includes("SUM(")) return Promise.resolve(groups); - return Promise.resolve([]); - }) as never); -} - -/** The `$queryRaw` calls whose text contains `needle`. */ -function rawCallsWith(needle: string) { - return vi - .mocked(prisma.$queryRaw) - .mock.calls.filter(([strings]) => - (strings as unknown as string[]).join("").includes(needle), - ); -} - beforeEach(() => { vi.resetAllMocks(); vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); - // v1.4.43 W6 — validation-failed paths fire a best-effort audit-row + // v1.4.43 — validation-failed paths fire a best-effort audit-row // write; the route swallows rejections so the test only needs a // resolved mock to keep the catch-block silent. vi.mocked(prisma.auditLog.create).mockResolvedValue({} as never); @@ -150,176 +97,36 @@ beforeEach(() => { vi.mocked(prisma.user.findUnique).mockResolvedValue(null as never); }); -describe("GET /api/measurements — groupBy=day (W7c collapsed list)", () => { - it("collapses per-sample step rows into one row per user-TZ day with sampleCount", async () => { - // Three Apple-Watch chunks on 2026-05-15 (Europe/Berlin) summing to 5400 - // and two on 2026-05-16 summing to 7000, as SQL folds them. - mockDayPage(["2026-05-16", "2026-05-15"], 2, [ - group("2026-05-15", 5400, 3), - group("2026-05-16", 7000, 2), - ]); - +describe("GET /api/measurements — groupBy=day", () => { + it("hands a cumulative type to the day-grouped list with the request's filters", async () => { const res = await GET( getRequest( - "type=ACTIVITY_STEPS&groupBy=day&from=2026-05-15T00:00:00Z&to=2026-05-17T00:00:00Z", + "type=ACTIVITY_STEPS&groupBy=day&limit=25&offset=50&sourceEq=APPLE_HEALTH&from=2026-05-01T00:00:00Z", ), ); expect(res.status).toBe(200); - const json = (await res.json()) as { - data: { - measurements: Array<{ - type: string; - value: number; - unit: string; - dayKey: string; - sampleCount: number; - measuredAt: string; - }>; - meta: { groupBy?: string; total: number }; - }; - }; - expect(json.data.meta.groupBy).toBe("day"); - expect(json.data.measurements.length).toBe(2); - // Sort is desc by default — newer day first. - const [day16, day15] = json.data.measurements; - expect(day16.dayKey).toBe("2026-05-16"); - expect(day16.value).toBe(7000); - expect(day16.sampleCount).toBe(2); - expect(day16.unit).toBe("steps"); - expect(day15.dayKey).toBe("2026-05-15"); - expect(day15.value).toBe(5400); - expect(day15.sampleCount).toBe(3); - // The collapsed row's measuredAt anchors to local noon so the - // row sorts cleanly between same-day spot samples (canonical - // daily-timestamp contract shared with the drain helper). - expect(day15.measuredAt).toContain("2026-05-15T10:00:00"); - }); - - // #1245 — the fold covers exactly the page's days: from the start of the - // oldest page day to the end of the newest, in the user's zone, on top of - // the request's own filters. Nothing outside the page is scanned, and no - // raw row is read into the process at all. - it("folds only the page's days, in SQL", async () => { - mockDayPage(["2026-05-16", "2026-05-15"], 40); - - await GET(getRequest("type=ACTIVITY_STEPS&groupBy=day")); - - const [fold] = rawCallsWith("SUM("); - const instants = (fold ?? []) - .slice(1) - .filter((v): v is Date => v instanceof Date) - .map((d) => d.toISOString()); - // Europe/Berlin is UTC+2 in May. - expect(instants).toEqual([ - "2026-05-14T22:00:00.000Z", - "2026-05-16T22:00:00.000Z", - ]); + expect(dayGroupedListResponse).toHaveBeenCalledWith( + expect.objectContaining({ id: "user-1" }), + expect.objectContaining({ + type: "ACTIVITY_STEPS", + limit: 25, + offset: 50, + sortDir: "desc", + sourceEq: "APPLE_HEALTH", + from: new Date("2026-05-01T00:00:00Z"), + }), + ); expect(prisma.measurement.findMany).not.toHaveBeenCalled(); }); - it("folds nothing when the page holds no day", async () => { - mockDayPage([], 3); + // A day-grouped row is a whole day's sum; a value range filters single + // readings and would leave partial days behind, so the pair is refused. + it("refuses a value range beside groupBy=day", async () => { const res = await GET( - getRequest("type=ACTIVITY_STEPS&groupBy=day&limit=25&offset=25"), + getRequest("type=ACTIVITY_STEPS&groupBy=day&valueMin=100"), ); - expect(res.status).toBe(200); - const json = (await res.json()) as { - data: { measurements: unknown[]; meta: { total: number } }; - }; - expect(json.data.measurements).toEqual([]); - expect(json.data.meta.total).toBe(3); - expect(rawCallsWith("SUM(")).toHaveLength(0); - // Past the end, and only there, the total is counted on its own. - expect(rawCallsWith("COUNT(DISTINCT")).toHaveLength(1); - }); - - // v1.29.6 — a day with both an Apple Health AND a Withings write for the - // same cumulative metric must collapse to the ladder-canonical source - // before summing, exactly like the dashboard rollup tile - // (`collapseRollupRowsBySource`) does. Without the collapse, 9000 - // Apple-Health steps + 8800 Withings steps summed to a fabricated 17800 — - // a number the tile never showed. - it("collapses cross-source rows to the ladder-canonical source before summing", async () => { - mockDayPage(["2026-05-15"], 1, [ - group("2026-05-15", 8800, 1, { source: "WITHINGS" }), - group("2026-05-15", 9000, 1), - ]); - - const res = await GET( - getRequest( - "type=ACTIVITY_STEPS&groupBy=day&from=2026-05-15T00:00:00Z&to=2026-05-16T00:00:00Z", - ), - ); - expect(res.status).toBe(200); - const json = (await res.json()) as { - data: { - measurements: Array<{ - dayKey: string; - value: number; - source: string; - sampleCount: number; - }>; - meta: { droppedDuplicates?: number }; - }; - }; - expect(json.data.measurements.length).toBe(1); - const [day] = json.data.measurements; - // APPLE_HEALTH beats WITHINGS on the default `steps` ladder — the - // Withings row drops out of the sum entirely rather than adding on top. - expect(day.value).toBe(9000); - expect(day.source).toBe("APPLE_HEALTH"); - expect(day.sampleCount).toBe(1); - expect(json.data.meta.droppedDuplicates).toBe(1); - }); - - // #1245 — `limit` / `offset` page over days and reach the SQL that picks - // them; `meta.total` is the distinct-day count, not the page length, and - // no row is marked partial because every page day is read whole. - it("pages over days and reports the distinct-day total", async () => { - mockDayPage(["2026-04-20"], 40, [group("2026-04-20", 3400, 1)]); - - const res = await GET( - getRequest("type=ACTIVITY_STEPS&groupBy=day&limit=25&offset=25"), - ); - expect(res.status).toBe(200); - const json = (await res.json()) as { - data: { - measurements: Array<{ dayKey: string; partial?: boolean }>; - meta: { total: number; limit: number; offset: number }; - }; - }; - expect(json.data.meta).toMatchObject({ total: 40, limit: 25, offset: 25 }); - expect(json.data.measurements.map((m) => m.dayKey)).toEqual(["2026-04-20"]); - expect(json.data.measurements[0].partial).toBeUndefined(); - const [pageCall] = rawCallsWith("COUNT(*) OVER ()"); - expect(pageCall?.slice(1)).toEqual(expect.arrayContaining([25])); - expect(pageCall?.slice(1).filter((v) => v === 25)).toHaveLength(2); - // The total rode on the page's own rows; there was no second pass over - // the history, only the fold of the page's one day. - expect(prisma.$queryRaw).toHaveBeenCalledTimes(2); - expect(rawCallsWith("COUNT(DISTINCT")).toHaveLength(0); - }); - - // A device split inside the winning source keeps the top device's - // groups only, the same narrowing the picker applies to raw rows. - it("keeps the top-ranked device's groups within the winning source", async () => { - mockDayPage(["2026-05-15"], 1, [ - group("2026-05-15", 6000, 40, { device_type: "watch" }), - group("2026-05-15", 5500, 12, { device_type: "phone" }), - ]); - const res = await GET(getRequest("type=ACTIVITY_STEPS&groupBy=day")); - const json = (await res.json()) as { - data: { - measurements: Array<{ value: number; sampleCount: number }>; - meta: { droppedDuplicates: number }; - }; - }; - expect(json.data.measurements).toHaveLength(1); - expect(json.data.measurements[0]).toMatchObject({ - value: 6000, - sampleCount: 40, - }); - expect(json.data.meta.droppedDuplicates).toBe(12); + expect(res.status).toBe(422); + expect(dayGroupedListResponse).not.toHaveBeenCalled(); }); it("falls back to the legacy findMany path when groupBy=day is set on a non-cumulative type", async () => { @@ -351,7 +158,7 @@ describe("GET /api/measurements — groupBy=day (W7c collapsed list)", () => { }); }); -describe("GET /api/measurements — dayKey drill-down (W7c)", () => { +describe("GET /api/measurements — dayKey drill-down", () => { it("returns per-sample rows for the requested calendar day in the user's TZ", async () => { vi.mocked(prisma.measurement.findMany).mockResolvedValue([ { @@ -402,7 +209,7 @@ describe("GET /api/measurements — dayKey drill-down (W7c)", () => { ); }); - // v1.4.37 W10 H-1 — DST awareness. The drill-down window must honour + // v1.4.37 — DST awareness. The drill-down window must honour // the day's true local span (23 h on spring-forward, 25 h on // fall-back) so the row count matches what the collapsed daily row // displays. Previous shape (`canonicalDailyTimestamp ± 12h`) leaked @@ -451,24 +258,23 @@ describe("GET /api/measurements — dayKey drill-down (W7c)", () => { const json = (await res.json()) as { data: { meta: { dayKey?: string } }; }; - // No dayKey echo on the legacy path — proves the W7c branch + // No dayKey echo on the legacy path — proves the drill-down branch // didn't fire for the spot metric. expect(json.data.meta.dayKey).toBeUndefined(); }); }); -describe("GET /api/measurements — schema rejections (W10 reconcile)", () => { +describe("GET /api/measurements — schema rejections", () => { // #1245 — the day-grouped mode pages over days, so an offset is a real // page request now rather than a refused combination. it("accepts offset>0 when groupBy=day is set", async () => { - mockDayPage([], 0); const res = await GET( getRequest("type=ACTIVITY_STEPS&groupBy=day&offset=25"), ); expect(res.status).toBe(200); }); - // v1.4.37 W10 H1 — same restriction for the drill-down branch; it + // v1.4.37 — same restriction for the drill-down branch; it // returns a single bounded page per dayKey, not a cursor. it("rejects offset>0 when dayKey is set", async () => { const res = await GET( @@ -478,7 +284,7 @@ describe("GET /api/measurements — schema rejections (W10 reconcile)", () => { expect(prisma.measurement.findMany).not.toHaveBeenCalled(); }); - // v1.4.37 W10 H2 — `2026-02-30` is a YYYY-MM-DD string but not a + // v1.4.37 — `2026-02-30` is a YYYY-MM-DD string but not a // real calendar date. `new Date()` silently overflows it to March // 2, so the drill-down would point at the wrong day. The validator // refine must reject it at parse time. diff --git a/src/app/api/measurements/route.ts b/src/app/api/measurements/route.ts index 1efeef64c..7d8467d88 100644 --- a/src/app/api/measurements/route.ts +++ b/src/app/api/measurements/route.ts @@ -29,12 +29,6 @@ import { type AggregateGrain, } from "@/lib/measurements/range-aggregation"; import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; -import { metricKeyForType } from "@/lib/measurements/cumulative-day-sum"; -import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority"; -import { - canonicalDailyTimestamp, - dayKeyForUserTz, -} from "@/lib/measurements/drain-per-sample-cumulative"; import { localDayWindow } from "@/lib/tz/local-day"; import { withIdempotency } from "@/lib/idempotency"; import { isP2002 } from "@/lib/prisma-errors"; @@ -50,11 +44,8 @@ import { readDailySeries, readLiveBuckets, } from "@/lib/measurements/daily-series-read"; -import { - reconstructSleepSessions, - pickMainNightAndNaps, - SESSION_GAP_MS, -} from "@/lib/analytics/sleep-night"; +import { dayGroupedListResponse } from "@/lib/measurements/list/day-grouped"; +import { sleepListResponse } from "@/lib/measurements/list/sleep-list"; import { resolveStoredTimezone, resolveUserTimezone } from "@/lib/tz/resolver"; import { isNamedSleepStageBody, @@ -72,12 +63,10 @@ import { oppositeMergeSource, } from "@/lib/measurements/cross-source-merge"; import { NextRequest } from "next/server"; -import { - Prisma, - type MeasurementType, - type MeasurementSource, - type GlucoseContext, - type SleepStage, +import type { + MeasurementType, + MeasurementSource, + GlucoseContext, } from "@/generated/prisma/client"; export const GET = apiHandler(async (request: NextRequest) => { @@ -86,7 +75,7 @@ export const GET = apiHandler(async (request: NextRequest) => { const params = Object.fromEntries(request.nextUrl.searchParams); const parsed = listMeasurementsSchema.safeParse(params); if (!parsed.success) { - // v1.4.43 W6 — surface every Zod issue under `details.issues` so a + // v1.4.43 — surface every Zod issue under `details.issues` so a // multi-field iOS query (chart loader bug, drill-down regression) // doesn't iterate one error per round-trip. Best-effort audit- // ledger breadcrumb keyed `measurements.list.validation-failed`. @@ -133,7 +122,7 @@ export const GET = apiHandler(async (request: NextRequest) => { const where = { userId: user.id, - // v1.4.40 W-DELETED — soft-deleted readings are invisible to every + // v1.4.40 — soft-deleted readings are invisible to every // list / count / aggregate read. The dedicated `sync/state` route // keeps surfacing them via its own filter so iOS can reconcile // tombstones; every other consumer (list view, analytics fan-out, @@ -185,7 +174,7 @@ export const GET = apiHandler(async (request: NextRequest) => { }); } - // v1.4.37 W7c — list-view drill-down to per-sample rows for a single + // v1.4.37 — list-view drill-down to per-sample rows for a single // calendar day in the user's IANA timezone. Resolves the day's UTC // bounds against `User.timezone`, then runs a plain `findMany` so the // expandable list row shows every per-sample chunk the daily collapse @@ -196,7 +185,7 @@ export const GET = apiHandler(async (request: NextRequest) => { // drill-down opens exactly the day the row summed. const tz = await resolveStoredTimezone(user.timezone); // Resolve `[dayStart, dayEnd)` for the requested calendar day in - // the user's local zone. v1.4.37 W10 — `localDayWindow` honours + // the user's local zone. v1.4.37 — `localDayWindow` honours // DST: the previous shape (`canonicalDailyTimestamp ± 12 h`) // silently leaked or hid one hour of samples on spring-forward // (23-h day) and fall-back (25-h day) by always spanning exactly @@ -232,7 +221,7 @@ export const GET = apiHandler(async (request: NextRequest) => { }); } - // v1.4.37 W7c — collapsed "one row per day" mode for cumulative + // v1.4.37 — collapsed "one row per day" mode for cumulative // types. The list page renders the day's SUM as a single synthesised // row with a `sampleCount` chevron that drills back into the // per-sample rows via the `dayKey=…` branch above. `limit` / `offset` @@ -248,26 +237,24 @@ export const GET = apiHandler(async (request: NextRequest) => { offset, sortDir, sourceEq, - valueMin, - valueMax, from, to, }); } - // v1.4.28 FB-D2 — server-side aggregation. Gated on an explicit + // v1.4.28 — server-side aggregation. Gated on an explicit // `aggregate` param so the iOS contract (raw `MeasurementWireDTO` // shape on `GET /api/measurements`) is byte-stable for any caller // that omits the new query parameter. The chart-data client opts // in; iOS does not. // - // R4-CODE-C1 — `take` no longer applies BEFORE bucketising. The + // `take` no longer applies BEFORE bucketising. The // aggregation runs in Postgres via `date_trunc`, the bucket cap is // applied AFTER, so a 1-year `aggregate=daily` window walks every // row in the window and returns up to 365 buckets per type instead // of truncating to the first N raw rows. // - // SD-H1 — "All time" range semantics. When the chart's "All" tab + // "All time" range semantics. When the chart's "All" tab // is active, the client passes `from` = the user's earliest // measurement (or a sentinel like 1970-01-01) plus `to` = now plus // `aggregate=monthly` (or `weekly` when full history < 2 years). @@ -369,574 +356,6 @@ export const GET = apiHandler(async (request: NextRequest) => { }); }); -/** - * The day-grouped list for a cumulative type, paged over local days. - * - * Everything heavy runs in Postgres. The first query picks the page: the - * distinct local days that hold a reading, sorted and cut by `limit` / - * `offset`, each row carrying the distinct-day count of the whole filtered - * set as `meta.total` from a window count. The second folds only the page's - * days, one row per day, source, device and unit with its sum and sample - * count, and the source ladder picks among those groups exactly as it would - * among the raw rows (it keeps or drops whole source and device groups per - * day, never single rows). So a page costs its own days whatever the length - * of the history, every day is complete, and no raw row reaches the heap. - * - * Until v1.43.6 the branch read the newest `limit` raw rows (the web asked - * for 5000), bucketed them and returned every bucket. A long history came - * back as thousands of rows in one response, and every day older than the - * 5000th raw row was missing with nothing to say so. - * - * A day exists for the page when any reading of the type falls on it; the - * source collapse narrows a day's groups but never empties one, so the count - * and the folded rows agree. - */ -async function dayGroupedListResponse( - user: { id: string; timezone?: string | null }, - opts: { - type: MeasurementType; - limit: number; - offset: number; - sortDir: "asc" | "desc"; - sourceEq?: MeasurementSource; - valueMin?: number; - valueMax?: number; - from?: Date; - to?: Date; - }, -) { - const { type, limit, offset, sortDir } = opts; - // One resolver for the stored zone, shared with the drill-down and the - // sleep list: a usable IANA name or the instance default, never a value - // Postgres would refuse. The drill-down must cut the same days. - const tz = await resolveStoredTimezone(user.timezone); - - // The request's filters, as SQL: the same ones the plain list applies. - const filters = Prisma.sql` - m."user_id" = ${user.id} - AND m."type" = ${type}::measurement_type - AND m."deleted_at" IS NULL - ${opts.sourceEq ? Prisma.sql`AND m."source" = ${opts.sourceEq}::measurement_source` : Prisma.empty} - ${opts.valueMin != null ? Prisma.sql`AND m."value" >= ${opts.valueMin}` : Prisma.empty} - ${opts.valueMax != null ? Prisma.sql`AND m."value" <= ${opts.valueMax}` : Prisma.empty} - ${opts.from ? Prisma.sql`AND m."measured_at" >= ${opts.from}` : Prisma.empty} - ${opts.to ? Prisma.sql`AND m."measured_at" <= ${opts.to}` : Prisma.empty} - `; - const localDay = Prisma.sql`to_char((m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD')`; - // Direction comes from the Zod enum, never from raw input. The window - // count runs before LIMIT, so every row of the page carries the - // distinct-day total of the whole filtered set. - const direction = Prisma.raw(sortDir === "asc" ? "ASC" : "DESC"); - const pageDays = await prisma.$queryRaw< - Array<{ day_key: string; total: number }> - >` - SELECT d.day_key, (COUNT(*) OVER ())::int AS total - FROM ( - SELECT ${localDay} AS day_key - FROM measurements m - WHERE ${filters} - GROUP BY 1 - ) d - ORDER BY d.day_key ${direction} - LIMIT ${limit} OFFSET ${offset} - `; - // A page past the end has no row to carry the total; only then is it - // counted on its own. - const total = - pageDays.length > 0 - ? Number(pageDays[0].total) - : offset > 0 - ? Number( - ( - await prisma.$queryRaw>` - SELECT COUNT(DISTINCT ${localDay})::int AS total - FROM measurements m - WHERE ${filters} - ` - )[0]?.total ?? 0, - ) - : 0; - const dayKeys = pageDays.map((d) => d.day_key); - - // The page's days are consecutive in the sorted day set, so the span from - // the first one's start to the last one's end holds exactly their rows. - let groups: Array<{ - day_key: string; - source: MeasurementSource; - device_type: string | null; - unit: string; - total: number; - n: number; - }> = []; - if (dayKeys.length > 0) { - const sorted = [...dayKeys].sort(); - const spanStart = localDayWindow(sorted[0], tz).dayStart; - const spanEnd = localDayWindow(sorted[sorted.length - 1], tz).dayEnd; - // Newest group first within a day, so the day's label comes from its - // latest reading, as it did when raw rows were read newest first. - groups = await prisma.$queryRaw` - SELECT - ${localDay} AS day_key, - m."source"::text AS source, - m."device_type" AS device_type, - m."unit" AS unit, - SUM(m."value")::float8 AS total, - COUNT(*)::int AS n - FROM measurements m - WHERE ${filters} - AND m."measured_at" >= ${spanStart} - AND m."measured_at" < ${spanEnd} - GROUP BY 1, 2, 3, 4 - ORDER BY 1, MAX(m."measured_at") DESC, 2, 3, 4 - `; - } - - // v1.29.6 — collapse overlapping sources to the ladder-canonical - // reading per day BEFORE summing. Without this, a day with both an - // Apple Health AND a Withings write for the same cumulative metric - // (e.g. steps) summed BOTH sources into one inflated total — a day - // the dashboard tile (which reads through the source-collapsed - // rollup, `collapseRollupRowsBySource` in - // `src/lib/rollups/measurement-read.ts`) showed correctly. Uses the - // same `metricKeyForType` ladder lookup the rollup collapse uses, so - // this list and the tile agree on which source wins. A type with no - // ladder (e.g. TIME_IN_DAYLIGHT, FALL_COUNT) has nothing to collapse - // against — every group passes through unchanged. Each group stands at - // its day's local noon, so the picker files it under the day SQL cut. - const groupRows = groups.map((g) => ({ - ...g, - type, - deviceType: g.device_type, - measuredAt: canonicalDailyTimestamp(g.day_key, tz), - })); - const metricKey = metricKeyForType(type); - const priorityJson = metricKey ? await loadUserSourcePriority(user.id) : null; - const canonicalGroups = metricKey - ? pickCanonicalSourceRows(groupRows, metricKey, priorityJson, (d) => - dayKeyForUserTz(d, tz), - ).canonicalRows - : groupRows; - - // Sum per local day; unit and source come from the day's first kept group - // so the rendered row carries a sensible label even when sources mix. - const buckets = new Map< - string, - { total: number; sampleCount: number; unit: string; source: string } - >(); - for (const g of canonicalGroups) { - const slot = buckets.get(g.day_key); - if (!slot) { - buckets.set(g.day_key, { - total: g.total, - sampleCount: g.n, - unit: g.unit, - source: g.source, - }); - } else { - slot.total += g.total; - slot.sampleCount += g.n; - } - } - const rowsScanned = groups.reduce((sum, g) => sum + g.n, 0); - const rowsKept = canonicalGroups.reduce((sum, g) => sum + g.n, 0); - - // Rows follow the page's day order from SQL. - const measurements = dayKeys.flatMap((dayKey) => { - const b = buckets.get(dayKey); - if (!b) return []; - return [ - { - // Synthetic id so the list-row key stays stable across pages - // and the row never collides with a real `Measurement.id`. - id: `day:${type}:${dayKey}`, - type, - value: b.total, - unit: b.unit, - source: b.source, - // Anchor the row's timestamp to local noon so the row sorts - // cleanly between same-day spot samples — matches the - // `canonicalDailyTimestamp` convention used by the drain. - measuredAt: canonicalDailyTimestamp(dayKey, tz).toISOString(), - notes: null, - dayKey, - sampleCount: b.sampleCount, - }, - ]; - }); - - annotate({ - action: { name: "measurement.list" }, - meta: { - total, - type, - groupBy: "day", - days: measurements.length, - rowsScanned, - droppedDuplicates: rowsScanned - rowsKept, - }, - }); - return apiSuccess({ - measurements, - meta: { - total, - limit, - offset, - groupBy: "day", - droppedDuplicates: rowsScanned - rowsKept, - }, - }); -} - -/** - * A sleep page holds at most this many nights. The nights' stage rows are - * read and reconstructed in the process, so the bound sits on the nights; - * the applied value comes back as `meta.limit`, and `meta.total` still counts - * every night, so a caller asking for more pages on rather than losing any. - * Until v1.43.6 the list read no more than a year of nights however large - * the limit, so this is a widening, never a new cut. - */ -const SLEEP_NIGHT_PAGE_MAX = 1000; - -/** - * v1.11.5 — per-night collapse for the SLEEP_DURATION list. Sleep is - * stored one row per STAGE per night (minutes); a plain list leaks every - * stage segment. This collapses the raw rows into the canonical per-night - * shape used by the chart series + dashboard tile. - * - * Two modes: - * - `dayKey` set → drill-down: return the night's stage SEGMENTS (the - * canonical source's per-stage spans) so the list row can reveal the - * hypnogram's underlying rows, mirroring the cumulative-type drill-down. - * - no `dayKey` → one synthetic row per night, value = TIME ASLEEP in - * minutes (the canonical unit), with the per-night stage breakdown, - * nap count, and awakenings in the row so the UI can caption the night - * without a second fetch. - * - * The nights are paged in Postgres (`sleepNightPage`), like the days of the - * cumulative list, and only the stage rows of the page's nights are read. - * Until v1.43.6 the read stopped at a year before `to` and the list ended - * there without saying so to the client. - */ -async function sleepListResponse( - user: { id: string; timezone?: string | null }, - opts: { - from?: Date; - to?: Date; - limit: number; - offset: number; - sortDir: "asc" | "desc"; - dayKey?: string; - }, -) { - const { from, to, offset, sortDir, dayKey } = opts; - const limit = Math.min(opts.limit, SLEEP_NIGHT_PAGE_MAX); - const [tz, priorityJson] = await Promise.all([ - resolveStoredTimezone(user.timezone), - loadUserSourcePriority(user.id), - ]); - - const page = await sleepNightPage(user.id, tz, { - from, - to, - sortDir, - ...(dayKey ? { night: dayKey, limit: 1, offset: 0 } : { limit, offset }), - }); - const nightKeys = page.nights.map((n) => n.night); - - // The page's sessions lie between the first one's start and the last one's - // end, and every other session is more than the session gap away from that - // span, so this read holds the page's sessions whole and no part of any - // other. - let rows: Array<{ - id: string; - value: number; - measuredAt: Date; - sleepStage: SleepStage | null; - source: MeasurementSource; - deviceType: string | null; - }> = []; - if (page.nights.length > 0) { - let spanStart = page.nights[0].spanStart; - let spanEnd = page.nights[0].spanEnd; - for (const n of page.nights) { - if (n.spanStart < spanStart) spanStart = n.spanStart; - if (n.spanEnd > spanEnd) spanEnd = n.spanEnd; - } - rows = await prisma.measurement.findMany({ - where: { - userId: user.id, - type: "SLEEP_DURATION", - deletedAt: null, - AND: [ - { measuredAt: { gte: spanStart, lte: spanEnd } }, - ...(from ? [{ measuredAt: { gte: from } }] : []), - ...(to ? [{ measuredAt: { lte: to } }] : []), - ], - }, - orderBy: [{ measuredAt: "asc" }, { id: "asc" }], - select: { - id: true, - value: true, - measuredAt: true, - sleepStage: true, - source: true, - // Writer-level collapse: two HealthKit apps behind one source (watch - // stages vs phone in-bed) must not blend into one night. - deviceType: true, - }, - }); - } - // Each reconstructed session is filed under the page night whose span - // holds it, and takes that night's label. Matching by the reconstruction's - // own label lost a night whenever the two named different days: the page - // counted it under one, the reconstruction filed it under the other, and - // the filter dropped it from both. - const sessions = reconstructSleepSessions(rows, tz, priorityJson).flatMap( - (session) => { - // A second of slack absorbs the sub-millisecond rounding between the - // two start computations; sessions sit hours apart, so it can never - // reach a neighbour. - const home = page.nights.find( - (n) => - n.spanStart.getTime() - 1000 <= session.start.getTime() && - session.end.getTime() <= n.spanEnd.getTime() + 1000, - ); - return home ? [{ ...session, night: home.night }] : []; - }, - ); - - // Drill-down — return the requested night's canonical stage segments. - if (dayKey) { - const { main, naps } = pickMainNightAndNaps( - sessions.filter((s) => s.night === dayKey), - ); - const dayUnit = getUnitForType("SLEEP_DURATION"); - // One row per stage segment of the main night + each nap, so the - // drill-down reveals exactly the rows that summed into the headline. - const segments = [main, ...naps] - .filter((s): s is NonNullable => s != null) - .flatMap((s) => s.segments); - const measurements = segments - .sort((a, b) => - sortDir === "asc" - ? a.start.getTime() - b.start.getTime() - : b.start.getTime() - a.start.getTime(), - ) - .map((seg, i) => ({ - id: `sleep-seg:${dayKey}:${i}`, - type: "SLEEP_DURATION", - value: Math.round(seg.minutes), - unit: dayUnit, - source: main?.source ?? naps[0]?.source ?? "MANUAL", - measuredAt: seg.end.toISOString(), - sleepStage: seg.stage, - notes: null, - })); - annotate({ - action: { name: "measurement.list" }, - meta: { - total: measurements.length, - type: "SLEEP_DURATION", - dayKey, - mode: "sleep-drill-down", - }, - }); - return apiSuccess({ - measurements, - meta: { total: measurements.length, limit, offset: 0, dayKey }, - }); - } - - // Per-night collapse — one synthetic row per wake day. The headline - // value is the MAIN night's TIME ASLEEP (the longest session that day), - // per the nap convention: naps are surfaced as a count, never folded - // into the main night's headline. - const byDay = new Map(); - for (const s of sessions) { - const list = byDay.get(s.night) ?? []; - list.push(s); - byDay.set(s.night, list); - } - - const dayUnit = getUnitForType("SLEEP_DURATION"); - // Rows follow the page's night order from SQL. - const measurements = nightKeys.flatMap((day) => { - const { main, naps } = pickMainNightAndNaps(byDay.get(day) ?? []); - if (!main) return []; - return [ - { - // Synthetic id mirrors the series route's `sleep:` shape. - id: `sleep:${day}`, - type: "SLEEP_DURATION", - value: Math.round(main.asleepMinutes), - unit: dayUnit, - source: main.source ?? "MANUAL", - measuredAt: main.end.toISOString(), - notes: null, - dayKey: day, - // Drives the drill-down chevron — number of stage segments behind - // the night, matching the cumulative-type `sampleCount` contract. - sampleCount: main.segments.length, - sleepStages: main.stages as Record, - inBedMinutes: main.inBedMinutes, - awakeMinutes: main.awakeMinutes, - napCount: naps.length, - napAsleepMinutes: naps.reduce((sum, n) => sum + n.asleepMinutes, 0), - awakenings: main.awakenings, - }, - ]; - }); - - annotate({ - action: { name: "measurement.list" }, - meta: { - total: page.total, - type: "SLEEP_DURATION", - mode: "sleep-per-night", - nights: measurements.length, - rowsScanned: rows.length, - }, - }); - return apiSuccess({ - measurements, - meta: { - total: page.total, - limit, - offset, - groupBy: "night", - }, - }); -} - -/** - * One page of sleep nights, chosen in Postgres. - * - * The stage rows are cut into sessions the way `reconstructSleepSessions` - * cuts them: ordered by start (`measured_at` minus the minutes), a new - * session wherever a row starts more than `SESSION_GAP_MS` after every - * earlier row has ended. A session counts when it holds an asleep row (a - * bare or granular asleep stage, or a stage-less one, with minutes), and its - * night is the local day its last row ends on. Nights are grouped, sorted - * and cut by `limit` / `offset`; `total` is the night count of the whole - * filtered set from the same query. `spanStart` / `spanEnd` bound the - * night's counting sessions, so the caller reads exactly their rows. - * - * This label is the list's own: the page is cut by it, a row carries it and - * a drill-down asks by it. It is taken over every writer, where the JS - * reconstruction takes the winning writer only, so the two can name - * different days (a losing phone's in-bed span running past midnight while - * the watch stops before it). The caller therefore never matches the two by - * label: it files each reconstructed session under the night whose span - * holds it, which is exact, because a reconstructed session is one of these - * sessions narrowed to its winner. - * - * With `night` set (a drill-down) the scan is cut to a window of 36 hours - * either side of that local day. Sessions are split at a three-hour gap, so - * a session ending on that day starts inside the window unless it lasted - * over a day and a half, and every session the cut truncates ends on another - * day. - */ -/** How far a drill-down's session scan reaches either side of its day. */ -const DRILL_DOWN_REACH_MS = 36 * 60 * 60 * 1000; - -async function sleepNightPage( - userId: string, - tz: string, - opts: { - from?: Date; - to?: Date; - sortDir: "asc" | "desc"; - limit: number; - offset: number; - night?: string; - }, -): Promise<{ - nights: Array<{ night: string; spanStart: Date; spanEnd: Date }>; - total: number; -}> { - const gapSeconds = SESSION_GAP_MS / 1000; - const window = opts.night ? localDayWindow(opts.night, tz) : null; - // Direction comes from the Zod enum, never from raw input. - const direction = Prisma.raw(opts.sortDir === "asc" ? "ASC" : "DESC"); - const nightsCte = Prisma.sql` - WITH r AS ( - SELECT - m."measured_at" AS e, - m."measured_at" - m."value" * interval '1 minute' AS s, - (m."value" > 0 AND (m."sleep_stage" IS NULL - OR m."sleep_stage" IN ('ASLEEP', 'REM', 'CORE', 'DEEP'))) AS asleep - FROM measurements m - WHERE m."user_id" = ${userId} - AND m."type" = 'SLEEP_DURATION'::measurement_type - AND m."deleted_at" IS NULL - ${opts.from ? Prisma.sql`AND m."measured_at" >= ${opts.from}` : Prisma.empty} - ${opts.to ? Prisma.sql`AND m."measured_at" <= ${opts.to}` : Prisma.empty} - ${ - window - ? Prisma.sql`AND m."measured_at" >= ${new Date(window.dayStart.getTime() - DRILL_DOWN_REACH_MS)} - AND m."measured_at" < ${new Date(window.dayEnd.getTime() + DRILL_DOWN_REACH_MS)}` - : Prisma.empty - } - ), o AS ( - SELECT r.*, MAX(e) OVER ( - ORDER BY s, e ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING - ) AS prev_end - FROM r - ), g AS ( - SELECT o.*, SUM( - CASE WHEN prev_end IS NULL - OR s - prev_end > make_interval(secs => ${gapSeconds}::double precision) THEN 1 ELSE 0 END - ) OVER (ORDER BY s, e ROWS UNBOUNDED PRECEDING) AS sid - FROM o - ), sess AS ( - SELECT MIN(s) AS s, MAX(e) AS e - FROM g - GROUP BY sid - HAVING BOOL_OR(asleep) - ), nights AS ( - SELECT - to_char((e AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD') AS night, - MIN(s) AS span_start, - MAX(e) AS span_end - FROM sess - GROUP BY 1 - ) - `; - const rows = await prisma.$queryRaw< - Array<{ night: string; span_start: Date; span_end: Date; total: number }> - >` - ${nightsCte} - SELECT night, span_start, span_end, (COUNT(*) OVER ())::int AS total - FROM nights - ${opts.night ? Prisma.sql`WHERE night = ${opts.night}` : Prisma.empty} - ORDER BY night ${direction} - LIMIT ${opts.limit} OFFSET ${opts.offset} - `; - // A page past the end has no row to carry the total; only then is it - // counted on its own. - const total = - rows.length > 0 - ? Number(rows[0].total) - : opts.offset > 0 - ? Number( - ( - await prisma.$queryRaw>` - ${nightsCte} - SELECT COUNT(*)::int AS total FROM nights - ` - )[0]?.total ?? 0, - ) - : 0; - return { - nights: rows.map((r) => ({ - night: r.night, - spanStart: r.span_start, - spanEnd: r.span_end, - })), - total, - }; -} - /** * Does this body attribute a source to any reading — any source at all? * diff --git a/src/lib/measurements/list/day-grouped.ts b/src/lib/measurements/list/day-grouped.ts new file mode 100644 index 000000000..cd40b9d80 --- /dev/null +++ b/src/lib/measurements/list/day-grouped.ts @@ -0,0 +1,233 @@ +import { prisma } from "@/lib/db"; +import { annotate } from "@/lib/logging/context"; +import { apiSuccess } from "@/lib/api-response"; +import { metricKeyForType } from "@/lib/measurements/cumulative-day-sum"; +import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority"; +import { + canonicalDailyTimestamp, + dayKeyForUserTz, +} from "@/lib/measurements/drain-per-sample-cumulative"; +import { localDayWindow } from "@/lib/tz/local-day"; +import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { resolveStoredTimezone } from "@/lib/tz/resolver"; +import { + Prisma, + type MeasurementSource, + type MeasurementType, +} from "@/generated/prisma/client"; +import { pageWithWindowTotal } from "./paging"; + +/** + * The day-grouped list for a cumulative type, paged over local days. + * + * Everything heavy runs in Postgres. The first query picks the page: the + * distinct local days that hold a reading, sorted and cut by `limit` / + * `offset`, each row carrying the distinct-day count of the whole filtered + * set as `meta.total` from a window count. The second folds only the page's + * days, one row per day, source, device and unit with its sum and sample + * count, and the source ladder picks among those groups exactly as it would + * among the raw rows (it keeps or drops whole source and device groups per + * day, never single rows). So a page costs its own days whatever the length + * of the history, every day is complete, and no raw row reaches the heap. + * + * Until v1.43.6 the branch read the newest `limit` raw rows (the web asked + * for 5000), bucketed them and returned every bucket. A long history came + * back as thousands of rows in one response, and every day older than the + * 5000th raw row was missing with nothing to say so. + * + * A day exists for the page when any reading of the type falls on it; the + * source collapse narrows a day's groups but never empties one, so the count + * and the folded rows agree. A value range would filter single readings and + * leave partial days behind, so the validator refuses `valueMin` / + * `valueMax` beside `groupBy=day`. + * + * Every page reads the filtered history to find its days. After the nightly + * drain a cumulative type holds about one row per day and source, so the + * scan is the index-only walk over days it would be anyway. + */ +export async function dayGroupedListResponse( + user: { id: string; timezone?: string | null }, + opts: { + type: MeasurementType; + limit: number; + offset: number; + sortDir: "asc" | "desc"; + sourceEq?: MeasurementSource; + from?: Date; + to?: Date; + }, +) { + const { type, limit, offset, sortDir } = opts; + // One resolver for the stored zone, shared with the drill-down and the + // sleep list: a usable IANA name or the instance default, never a value + // Postgres would refuse. The drill-down must cut the same days. + const tz = await resolveStoredTimezone(user.timezone); + + // The request's filters, as SQL: the same ones the plain list applies. + const filters = Prisma.sql` + m."user_id" = ${user.id} + AND m."type" = ${type}::measurement_type + AND m."deleted_at" IS NULL + ${opts.sourceEq ? Prisma.sql`AND m."source" = ${opts.sourceEq}::measurement_source` : Prisma.empty} + ${opts.from ? Prisma.sql`AND m."measured_at" >= ${opts.from}` : Prisma.empty} + ${opts.to ? Prisma.sql`AND m."measured_at" <= ${opts.to}` : Prisma.empty} + `; + const localDay = Prisma.sql`to_char((m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD')`; + // Direction comes from the Zod enum, never from raw input. The window + // count runs before LIMIT, so every row of the page carries the + // distinct-day total of the whole filtered set. + const direction = Prisma.raw(sortDir === "asc" ? "ASC" : "DESC"); + const { rows: pageDays, total } = await pageWithWindowTotal( + await prisma.$queryRaw>` + SELECT d.day_key, (COUNT(*) OVER ())::int AS total + FROM ( + SELECT ${localDay} AS day_key + FROM measurements m + WHERE ${filters} + GROUP BY 1 + ) d + ORDER BY d.day_key ${direction} + LIMIT ${limit} OFFSET ${offset} + `, + offset, + async () => + Number( + ( + await prisma.$queryRaw>` + SELECT COUNT(DISTINCT ${localDay})::int AS total + FROM measurements m + WHERE ${filters} + ` + )[0]?.total ?? 0, + ), + ); + const dayKeys = pageDays.map((d) => d.day_key); + + // The page's days are consecutive in the sorted day set, so the span from + // the first one's start to the last one's end holds exactly their rows. + let groups: Array<{ + day_key: string; + source: MeasurementSource; + device_type: string | null; + unit: string; + total: number; + n: number; + }> = []; + if (dayKeys.length > 0) { + const sorted = [...dayKeys].sort(); + const spanStart = localDayWindow(sorted[0], tz).dayStart; + const spanEnd = localDayWindow(sorted[sorted.length - 1], tz).dayEnd; + // Newest group first within a day, so the day's label comes from its + // latest reading, as it did when raw rows were read newest first. + groups = await prisma.$queryRaw` + SELECT + ${localDay} AS day_key, + m."source"::text AS source, + m."device_type" AS device_type, + m."unit" AS unit, + SUM(m."value")::float8 AS total, + COUNT(*)::int AS n + FROM measurements m + WHERE ${filters} + AND m."measured_at" >= ${spanStart} + AND m."measured_at" < ${spanEnd} + GROUP BY 1, 2, 3, 4 + ORDER BY 1, MAX(m."measured_at") DESC, 2, 3, 4 + `; + } + + // v1.29.6 — collapse overlapping sources to the ladder-canonical + // reading per day BEFORE summing. Without this, a day with both an + // Apple Health AND a Withings write for the same cumulative metric + // (e.g. steps) summed BOTH sources into one inflated total — a day + // the dashboard tile (which reads through the source-collapsed + // rollup, `collapseRollupRowsBySource` in + // `src/lib/rollups/measurement-read.ts`) showed correctly. Uses the + // same `metricKeyForType` ladder lookup the rollup collapse uses, so + // this list and the tile agree on which source wins. A type with no + // ladder (e.g. TIME_IN_DAYLIGHT, FALL_COUNT) has nothing to collapse + // against — every group passes through unchanged. Each group stands at + // its day's local noon, so the picker files it under the day SQL cut. + const groupRows = groups.map((g) => ({ + ...g, + type, + deviceType: g.device_type, + measuredAt: canonicalDailyTimestamp(g.day_key, tz), + })); + const metricKey = metricKeyForType(type); + const priorityJson = metricKey ? await loadUserSourcePriority(user.id) : null; + const canonicalGroups = metricKey + ? pickCanonicalSourceRows(groupRows, metricKey, priorityJson, (d) => + dayKeyForUserTz(d, tz), + ).canonicalRows + : groupRows; + + // Sum per local day; unit and source come from the day's first kept group + // so the rendered row carries a sensible label even when sources mix. + const buckets = new Map< + string, + { total: number; sampleCount: number; unit: string; source: string } + >(); + for (const g of canonicalGroups) { + const slot = buckets.get(g.day_key); + if (!slot) { + buckets.set(g.day_key, { + total: g.total, + sampleCount: g.n, + unit: g.unit, + source: g.source, + }); + } else { + slot.total += g.total; + slot.sampleCount += g.n; + } + } + const rowsScanned = groups.reduce((sum, g) => sum + g.n, 0); + const rowsKept = canonicalGroups.reduce((sum, g) => sum + g.n, 0); + + // Rows follow the page's day order from SQL. + const measurements = dayKeys.flatMap((dayKey) => { + const b = buckets.get(dayKey); + if (!b) return []; + return [ + { + // Synthetic id so the list-row key stays stable across pages + // and the row never collides with a real `Measurement.id`. + id: `day:${type}:${dayKey}`, + type, + value: b.total, + unit: b.unit, + source: b.source, + // Anchor the row's timestamp to local noon so the row sorts + // cleanly between same-day spot samples — matches the + // `canonicalDailyTimestamp` convention used by the drain. + measuredAt: canonicalDailyTimestamp(dayKey, tz).toISOString(), + notes: null, + dayKey, + sampleCount: b.sampleCount, + }, + ]; + }); + + annotate({ + action: { name: "measurement.list" }, + meta: { + total, + type, + groupBy: "day", + days: measurements.length, + rowsScanned, + droppedDuplicates: rowsScanned - rowsKept, + }, + }); + return apiSuccess({ + measurements, + meta: { + total, + limit, + offset, + groupBy: "day", + droppedDuplicates: rowsScanned - rowsKept, + }, + }); +} diff --git a/src/lib/measurements/list/paging.ts b/src/lib/measurements/list/paging.ts new file mode 100644 index 000000000..f96ccd262 --- /dev/null +++ b/src/lib/measurements/list/paging.ts @@ -0,0 +1,17 @@ +/** + * A page cut in SQL that carries the size of the whole set. + * + * The page query selects `(COUNT(*) OVER ())::int AS total` beside its rows: + * the window count runs before `LIMIT`, so every row of the page carries the + * count of the whole filtered set and one round trip answers both. A page + * past the end has no row to carry it; only then is the set counted on its + * own, by `countAll`. + */ +export async function pageWithWindowTotal( + rows: T[], + offset: number, + countAll: () => Promise, +): Promise<{ rows: T[]; total: number }> { + if (rows.length > 0) return { rows, total: Number(rows[0].total) }; + return { rows, total: offset > 0 ? await countAll() : 0 }; +} diff --git a/src/lib/measurements/list/sleep-list.ts b/src/lib/measurements/list/sleep-list.ts new file mode 100644 index 000000000..32b205de8 --- /dev/null +++ b/src/lib/measurements/list/sleep-list.ts @@ -0,0 +1,169 @@ +/** + * The SLEEP_DURATION list: one row per night, or one night's stage segments. + * + * Sleep is stored one row per STAGE per night (minutes); a plain list would + * leak every stage segment as its own "value". The list collapses the raw rows + * into the canonical per-night shape used by the chart series and the + * dashboard tile. + * + * Two modes: + * - `dayKey` set → drill-down: the night's stage SEGMENTS (the canonical + * source's per-stage spans), read exactly as the night view reads them. + * - no `dayKey` → one synthetic row per night, value = the main sleep's TIME + * ASLEEP in minutes (the canonical unit), with the per-night stage + * breakdown, nap count and awakenings in the row so the UI can caption the + * night without a second fetch. + * + * The nights are paged in Postgres (`sleepNightPage`) and only the stage rows + * of the page's nights are read. Every page night gets a row, so a page is as + * long as its limit and `meta.total` counts exactly the rows a full walk sees. + * A day whose only sleep is daytime naps has no main night: its row carries + * 0 minutes asleep and its naps, as `/api/sleep/night` answers the same day + * with `main: null` and the naps. + */ +import { annotate } from "@/lib/logging/context"; +import { apiSuccess } from "@/lib/api-response"; +import { getUnitForType } from "@/lib/validations/measurement"; +import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { resolveStoredTimezone } from "@/lib/tz/resolver"; +import { + pickMainNightAndNaps, + reconstructSleepSessions, + type SleepSession, +} from "@/lib/analytics/sleep-night"; +import { + readNightSessions, + readNightSpanRows, + sleepNightPage, +} from "./sleep-nights"; + +/** + * A sleep page holds at most this many nights. The nights' stage rows are + * read and reconstructed in the process, so the bound sits on the nights; + * the applied value comes back as `meta.limit`, and `meta.total` still counts + * every night, so a caller asking for more pages on rather than losing any. + */ +export const SLEEP_NIGHT_PAGE_MAX = 1000; + +export async function sleepListResponse( + user: { id: string; timezone?: string | null }, + opts: { + from?: Date; + to?: Date; + limit: number; + offset: number; + sortDir: "asc" | "desc"; + dayKey?: string; + }, +) { + const { from, to, offset, sortDir, dayKey } = opts; + const limit = Math.min(opts.limit, SLEEP_NIGHT_PAGE_MAX); + const [tz, priorityJson] = await Promise.all([ + resolveStoredTimezone(user.timezone), + loadUserSourcePriority(user.id), + ]); + const dayUnit = getUnitForType("SLEEP_DURATION"); + + if (dayKey) { + const { sessions } = await readNightSessions( + user.id, + dayKey, + tz, + priorityJson, + { from, to }, + ); + const { main, naps } = pickMainNightAndNaps(sessions); + // One row per stage segment of the main night + each nap, so the + // drill-down reveals exactly the rows that summed into the row. + const measurements = [main, ...naps] + .filter((s): s is SleepSession => s != null) + .flatMap((s) => s.segments) + .sort((a, b) => + sortDir === "asc" + ? a.start.getTime() - b.start.getTime() + : b.start.getTime() - a.start.getTime(), + ) + .map((seg, i) => ({ + id: `sleep-seg:${dayKey}:${i}`, + type: "SLEEP_DURATION", + value: Math.round(seg.minutes), + unit: dayUnit, + source: main?.source ?? naps[0]?.source ?? "MANUAL", + measuredAt: seg.end.toISOString(), + sleepStage: seg.stage, + notes: null, + })); + annotate({ + action: { name: "measurement.list" }, + meta: { + total: measurements.length, + type: "SLEEP_DURATION", + dayKey, + mode: "sleep-drill-down", + }, + }); + return apiSuccess({ + measurements, + meta: { total: measurements.length, limit, offset: 0, dayKey }, + }); + } + + const page = await sleepNightPage(user.id, tz, { + from, + to, + sortDir, + limit, + offset, + }); + const rows = await readNightSpanRows(user.id, page.nights, { from, to }); + const byDay = new Map(); + for (const s of reconstructSleepSessions(rows, tz, priorityJson)) { + const list = byDay.get(s.night) ?? []; + list.push(s); + byDay.set(s.night, list); + } + + // Rows follow the page's night order from SQL. + const measurements = page.nights.map(({ night, spanEnd }) => { + const { main, naps } = pickMainNightAndNaps(byDay.get(night) ?? []); + const lastNap = naps.at(-1); + return { + // Synthetic id mirrors the series route's `sleep:` shape. + id: `sleep:${night}`, + type: "SLEEP_DURATION", + value: main ? Math.round(main.asleepMinutes) : 0, + unit: dayUnit, + source: main?.source ?? naps[0]?.source ?? "MANUAL", + measuredAt: (main?.end ?? lastNap?.end ?? spanEnd).toISOString(), + notes: null, + dayKey: night, + // Drives the drill-down chevron: the stage segments behind the row, + // the main night's and the naps', exactly what the drill-down returns. + sampleCount: [main, ...naps].reduce( + (sum, s) => sum + (s?.segments.length ?? 0), + 0, + ), + sleepStages: (main?.stages ?? {}) as Record, + inBedMinutes: main?.inBedMinutes ?? null, + awakeMinutes: main?.awakeMinutes ?? null, + napCount: naps.length, + napAsleepMinutes: naps.reduce((sum, n) => sum + n.asleepMinutes, 0), + awakenings: main?.awakenings ?? 0, + }; + }); + + annotate({ + action: { name: "measurement.list" }, + meta: { + total: page.total, + type: "SLEEP_DURATION", + mode: "sleep-per-night", + nights: measurements.length, + rowsScanned: rows.length, + }, + }); + return apiSuccess({ + measurements, + meta: { total: page.total, limit, offset, groupBy: "night" }, + }); +} diff --git a/src/lib/measurements/list/sleep-nights.ts b/src/lib/measurements/list/sleep-nights.ts new file mode 100644 index 000000000..ee6f3608a --- /dev/null +++ b/src/lib/measurements/list/sleep-nights.ts @@ -0,0 +1,321 @@ +/** + * Sleep nights read from Postgres, a page or a night at a time. + * + * Sleep is stored one row per stage, so a long history is a large row set: + * every reader here keeps the stage rows that reach the process to the + * nights it was asked about, and the reconstruction itself stays the one in + * `sleep-night.ts`. The SQL below only decides which nights exist and where + * their sessions lie; what a night holds is always the JS reconstruction's. + */ +import { prisma } from "@/lib/db"; +import { Prisma } from "@/generated/prisma/client"; +import type { MeasurementSource, SleepStage } from "@/generated/prisma/client"; +import { + ASLEEP_STAGES, + SESSION_GAP_MS, + reconstructSleepNights, + reconstructSleepSessions, + type SleepNight, + type SleepSession, +} from "@/lib/analytics/sleep-night"; +import { localDayWindow } from "@/lib/tz/local-day"; +import { pageWithWindowTotal } from "./paging"; + +/** A stage row as every sleep reader selects it. */ +export interface SleepRow { + id: string; + value: number; + measuredAt: Date; + sleepStage: SleepStage | null; + source: MeasurementSource; + deviceType: string | null; +} + +/** The read bounds a list request passes through, both inclusive. */ +export interface SleepReadBounds { + from?: Date; + to?: Date; +} + +/** + * How far a one-night read reaches either side of its day. Sessions split at + * a three-hour gap, so a session ending on that day starts inside the reach + * unless it lasted over a day and a half, and every session the cut + * truncates ends on another day. + */ +export const NIGHT_READ_REACH_MS = 36 * 60 * 60 * 1000; + +/** + * The asleep stages as a SQL list. A whitelist splice: each value is asserted + * against the closed enum shape before it is joined, so nothing from a + * request can reach it, and the list is `ASLEEP_STAGES`, the very set the + * JS reconstruction counts as asleep. + */ +const ASLEEP_STAGE_SQL = Prisma.raw( + ASLEEP_STAGES.map((stage) => { + if (!/^[A-Z_]+$/.test(stage)) throw new Error(`bad sleep stage ${stage}`); + return `'${stage}'`; + }).join(", "), +); + +/** + * Stage rows between two instants (inclusive), oldest first, narrowed by the + * request's own bounds. + */ +export async function readSleepRows( + userId: string, + gte: Date, + lte: Date, + bounds: SleepReadBounds = {}, +): Promise { + return prisma.measurement.findMany({ + where: { + userId, + type: "SLEEP_DURATION", + deletedAt: null, + AND: [ + { measuredAt: { gte, lte } }, + ...(bounds.from ? [{ measuredAt: { gte: bounds.from } }] : []), + ...(bounds.to ? [{ measuredAt: { lte: bounds.to } }] : []), + ], + }, + orderBy: [{ measuredAt: "asc" }, { id: "asc" }], + select: { + id: true, + value: true, + measuredAt: true, + sleepStage: true, + source: true, + // Writer-level collapse: two HealthKit apps behind one source (watch + // stages vs phone in-bed) must not blend into one night. + deviceType: true, + }, + }); +} + +/** + * The sessions of one wake day: the stage rows within `NIGHT_READ_REACH_MS` + * of the day, reconstructed, and the sessions filed under it kept. + */ +export async function readNightSessions( + userId: string, + day: string, + tz: string, + priorityJson: unknown, + bounds: SleepReadBounds = {}, +): Promise<{ sessions: SleepSession[]; rows: SleepRow[] }> { + const { dayStart, dayEnd } = localDayWindow(day, tz); + const rows = await readSleepRows( + userId, + new Date(dayStart.getTime() - NIGHT_READ_REACH_MS), + new Date(dayEnd.getTime() + NIGHT_READ_REACH_MS), + bounds, + ); + const sessions = reconstructSleepSessions(rows, tz, priorityJson).filter( + (s) => s.night === day, + ); + return { sessions, rows }; +} + +/** + * The wake day of the most recent night that holds sleep, or null. Found + * from the latest asleep row: the night holding it is the newest one, and + * its sessions lie within the one-night reach of that row. + */ +export async function latestSleepNight( + userId: string, + tz: string, + priorityJson: unknown, +): Promise { + const latest = await prisma.measurement.findFirst({ + where: { + userId, + type: "SLEEP_DURATION", + deletedAt: null, + value: { gt: 0 }, + OR: [{ sleepStage: null }, { sleepStage: { in: [...ASLEEP_STAGES] } }], + }, + orderBy: { measuredAt: "desc" }, + select: { measuredAt: true }, + }); + if (!latest) return null; + const at = latest.measuredAt.getTime(); + const rows = await readSleepRows( + userId, + new Date(at - NIGHT_READ_REACH_MS), + new Date(at + NIGHT_READ_REACH_MS), + ); + const scorable = reconstructSleepSessions(rows, tz, priorityJson).filter( + (s) => s.asleepMinutes > 0, + ); + return scorable.at(-1)?.night ?? null; +} + +/** One night of a page: its wake day and the span its sessions cover. */ +export interface SleepNightSpan { + night: string; + spanStart: Date; + spanEnd: Date; +} + +/** + * One page of sleep nights, chosen in Postgres. + * + * The stage rows are cut into sessions the way `reconstructSleepSessions` + * cuts them: ordered by start (`measured_at` minus the minutes), a new + * session wherever a row starts more than `SESSION_GAP_MS` after every + * earlier row has ended. A session counts when it holds an asleep row (an + * `ASLEEP_STAGES` stage, or a stage-less row, with minutes), and its night is + * the local day its last row ends on: the reconstruction's own rule, so a + * page night and a reconstructed night name the same day. Nights are grouped, + * sorted and cut by `limit` / `offset`; `total` counts the whole filtered set. + * `spanStart` / `spanEnd` bound the night's counting sessions, so the caller + * reads exactly their rows. + * + * Before the cut, rows are folded into hour buckets by their end: min start, + * max end, any asleep. That cannot move a session boundary. Two rows whose + * ends lie less than the session gap apart always share a session (the later + * one starts before its own end, which is within the gap of the earlier end), + * so a bucket narrower than the gap only ever holds rows of one session, and + * its bounds are that group's bounds. It cuts the rows the window functions + * sort from every stage to a handful per night. + */ +export async function sleepNightPage( + userId: string, + tz: string, + opts: SleepReadBounds & { + sortDir: "asc" | "desc"; + limit: number; + offset: number; + }, +): Promise<{ nights: SleepNightSpan[]; total: number }> { + const gapSeconds = SESSION_GAP_MS / 1000; + // Direction comes from the Zod enum, never from raw input. + const direction = Prisma.raw(opts.sortDir === "asc" ? "ASC" : "DESC"); + const nightsCte = Prisma.sql` + WITH r AS ( + SELECT + MIN(m."measured_at" - m."value" * interval '1 minute') AS s, + MAX(m."measured_at") AS e, + BOOL_OR(m."value" > 0 AND (m."sleep_stage" IS NULL + OR m."sleep_stage" IN (${ASLEEP_STAGE_SQL}))) AS asleep + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."type" = 'SLEEP_DURATION'::measurement_type + AND m."deleted_at" IS NULL + ${opts.from ? Prisma.sql`AND m."measured_at" >= ${opts.from}` : Prisma.empty} + ${opts.to ? Prisma.sql`AND m."measured_at" <= ${opts.to}` : Prisma.empty} + GROUP BY date_bin(interval '1 hour', m."measured_at", TIMESTAMPTZ '2000-01-01 00:00:00+00') + ), o AS ( + SELECT r.*, MAX(e) OVER ( + ORDER BY s, e ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING + ) AS prev_end + FROM r + ), g AS ( + SELECT o.*, SUM( + CASE WHEN prev_end IS NULL + OR s - prev_end > make_interval(secs => ${gapSeconds}::double precision) THEN 1 ELSE 0 END + ) OVER (ORDER BY s, e ROWS UNBOUNDED PRECEDING) AS sid + FROM o + ), sess AS ( + SELECT MIN(s) AS s, MAX(e) AS e + FROM g + GROUP BY sid + HAVING BOOL_OR(asleep) + ), nights AS ( + SELECT + to_char((e AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD') AS night, + MIN(s) AS span_start, + MAX(e) AS span_end + FROM sess + GROUP BY 1 + ) + `; + const { rows, total } = await pageWithWindowTotal( + await prisma.$queryRaw< + Array<{ night: string; span_start: Date; span_end: Date; total: number }> + >` + ${nightsCte} + SELECT night, span_start, span_end, (COUNT(*) OVER ())::int AS total + FROM nights + ORDER BY night ${direction} + LIMIT ${opts.limit} OFFSET ${opts.offset} + `, + opts.offset, + async () => + Number( + ( + await prisma.$queryRaw>` + ${nightsCte} + SELECT COUNT(*)::int AS total FROM nights + ` + )[0]?.total ?? 0, + ), + ); + return { + nights: rows.map((r) => ({ + night: r.night, + spanStart: r.span_start, + spanEnd: r.span_end, + })), + total, + }; +} + +/** + * The stage rows of a set of page nights, read in one span. The nights are + * consecutive in the sorted night set, so the span from the first one's start + * to the last one's end holds their sessions whole; any other session in it + * holds no sleep. + */ +export async function readNightSpanRows( + userId: string, + nights: readonly SleepNightSpan[], + bounds: SleepReadBounds = {}, +): Promise { + if (nights.length === 0) return []; + let spanStart = nights[0].spanStart; + let spanEnd = nights[0].spanEnd; + for (const n of nights) { + if (n.spanStart < spanStart) spanStart = n.spanStart; + if (n.spanEnd > spanEnd) spanEnd = n.spanEnd; + } + return readSleepRows(userId, spanStart, spanEnd, bounds); +} + +/** How many nights one chunked read reconstructs at a time. */ +const NIGHT_CHUNK = 90; + +/** + * Every night in a window as `reconstructSleepNights` builds it, oldest + * first, without holding the window's stage rows at once: Postgres names the + * nights and their spans, and the rows are read and reconstructed ninety + * nights at a time. Any window length costs a bounded heap, so a reader never + * has to cut a long range short to stay safe. `maxNights` keeps the oldest + * nights of the window, or with `newest` the most recent ones. + */ +export async function readSleepNights( + userId: string, + tz: string, + priorityJson: unknown, + bounds: SleepReadBounds & { maxNights: number; newest?: boolean }, +): Promise { + const page = await sleepNightPage(userId, tz, { + from: bounds.from, + to: bounds.to, + sortDir: bounds.newest ? "desc" : "asc", + limit: bounds.maxNights, + offset: 0, + }); + const nights = bounds.newest ? page.nights.reverse() : page.nights; + const out: SleepNight[] = []; + for (let i = 0; i < nights.length; i += NIGHT_CHUNK) { + const chunk = nights.slice(i, i + NIGHT_CHUNK); + const keys = new Set(chunk.map((n) => n.night)); + const rows = await readNightSpanRows(userId, chunk, bounds); + for (const night of reconstructSleepNights(rows, tz, priorityJson)) { + if (keys.has(night.night)) out.push(night); + } + } + return out; +} diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index 35e46830c..ab61bae48 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -465,7 +465,7 @@ const sleepNightResponse = z .meta({ id: "SleepNightResource", description: - "One reconstructed sleep night: the main session's hypnogram segments + breakdown, plus same-wake-day naps surfaced separately.", + "One reconstructed sleep night: the main session's hypnogram segments + breakdown, plus same-wake-day naps surfaced separately. A session is a nap when it holds under three hours asleep centred between 10:00 and 20:00 local time; the main session is the longest session that is not a nap. A day whose only sleep is naps has `main: null` and its naps.", }); // ── Sleep rhythm (sleep-debt + chronotype, v1.17.0) ────────────────── @@ -804,7 +804,7 @@ export const measurementPaths: NonNullable = { summary: "List measurements", description: "Filter by type + date range. Response is paged via offset/limit; default limit 100.\n\n" + - "With `groupBy=day` on a cumulative type the rows are one sum per local day, and `limit` / `offset` page over those days; every returned day is complete. `type=SLEEP_DURATION` returns one row per night, paged the same way over nights. On both, `meta.total` counts the days or nights in the filtered window. A day-sum page takes `limit` as asked (up to the list's 5000); a sleep page holds at most 1000 nights, and `meta.limit` echoes the value applied.", + "With `groupBy=day` on a cumulative type the rows are one sum per local day, and `limit` / `offset` page over those days; every returned day is complete, which is why `valueMin` / `valueMax` (a filter on single readings) are refused with 422 beside `groupBy=day`. `type=SLEEP_DURATION` returns one row per night, paged the same way over nights; every night of the page has a row, and a day whose only sleep is daytime naps carries `value` 0 with its naps in `napCount` / `napAsleepMinutes`. Its `dayKey` is the same day `/api/sleep/night?date=` answers for. On both, `meta.total` counts the days or nights in the filtered window. A day-sum page takes `limit` as asked (up to the list's 5000); a sleep page holds at most 1000 nights, and `meta.limit` echoes the value applied.", requestParams: { query: listMeasurementsSchema, }, @@ -1118,7 +1118,7 @@ export const measurementPaths: NonNullable = { tags: ["Measurements"], summary: "Reconstructed sleep night (hypnogram source, v1.11.5)", description: - "Returns one night's reconstructed sleep session for the phase-progression (hypnogram) view: the canonical source's stage segments (each with an absolute start/end), the per-stage breakdown, asleep/in-bed/awake totals, the mid-sleep awakenings count, and same-wake-day naps surfaced separately. Session-clustered, keyed by the local wake day, collapsed to one source via the sleep priority ladder so two sources never overlay. `date` omitted returns the most recent night. A read-only view over existing per-stage SLEEP_DURATION rows — no schema, no new measurement type.", + "Returns one night's reconstructed sleep session for the phase-progression (hypnogram) view: the canonical source's stage segments (each with an absolute start/end), the per-stage breakdown, asleep/in-bed/awake totals, the mid-sleep awakenings count, and same-wake-day naps surfaced separately. Session-clustered, keyed by the local wake day, collapsed to one source via the sleep priority ladder so two sources never overlay. `date` omitted returns the most recent night holding sleep, however old. Any past date is answered, with no window cut. A read-only view over existing per-stage SLEEP_DURATION rows — no schema, no new measurement type.", requestParams: { query: z.object({ date: z diff --git a/src/lib/validations/measurement.ts b/src/lib/validations/measurement.ts index 24eb63e4e..a68336186 100644 --- a/src/lib/validations/measurement.ts +++ b/src/lib/validations/measurement.ts @@ -589,7 +589,7 @@ export const listMeasurementsSchema = z // Omitting `aggregate` keeps the raw wire shape (iOS contract); the // chart-data client must opt in explicitly. aggregate: z.enum(["raw", "daily", "weekly", "monthly"]).optional(), - // v1.4.36 W1 — opt-in source switch for daily-aggregate reads. When + // v1.4.36 — opt-in source switch for daily-aggregate reads. When // `source=rollup` + `aggregate=daily`, the route reads from the // persistent `measurement_rollups` DAY buckets instead of running a // live `date_trunc` GROUP BY scan over the raw `measurements` table. @@ -612,7 +612,7 @@ export const listMeasurementsSchema = z // `value: { gte, lte }` range. Server-authoritative; validated min<=max. valueMin: z.coerce.number().finite().optional(), valueMax: z.coerce.number().finite().optional(), - // v1.4.37 W7c — list-view "one row per day" mode for cumulative + // v1.4.37 — list-view "one row per day" mode for cumulative // types (steps, active energy, distance, flights, daylight). When // `groupBy=day` is set and `type` is a cumulative HK type, the route // returns one synthesised row per user-TZ day with `value` = SUM @@ -623,18 +623,18 @@ export const listMeasurementsSchema = z // picks the page's local days in Postgres and reports the distinct-day // count as `meta.total` (#1245). groupBy: z.enum(["day"]).optional(), - // v1.4.37 W7c — drill-down to per-sample rows for a single day in + // v1.4.37 — drill-down to per-sample rows for a single day in // the user's IANA timezone. Format `YYYY-MM-DD`; the route resolves // the day boundary against the user's `User.timezone`. Used by the // expandable list row to reveal the chunks that contributed to the // collapsed daily total. // - // v1.4.37 W10 — a drill-down returns one bounded page rather than a + // v1.4.37 — a drill-down returns one bounded page rather than a // cursor, so it refuses a non-zero `offset`. dayKey: z .string() .regex(/^\d{4}-\d{2}-\d{2}$/, "dayKey must be YYYY-MM-DD") - // v1.4.37 W10 — a `YYYY-MM-DD` string can satisfy the regex while + // v1.4.37 — a `YYYY-MM-DD` string can satisfy the regex while // still being an impossible calendar date (`2026-02-30`, // `2026-13-01`). `new Date("2026-02-30T00:00:00Z")` silently // overflows to March 2, so the drill-down would return rows from @@ -665,6 +665,19 @@ export const listMeasurementsSchema = z "limit must be <= 1000 when dayKey is set; drill-down responses are capped at 1000 rows", path: ["limit"], }) + // A value range filters single readings, and a day-grouped row is the sum + // of a whole day: filtering its readings would return partial days summed + // as if they were whole. The combination has no honest answer, so it is + // refused rather than served. + .refine( + ({ groupBy, valueMin, valueMax }) => + groupBy !== "day" || (valueMin == null && valueMax == null), + { + message: + "valueMin / valueMax filter single readings and cannot be combined with groupBy=day", + path: ["groupBy"], + }, + ) // v1.18.5 — an inverted value range (`valueMin > valueMax`) is a client // bug, not an empty result set; reject it at the validator with a 422 so // the filter UI can surface the mismatch instead of silently returning diff --git a/tests/integration/measurements-day-grouped-paging.test.ts b/tests/integration/measurements-day-grouped-paging.test.ts index 738a86f6d..17adde8f9 100644 --- a/tests/integration/measurements-day-grouped-paging.test.ts +++ b/tests/integration/measurements-day-grouped-paging.test.ts @@ -239,4 +239,56 @@ describe("GET /api/measurements?groupBy=day paging (#1245)", () => { expect(windowed.data.meta.total).toBe(inWindow); expect(windowed.data.measurements).toHaveLength(PAGE); }, 240_000); + + // The fold's per-day narrowing, pinned against Postgres rather than a mock + // that answers SQL by its text: a day is cut in the user's zone, and within + // the winning source the top-ranked device's groups stand alone. + it("cuts days in the user's zone and keeps the winning source's top device", async () => { + const user = await seedSession("day-paging-device-user"); + const key = dayKeyAgo(5); + const at = (iso: string) => new Date(`${key}T${iso}Z`); + const base = { + userId: user.id, + type: "ACTIVITY_STEPS" as const, + unit: "steps", + source: "APPLE_HEALTH" as const, + }; + await getPrismaClient().measurement.createMany({ + data: [ + // The watch and the phone both counted the morning. + { + ...base, + value: 6000, + deviceType: "watch", + measuredAt: at("08:00:00"), + externalId: "w1", + }, + { + ...base, + value: 5500, + deviceType: "phone", + measuredAt: at("08:05:00"), + externalId: "p1", + }, + // 23:30 UTC is already the next day in Berlin. + { + ...base, + value: 700, + deviceType: "watch", + measuredAt: at("23:30:00"), + externalId: "w2", + }, + ], + }); + const body = (await ( + await callList(`type=ACTIVITY_STEPS&groupBy=day&limit=${PAGE}`) + ).json()) as ListEnvelope & { + data: { meta: { droppedDuplicates?: number } }; + }; + const byDay = new Map(body.data.measurements.map((r) => [r.dayKey, r])); + expect(byDay.get(key)?.value).toBe(6000); + expect(byDay.get(dayKeyAgo(4))?.value).toBe(700); + expect(body.data.meta.total).toBe(2); + expect(body.data.meta.droppedDuplicates).toBe(1); + }, 240_000); }); diff --git a/tests/integration/sleep-night-sql-parity.test.ts b/tests/integration/sleep-night-sql-parity.test.ts new file mode 100644 index 000000000..129ec0697 --- /dev/null +++ b/tests/integration/sleep-night-sql-parity.test.ts @@ -0,0 +1,307 @@ +/** + * The sleep list pages its nights in SQL (`sleepNightPage`) and fills them + * from the JS reconstruction (`reconstructSleepSessions`). The two cut the + * same stage rows into sessions and name each session's night by the same + * rule, or the list drops, doubles or mislabels nights. This suite runs both + * over one fixture set of awkward shapes and holds them to the same nights, + * then holds the list, the night view and the series to one answer per day. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +type Stage = "IN_BED" | "AWAKE" | "ASLEEP" | "REM" | "CORE" | "DEEP" | null; + +interface Seg { + /** Segment end, minutes after the fixture day's UTC midnight. */ + end: number; + minutes: number; + stage: Stage; + source?: "APPLE_HEALTH" | "WHOOP" | "WITHINGS"; + deviceType?: string | null; +} + +const H = 60; + +/** A run of contiguous segments of `step` minutes cycling through stages. */ +function run( + start: number, + end: number, + step: number, + stages: Stage[], + extra: Partial = {}, +): Seg[] { + const out: Seg[] = []; + let i = 0; + for (let t = start + step; t <= end; t += step) { + out.push({ + end: t, + minutes: step, + stage: stages[i % stages.length], + ...extra, + }); + i += 1; + } + return out; +} + +const NIGHT: Stage[] = ["CORE", "DEEP", "CORE", "REM", "AWAKE"]; + +/** + * Ten shapes, one per day modulo ten, minutes relative to the day's UTC + * midnight (negative = the evening before). + */ +function shape(i: number): Seg[] { + switch (i % 10) { + case 0: + // A dense watch night in five-minute segments. + return run(-1 * H, 7 * H, 5, NIGHT, { deviceType: "watch" }); + case 1: + // The watch stops before midnight, the phone's in-bed runs past it. + return [ + ...run(-6 * H, -10, 10, NIGHT, { deviceType: "watch" }), + { end: 20, minutes: 6 * H + 20, stage: "IN_BED", deviceType: "phone" }, + ]; + case 2: + // An afternoon nap and no night. + return run(13 * H, 14 * H, 15, ["CORE", "REM"]); + case 3: + // A night and a nap. + return [ + ...run(-2 * H, 6 * H, 30, NIGHT), + ...run(14 * H, 14 * H + 40, 20, ["CORE"]), + ]; + case 4: + // A split night, three and a half hours apart: two sessions. + return [ + ...run(-2 * H, 1 * H, 30, NIGHT), + ...run(4 * H + 30, 7 * H, 30, NIGHT), + ]; + case 5: + // Exactly the session gap apart: one session. + return [ + ...run(-2 * H, 1 * H, 30, NIGHT), + ...run(4 * H, 7 * H, 30, NIGHT), + ]; + case 6: + // One long stage block. + return [ + { end: 3 * H, minutes: 4 * H, stage: "CORE" }, + { end: 5 * H, minutes: 2 * H, stage: "REM" }, + ]; + case 7: + // A bare aggregate with its granular twin, and a zero-minute granular + // row from a second writer beside a bare night. + return [ + { end: 6 * H, minutes: 7 * H, stage: "ASLEEP" }, + ...run(-1 * H, 6 * H, 60, ["CORE", "DEEP", "REM"]), + { end: 6 * H, minutes: 0, stage: "REM", source: "WITHINGS" }, + { end: 6 * H, minutes: 6 * H, stage: "ASLEEP", source: "WITHINGS" }, + ]; + case 8: + // An in-bed-only evening and a stage-less night. + return [ + { end: -8 * H, minutes: 40, stage: "IN_BED" }, + { end: -8 * H + 30, minutes: 20, stage: "AWAKE" }, + { end: 6 * H, minutes: 6 * H, stage: null }, + ]; + default: + // A summary-shaped writer (every stage on one instant) beside a + // partial timed night. + return [ + { end: 7 * H, minutes: 200, stage: "CORE", source: "WHOOP" }, + { end: 7 * H, minutes: 90, stage: "DEEP", source: "WHOOP" }, + { end: 7 * H, minutes: 80, stage: "REM", source: "WHOOP" }, + ...run(2 * H, 7 * H, 30, NIGHT), + ]; + } +} + +const BASE = Date.UTC(2024, 2, 1); // three years back, across two DST turns +const DAYS = 120; + +async function seed(timezone: string) { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { + username: `parity-${timezone.replace("/", "-")}`.toLowerCase(), + email: `parity-${timezone.length}@example.test`, + role: "USER", + timezone, + }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.set("healthlog_session", session.id); + const rows = []; + for (let d = 0; d < DAYS; d++) { + const midnight = BASE + d * 86_400_000; + for (const [k, seg] of shape(d).entries()) { + rows.push({ + userId: user.id, + type: "SLEEP_DURATION" as const, + value: seg.minutes, + unit: "minutes", + source: seg.source ?? ("APPLE_HEALTH" as const), + deviceType: seg.deviceType ?? null, + measuredAt: new Date(midnight + seg.end * 60_000), + externalId: `p-${d}-${k}`, + sleepStage: seg.stage, + }); + } + } + await prisma.measurement.createMany({ data: rows }); + return user; +} + +interface ListRow { + dayKey: string; + value: number; + napCount: number; + napAsleepMinutes: number; +} + +async function list(query: string) { + const { GET } = await import("@/app/api/measurements/route"); + const res = await GET( + new NextRequest(`http://localhost/api/measurements?${query}`), + ); + expect(res.status).toBe(200); + return (await res.json()) as { + data: { measurements: ListRow[]; meta: { total: number } }; + }; +} + +async function nightView(date: string) { + const { GET } = await import("@/app/api/sleep/night/route"); + const res = await GET( + new NextRequest(`http://localhost/api/sleep/night?date=${date}`), + ); + expect(res.status).toBe(200); + return (await res.json()) as { + data: { + night: string | null; + main: { asleepMinutes: number } | null; + naps: Array<{ asleepMinutes: number }>; + }; + }; +} + +describe("sleep nights: SQL page and JS reconstruction agree", () => { + for (const tz of ["UTC", "Europe/Berlin"]) { + it(`names the same nights over every shape (${tz})`, async () => { + const user = await seed(tz); + const { sleepNightPage, readSleepRows } = + await import("@/lib/measurements/list/sleep-nights"); + const { reconstructSleepSessions } = + await import("@/lib/analytics/sleep-night"); + const page = await sleepNightPage(user.id, tz, { + sortDir: "asc", + limit: 10_000, + offset: 0, + }); + const rows = await readSleepRows( + user.id, + new Date(BASE - 2 * 86_400_000), + new Date(BASE + (DAYS + 2) * 86_400_000), + ); + const scorable = reconstructSleepSessions(rows, tz, null).filter( + (s) => s.asleepMinutes > 0, + ); + const jsNights = [...new Set(scorable.map((s) => s.night))].sort(); + expect(page.nights.map((n) => n.night)).toEqual(jsNights); + expect(page.total).toBe(jsNights.length); + // Every reconstructed session lies inside its night's SQL span. + const span = new Map(page.nights.map((n) => [n.night, n])); + for (const s of scorable) { + const n = span.get(s.night)!; + expect(s.start.getTime()).toBeGreaterThanOrEqual( + n.spanStart.getTime() - 1000, + ); + expect(s.end.getTime()).toBeLessThanOrEqual(n.spanEnd.getTime() + 1000); + } + }, 240_000); + } + + it("gives every page night a row the night view answers for the same day", async () => { + await seed("Europe/Berlin"); + const first = await list("type=SLEEP_DURATION&limit=7&sortDir=asc"); + const total = first.data.meta.total; + const rows: ListRow[] = []; + for (let offset = 0; offset < total; offset += 7) { + const page = await list( + `type=SLEEP_DURATION&limit=7&offset=${offset}&sortDir=asc`, + ); + // No short page before the last one. + expect(page.data.measurements).toHaveLength(Math.min(7, total - offset)); + rows.push(...page.data.measurements); + } + expect(rows).toHaveLength(total); + let napOnly = 0; + for (const row of rows) { + const view = await nightView(row.dayKey); + expect(view.data.night).toBe(row.dayKey); + expect(view.data.main?.asleepMinutes ?? 0).toBe(row.value); + expect(view.data.naps).toHaveLength(row.napCount); + if (!view.data.main) { + napOnly += 1; + expect(row.napCount).toBeGreaterThan(0); + } + } + // The afternoon-nap days have no main night and still have their row. + expect(napOnly).toBe(DAYS / 10); + }, 480_000); + + it("reaches the oldest night from the series' All range", async () => { + await seed("Europe/Berlin"); + const { GET } = await import("@/app/api/measurements/series/route"); + const res = await GET( + new NextRequest( + "http://localhost/api/measurements/series?kind=sleep&days=3650", + ), + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { points: Array<{ id: string }> }; + }; + const oldest = (await list("type=SLEEP_DURATION&limit=1&sortDir=asc")).data + .measurements[0].dayKey; + // Years old, past any one-year cut. + expect(body.data.points[0].id).toBe(`sleep:${oldest}`); + expect(oldest < "2024-03-03").toBe(true); + }, 240_000); +});