diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 626d0e6b9..f1d4e182f 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -4198,7 +4198,14 @@ paths: tags: - Measurements summary: List measurements - description: Filter by type + date range. Response is paged via offset/limit; default limit 100, hard cap 500. + description: >- + Filter by type + date range. Response is paged via offset/limit; default limit 100. + + + With `groupBy=day` on a cumulative type the rows are one sum per local day, and `limit` / `offset` page over + those days; every returned day is complete. `type=SLEEP_DURATION` returns one row per night, paged the same way + over nights. On both, `meta.total` counts the days or nights in the filtered window, and a page holds at most + 1000 of them (`meta.limit` echoes the value applied). parameters: - in: query name: type @@ -46242,9 +46249,6 @@ components: type: integer minimum: -9007199254740991 maximum: 9007199254740991 - partial: - description: True on the one bucket whose day was cut off by the read cap, so its sum understates the real total. - type: boolean required: - id - type @@ -47069,9 +47073,6 @@ components: type: integer minimum: -9007199254740991 maximum: 9007199254740991 - partial: - description: True on the one bucket whose day was cut off by the read cap, so its sum understates the real total. - type: boolean required: - id - type @@ -47140,12 +47141,6 @@ components: type: integer minimum: 0 maximum: 9007199254740991 - windowFrom: - description: "Set on the per-night SLEEP_DURATION mode when the requested window was longer than 365 days: the instant - the read started from. The window is a year ending at `to` (or now); page further back by moving `to`." - type: string - format: date-time - pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ required: - total - limit @@ -47259,9 +47254,6 @@ components: type: integer minimum: -9007199254740991 maximum: 9007199254740991 - partial: - description: True on the one bucket whose day was cut off by the read cap, so its sum understates the real total. - type: boolean status: type: string enum: diff --git a/src/app/api/measurements/__tests__/group-by-day.test.ts b/src/app/api/measurements/__tests__/group-by-day.test.ts index 4f26f7c23..a37bb293c 100644 --- a/src/app/api/measurements/__tests__/group-by-day.test.ts +++ b/src/app/api/measurements/__tests__/group-by-day.test.ts @@ -74,6 +74,67 @@ function getRequest(query: string): NextRequest { }); } +interface Group { + day_key: string; + source: string; + device_type: string | null; + unit: string; + total: number; + n: number; +} + +/** One SQL fold row: a day's readings of one source, device and unit. */ +function group( + day_key: string, + total: number, + n: number, + over: Partial = {}, +): Group { + return { + day_key, + source: "APPLE_HEALTH", + device_type: null, + unit: "steps", + total, + n, + ...over, + }; +} + +/** + * The day-grouped branch answers from SQL alone: one query picks the page's + * days, each row carrying the distinct-day total from a window count; a + * second folds those days into source groups. A page past the end has no row + * to carry the total, so only then is it counted on its own. Answer each + * query by its text. + */ +function mockDayPage( + days: string[], + total = days.length, + groups: Group[] = [], +) { + vi.mocked(prisma.$queryRaw).mockImplementation((( + strings: TemplateStringsArray, + ) => { + const text = strings.join(""); + if (text.includes("COUNT(DISTINCT")) return Promise.resolve([{ total }]); + if (text.includes("COUNT(*) OVER ()")) { + return Promise.resolve(days.map((day_key) => ({ day_key, total }))); + } + if (text.includes("SUM(")) return Promise.resolve(groups); + return Promise.resolve([]); + }) as never); +} + +/** The `$queryRaw` calls whose text contains `needle`. */ +function rawCallsWith(needle: string) { + return vi + .mocked(prisma.$queryRaw) + .mock.calls.filter(([strings]) => + (strings as unknown as string[]).join("").includes(needle), + ); +} + beforeEach(() => { vi.resetAllMocks(); vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); @@ -91,55 +152,12 @@ beforeEach(() => { describe("GET /api/measurements — groupBy=day (W7c collapsed list)", () => { it("collapses per-sample step rows into one row per user-TZ day with sampleCount", async () => { - // Three Apple-Watch chunks on 2026-05-15 (Europe/Berlin) + - // two on 2026-05-16 — the route should sum each day's bucket. - vi.mocked(prisma.measurement.findMany).mockResolvedValue([ - { - id: "m-1", - type: "ACTIVITY_STEPS", - value: 1200, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-15T08:00:00.000Z"), - notes: null, - }, - { - id: "m-2", - type: "ACTIVITY_STEPS", - value: 3400, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-15T11:00:00.000Z"), - notes: null, - }, - { - id: "m-3", - type: "ACTIVITY_STEPS", - value: 800, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-15T16:00:00.000Z"), - notes: null, - }, - { - id: "m-4", - type: "ACTIVITY_STEPS", - value: 2500, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-16T07:00:00.000Z"), - notes: null, - }, - { - id: "m-5", - type: "ACTIVITY_STEPS", - value: 4500, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-16T14:00:00.000Z"), - notes: null, - }, - ] as never); + // Three Apple-Watch chunks on 2026-05-15 (Europe/Berlin) summing to 5400 + // and two on 2026-05-16 summing to 7000, as SQL folds them. + mockDayPage(["2026-05-16", "2026-05-15"], 2, [ + group("2026-05-15", 5400, 3), + group("2026-05-16", 7000, 2), + ]); const res = await GET( getRequest( @@ -177,21 +195,42 @@ describe("GET /api/measurements — groupBy=day (W7c collapsed list)", () => { expect(day15.measuredAt).toContain("2026-05-15T10:00:00"); }); - // v1.29.6 — the raw-row scan must read the MOST RECENT window before - // bucketing, mirroring the workouts-list ordering fix (05f80ad37): the - // canonical/day-bucket picker doesn't reorder its input, but the - // underlying `orderBy: asc` + `take: limit` sliced the OLDEST page of - // rows before the day-bucketing loop ran — on a long history the most - // recent days never appeared at all. - it("scans the raw rows newest-first before bucketing", async () => { - vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); + // #1245 — the fold covers exactly the page's days: from the start of the + // oldest page day to the end of the newest, in the user's zone, on top of + // the request's own filters. Nothing outside the page is scanned, and no + // raw row is read into the process at all. + it("folds only the page's days, in SQL", async () => { + mockDayPage(["2026-05-16", "2026-05-15"], 40); await GET(getRequest("type=ACTIVITY_STEPS&groupBy=day")); - const call = vi.mocked(prisma.measurement.findMany).mock.calls[0]?.[0] as { - orderBy?: { measuredAt?: "asc" | "desc" }; + const [fold] = rawCallsWith("SUM("); + const instants = (fold ?? []) + .slice(1) + .filter((v): v is Date => v instanceof Date) + .map((d) => d.toISOString()); + // Europe/Berlin is UTC+2 in May. + expect(instants).toEqual([ + "2026-05-14T22:00:00.000Z", + "2026-05-16T22:00:00.000Z", + ]); + expect(prisma.measurement.findMany).not.toHaveBeenCalled(); + }); + + it("folds nothing when the page holds no day", async () => { + mockDayPage([], 3); + const res = await GET( + getRequest("type=ACTIVITY_STEPS&groupBy=day&limit=25&offset=25"), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { + data: { measurements: unknown[]; meta: { total: number } }; }; - expect(call.orderBy).toEqual({ measuredAt: "desc" }); + expect(json.data.measurements).toEqual([]); + expect(json.data.meta.total).toBe(3); + expect(rawCallsWith("SUM(")).toHaveLength(0); + // Past the end, and only there, the total is counted on its own. + expect(rawCallsWith("COUNT(DISTINCT")).toHaveLength(1); }); // v1.29.6 — a day with both an Apple Health AND a Withings write for the @@ -201,28 +240,10 @@ describe("GET /api/measurements — groupBy=day (W7c collapsed list)", () => { // Apple-Health steps + 8800 Withings steps summed to a fabricated 17800 — // a number the tile never showed. it("collapses cross-source rows to the ladder-canonical source before summing", async () => { - vi.mocked(prisma.measurement.findMany).mockResolvedValue([ - { - id: "m-apple", - type: "ACTIVITY_STEPS", - value: 9000, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-15T08:00:00.000Z"), - notes: null, - deviceType: null, - }, - { - id: "m-withings", - type: "ACTIVITY_STEPS", - value: 8800, - unit: "steps", - source: "WITHINGS", - measuredAt: new Date("2026-05-15T09:00:00.000Z"), - notes: null, - deviceType: null, - }, - ] as never); + mockDayPage(["2026-05-15"], 1, [ + group("2026-05-15", 8800, 1, { source: "WITHINGS" }), + group("2026-05-15", 9000, 1), + ]); const res = await GET( getRequest( @@ -251,97 +272,54 @@ describe("GET /api/measurements — groupBy=day (W7c collapsed list)", () => { expect(json.data.meta.droppedDuplicates).toBe(1); }); - // v1.30.3 (QA F6) — a capped read's oldest bucket sits mid-day: every - // sample on that calendar day BEFORE the cutoff row was excluded, so the - // bucket's SUM understates the real day total. The row must carry a - // `partial: true` marker instead of rendering as if it were complete. - it("flags the oldest bucket partial when the raw-row read hits the cap", async () => { - // limit=3: the DESC-ordered read returns exactly 3 rows (the cap), so - // the oldest of the three (2026-05-14) is a truncated day — its bucket - // could be missing earlier same-day samples the cap cut off. - vi.mocked(prisma.measurement.findMany).mockResolvedValue([ - { - id: "m-1", - type: "ACTIVITY_STEPS", - value: 3400, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-15T11:00:00.000Z"), - notes: null, - }, - { - id: "m-2", - type: "ACTIVITY_STEPS", - value: 2500, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-15T07:00:00.000Z"), - notes: null, - }, - { - id: "m-3", - type: "ACTIVITY_STEPS", - value: 800, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-14T16:00:00.000Z"), - notes: null, - }, - ] as never); + // #1245 — `limit` / `offset` page over days and reach the SQL that picks + // them; `meta.total` is the distinct-day count, not the page length, and + // no row is marked partial because every page day is read whole. + it("pages over days and reports the distinct-day total", async () => { + mockDayPage(["2026-04-20"], 40, [group("2026-04-20", 3400, 1)]); const res = await GET( - getRequest("type=ACTIVITY_STEPS&groupBy=day&limit=3"), + getRequest("type=ACTIVITY_STEPS&groupBy=day&limit=25&offset=25"), ); expect(res.status).toBe(200); - const json = (await res.json()) as { - data: { - measurements: Array<{ - dayKey: string; - value: number; - partial?: boolean; - }>; - meta: { capped?: boolean }; - }; - }; - const day15 = json.data.measurements.find((m) => m.dayKey === "2026-05-15"); - const day14 = json.data.measurements.find((m) => m.dayKey === "2026-05-14"); - expect(day15?.partial).toBeUndefined(); - expect(day14?.partial).toBe(true); - }); - - it("flags no bucket partial when the read comes in under the cap (whole window covered)", async () => { - vi.mocked(prisma.measurement.findMany).mockResolvedValue([ - { - id: "m-1", - type: "ACTIVITY_STEPS", - value: 3400, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-15T11:00:00.000Z"), - notes: null, - }, - { - id: "m-3", - type: "ACTIVITY_STEPS", - value: 800, - unit: "steps", - source: "APPLE_HEALTH", - measuredAt: new Date("2026-05-14T16:00:00.000Z"), - notes: null, - }, - ] as never); - - // Default limit (100) is nowhere near hit by 2 rows. - const res = await GET(getRequest("type=ACTIVITY_STEPS&groupBy=day")); - expect(res.status).toBe(200); const json = (await res.json()) as { data: { measurements: Array<{ dayKey: string; partial?: boolean }>; + meta: { total: number; limit: number; offset: number }; }; }; - expect(json.data.measurements.every((m) => m.partial === undefined)).toBe( - true, - ); + expect(json.data.meta).toMatchObject({ total: 40, limit: 25, offset: 25 }); + expect(json.data.measurements.map((m) => m.dayKey)).toEqual(["2026-04-20"]); + expect(json.data.measurements[0].partial).toBeUndefined(); + const [pageCall] = rawCallsWith("COUNT(*) OVER ()"); + expect(pageCall?.slice(1)).toEqual(expect.arrayContaining([25])); + expect(pageCall?.slice(1).filter((v) => v === 25)).toHaveLength(2); + // The total rode on the page's own rows; there was no second pass over + // the history, only the fold of the page's one day. + expect(prisma.$queryRaw).toHaveBeenCalledTimes(2); + expect(rawCallsWith("COUNT(DISTINCT")).toHaveLength(0); + }); + + // A device split inside the winning source keeps the top device's + // groups only, the same narrowing the picker applies to raw rows. + it("keeps the top-ranked device's groups within the winning source", async () => { + mockDayPage(["2026-05-15"], 1, [ + group("2026-05-15", 6000, 40, { device_type: "watch" }), + group("2026-05-15", 5500, 12, { device_type: "phone" }), + ]); + const res = await GET(getRequest("type=ACTIVITY_STEPS&groupBy=day")); + const json = (await res.json()) as { + data: { + measurements: Array<{ value: number; sampleCount: number }>; + meta: { droppedDuplicates: number }; + }; + }; + expect(json.data.measurements).toHaveLength(1); + expect(json.data.measurements[0]).toMatchObject({ + value: 6000, + sampleCount: 40, + }); + expect(json.data.meta.droppedDuplicates).toBe(12); }); it("falls back to the legacy findMany path when groupBy=day is set on a non-cumulative type", async () => { @@ -480,17 +458,14 @@ describe("GET /api/measurements — dayKey drill-down (W7c)", () => { }); describe("GET /api/measurements — schema rejections (W10 reconcile)", () => { - // v1.4.37 W10 H1 — the groupBy=day collapsed path runs the collapse - // AFTER the per-sample scan, so it can't honour a non-zero offset - // without a Postgres-side date_trunc grouping that's a v1.4.38 - // backlog item. The validator must reject the combination instead - // of silently rendering "showing 1-25 of N" with the wrong N. - it("rejects offset>0 when groupBy=day is set", async () => { + // #1245 — the day-grouped mode pages over days, so an offset is a real + // page request now rather than a refused combination. + it("accepts offset>0 when groupBy=day is set", async () => { + mockDayPage([], 0); const res = await GET( getRequest("type=ACTIVITY_STEPS&groupBy=day&offset=25"), ); - expect(res.status).toBe(422); - expect(prisma.measurement.findMany).not.toHaveBeenCalled(); + expect(res.status).toBe(200); }); // v1.4.37 W10 H1 — same restriction for the drill-down branch; it diff --git a/src/app/api/measurements/route.ts b/src/app/api/measurements/route.ts index fa46fa1f7..6939d4777 100644 --- a/src/app/api/measurements/route.ts +++ b/src/app/api/measurements/route.ts @@ -53,6 +53,7 @@ import { import { reconstructSleepSessions, pickMainNightAndNaps, + SESSION_GAP_MS, } from "@/lib/analytics/sleep-night"; import { resolveUserTimezone } from "@/lib/tz/resolver"; import { @@ -71,12 +72,14 @@ import { oppositeMergeSource, } from "@/lib/measurements/cross-source-merge"; import { NextRequest } from "next/server"; -import type { - MeasurementType, - MeasurementSource, - GlucoseContext, +import { + Prisma, + type MeasurementType, + type MeasurementSource, + type GlucoseContext, + type SleepStage, } from "@/generated/prisma/client"; -import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; +import { DEFAULT_TIMEZONE, isValidTimezone } from "@/lib/tz/format"; export const GET = apiHandler(async (request: NextRequest) => { const { user } = await requireRecordAuth("read", "measurements"); @@ -234,161 +237,23 @@ export const GET = apiHandler(async (request: NextRequest) => { // v1.4.37 W7c — collapsed "one row per day" mode for cumulative // types. The list page renders the day's SUM as a single synthesised // row with a `sampleCount` chevron that drills back into the - // per-sample rows via the `dayKey=…` branch above. + // per-sample rows via the `dayKey=…` branch above. `limit` / `offset` + // page over DAYS, not raw rows (#1245). if ( groupBy === "day" && type && CUMULATIVE_HK_TYPES.has(type as MeasurementType) ) { - const tz = - user.timezone && user.timezone.length > 0 - ? user.timezone - : DEFAULT_TIMEZONE; - // v1.29.6 — DESC + take so a long history returns the MOST RECENT - // window of raw samples for bucketing. The previous `orderBy: asc` - // + `take: limit` sliced the OLDEST page of rows before the - // day-bucketing loop ran below — on an account with more than - // `limit` cumulative samples in its history, the buckets were built - // entirely from year-old rows and the recent days silently never - // appeared. Mirrors the workouts-list ordering fix (05f80ad37): - // re-establish "most recent" as the page the caller actually gets. - const rows = await prisma.measurement.findMany({ - where, - orderBy: { measuredAt: "desc" }, - // Per-sample row scan within the requested window — bounded by - // the existing `limit` cap (default 100, max 5000) so a wide - // multi-year window can still walk the whole range when the - // chart explicitly asks for it. - take: limit, - select: { - id: true, - type: true, - value: true, - unit: true, - source: true, - measuredAt: true, - notes: true, - deviceType: true, - }, - }); - - // v1.29.6 — collapse overlapping sources to the ladder-canonical - // reading per day BEFORE summing. Without this, a day with both an - // Apple Health AND a Withings write for the same cumulative metric - // (e.g. steps) summed BOTH sources into one inflated total — a day - // the dashboard tile (which reads through the source-collapsed - // rollup, `collapseRollupRowsBySource` in - // `src/lib/rollups/measurement-read.ts`) showed correctly. Uses the - // same `metricKeyForType` ladder lookup the rollup collapse uses, so - // this list and the tile agree on which source wins. A type with no - // ladder (e.g. TIME_IN_DAYLIGHT, FALL_COUNT) has nothing to collapse - // against — every row passes through unchanged, matching the - // pass-through fallback used at the other `pickCanonicalSourceRows` - // call sites. - const metricKey = metricKeyForType(type as MeasurementType); - const priorityJson = await loadUserSourcePriority(user.id); - const canonicalRows = metricKey - ? pickCanonicalSourceRows( - rows.map((r) => ({ ...r, type: r.type as MeasurementType })), - metricKey, - priorityJson, - (d) => dayKeyForUserTz(d, tz), - ).canonicalRows - : rows; - - // Group by the user's calendar day, sum values, keep a representative - // unit/source from the bucket's first row so the rendered row carries - // a sensible label even when the bucket mixes sources. - type Bucket = { - dayKey: string; - total: number; - sampleCount: number; - latest: Date; - unit: string; - source: string; - }; - const buckets = new Map(); - for (const row of canonicalRows) { - const key = dayKeyForUserTz(row.measuredAt, tz); - const slot = buckets.get(key); - if (!slot) { - buckets.set(key, { - dayKey: key, - total: row.value, - sampleCount: 1, - latest: row.measuredAt, - unit: row.unit, - source: row.source, - }); - } else { - slot.total += row.value; - slot.sampleCount += 1; - if (row.measuredAt > slot.latest) slot.latest = row.measuredAt; - } - } - - // v1.30.3 (QA F6) — the read above is DESC + take(limit), so once a - // dense account's in-window row count crosses the cap, the OLDEST - // returned row sits mid-day: every earlier sample on that same - // calendar day was cut off, and its bucket's SUM understates the real - // day total with no marker. Flag exactly that one bucket — the day - // containing the cutoff row (the minimum dayKey among the buckets - // actually built) — `partial: true` only when the cap was truly hit; - // a read that came in under the cap covers its whole window and no - // day is a truncation artifact. - const cappedRead = rows.length === limit; - let oldestDayKey: string | null = null; - if (cappedRead) { - for (const key of buckets.keys()) { - if (oldestDayKey === null || key < oldestDayKey) oldestDayKey = key; - } - } - - const measurements = Array.from(buckets.values()) - .map((b) => ({ - // Synthetic id so the list-row key stays stable across pages - // and the row never collides with a real `Measurement.id`. - id: `day:${type}:${b.dayKey}`, - type, - value: b.total, - unit: b.unit, - source: b.source, - // Anchor the row's timestamp to local noon so the row sorts - // cleanly between same-day spot samples — matches the - // `canonicalDailyTimestamp` convention used by the drain. - measuredAt: canonicalDailyTimestamp(b.dayKey, tz).toISOString(), - notes: null, - dayKey: b.dayKey, - sampleCount: b.sampleCount, - ...(b.dayKey === oldestDayKey ? { partial: true } : {}), - })) - .sort((a, b) => { - const cmp = a.dayKey < b.dayKey ? -1 : a.dayKey > b.dayKey ? 1 : 0; - return sortDir === "asc" ? cmp : -cmp; - }); - - annotate({ - action: { name: "measurement.list" }, - meta: { - total: measurements.length, - type, - groupBy: "day", - rowsScanned: rows.length, - droppedDuplicates: rows.length - canonicalRows.length, - // QA F6 — surfaces when the oldest bucket in this page is a - // partial-day sum (the read hit the cap). - capped: cappedRead, - }, - }); - return apiSuccess({ - measurements, - meta: { - total: measurements.length, - limit, - offset: 0, - groupBy: "day", - droppedDuplicates: rows.length - canonicalRows.length, - }, + return dayGroupedListResponse(user, { + type: type as MeasurementType, + limit, + offset, + sortDir, + sourceEq, + valueMin, + valueMax, + from, + to, }); } @@ -506,6 +371,237 @@ export const GET = apiHandler(async (request: NextRequest) => { }); }); +/** + * The day-grouped list for a cumulative type, paged over local days. + * + * Everything heavy runs in Postgres. The first query picks the page: the + * distinct local days that hold a reading, sorted and cut by `limit` / + * `offset`, each row carrying the distinct-day count of the whole filtered + * set as `meta.total` from a window count. The second folds only the page's + * days, one row per day, source, device and unit with its sum and sample + * count, and the source ladder picks among those groups exactly as it would + * among the raw rows (it keeps or drops whole source and device groups per + * day, never single rows). So a page costs its own days whatever the length + * of the history, every day is complete, and no raw row reaches the heap. + * + * Until v1.43.6 the branch read the newest `limit` raw rows (the web asked + * for 5000), bucketed them and returned every bucket. A long history came + * back as thousands of rows in one response, and every day older than the + * 5000th raw row was missing with nothing to say so. + * + * A day exists for the page when any reading of the type falls on it; the + * source collapse narrows a day's groups but never empties one, so the count + * and the folded rows agree. + */ +async function dayGroupedListResponse( + user: { id: string; timezone?: string | null }, + opts: { + type: MeasurementType; + limit: number; + offset: number; + sortDir: "asc" | "desc"; + sourceEq?: MeasurementSource; + valueMin?: number; + valueMax?: number; + from?: Date; + to?: Date; + }, +) { + const { type, limit, offset, sortDir } = opts; + const stored = + user.timezone && user.timezone.length > 0 + ? user.timezone + : DEFAULT_TIMEZONE; + // A zone Postgres does not know would 500 the read; the drill-down cuts + // days in the same zone, so both must agree on it. + const tz = isValidTimezone(stored) ? stored : DEFAULT_TIMEZONE; + + // The request's filters, as SQL: the same ones the plain list applies. + const filters = Prisma.sql` + m."user_id" = ${user.id} + AND m."type" = ${type}::measurement_type + AND m."deleted_at" IS NULL + ${opts.sourceEq ? Prisma.sql`AND m."source" = ${opts.sourceEq}::measurement_source` : Prisma.empty} + ${opts.valueMin != null ? Prisma.sql`AND m."value" >= ${opts.valueMin}` : Prisma.empty} + ${opts.valueMax != null ? Prisma.sql`AND m."value" <= ${opts.valueMax}` : Prisma.empty} + ${opts.from ? Prisma.sql`AND m."measured_at" >= ${opts.from}` : Prisma.empty} + ${opts.to ? Prisma.sql`AND m."measured_at" <= ${opts.to}` : Prisma.empty} + `; + const localDay = Prisma.sql`to_char((m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD')`; + // Direction comes from the Zod enum, never from raw input. The window + // count runs before LIMIT, so every row of the page carries the + // distinct-day total of the whole filtered set. + const direction = Prisma.raw(sortDir === "asc" ? "ASC" : "DESC"); + const pageDays = await prisma.$queryRaw< + Array<{ day_key: string; total: number }> + >` + SELECT d.day_key, (COUNT(*) OVER ())::int AS total + FROM ( + SELECT ${localDay} AS day_key + FROM measurements m + WHERE ${filters} + GROUP BY 1 + ) d + ORDER BY d.day_key ${direction} + LIMIT ${limit} OFFSET ${offset} + `; + // A page past the end has no row to carry the total; only then is it + // counted on its own. + const total = + pageDays.length > 0 + ? Number(pageDays[0].total) + : offset > 0 + ? Number( + ( + await prisma.$queryRaw>` + SELECT COUNT(DISTINCT ${localDay})::int AS total + FROM measurements m + WHERE ${filters} + ` + )[0]?.total ?? 0, + ) + : 0; + const dayKeys = pageDays.map((d) => d.day_key); + + // The page's days are consecutive in the sorted day set, so the span from + // the first one's start to the last one's end holds exactly their rows. + let groups: Array<{ + day_key: string; + source: MeasurementSource; + device_type: string | null; + unit: string; + total: number; + n: number; + }> = []; + if (dayKeys.length > 0) { + const sorted = [...dayKeys].sort(); + const spanStart = localDayWindow(sorted[0], tz).dayStart; + const spanEnd = localDayWindow(sorted[sorted.length - 1], tz).dayEnd; + // Newest group first within a day, so the day's label comes from its + // latest reading, as it did when raw rows were read newest first. + groups = await prisma.$queryRaw` + SELECT + ${localDay} AS day_key, + m."source"::text AS source, + m."device_type" AS device_type, + m."unit" AS unit, + SUM(m."value")::float8 AS total, + COUNT(*)::int AS n + FROM measurements m + WHERE ${filters} + AND m."measured_at" >= ${spanStart} + AND m."measured_at" < ${spanEnd} + GROUP BY 1, 2, 3, 4 + ORDER BY 1, MAX(m."measured_at") DESC, 2, 3, 4 + `; + } + + // v1.29.6 — collapse overlapping sources to the ladder-canonical + // reading per day BEFORE summing. Without this, a day with both an + // Apple Health AND a Withings write for the same cumulative metric + // (e.g. steps) summed BOTH sources into one inflated total — a day + // the dashboard tile (which reads through the source-collapsed + // rollup, `collapseRollupRowsBySource` in + // `src/lib/rollups/measurement-read.ts`) showed correctly. Uses the + // same `metricKeyForType` ladder lookup the rollup collapse uses, so + // this list and the tile agree on which source wins. A type with no + // ladder (e.g. TIME_IN_DAYLIGHT, FALL_COUNT) has nothing to collapse + // against — every group passes through unchanged. Each group stands at + // its day's local noon, so the picker files it under the day SQL cut. + const groupRows = groups.map((g) => ({ + ...g, + type, + deviceType: g.device_type, + measuredAt: canonicalDailyTimestamp(g.day_key, tz), + })); + const metricKey = metricKeyForType(type); + const priorityJson = metricKey ? await loadUserSourcePriority(user.id) : null; + const canonicalGroups = metricKey + ? pickCanonicalSourceRows(groupRows, metricKey, priorityJson, (d) => + dayKeyForUserTz(d, tz), + ).canonicalRows + : groupRows; + + // Sum per local day; unit and source come from the day's first kept group + // so the rendered row carries a sensible label even when sources mix. + const buckets = new Map< + string, + { total: number; sampleCount: number; unit: string; source: string } + >(); + for (const g of canonicalGroups) { + const slot = buckets.get(g.day_key); + if (!slot) { + buckets.set(g.day_key, { + total: g.total, + sampleCount: g.n, + unit: g.unit, + source: g.source, + }); + } else { + slot.total += g.total; + slot.sampleCount += g.n; + } + } + const rowsScanned = groups.reduce((sum, g) => sum + g.n, 0); + const rowsKept = canonicalGroups.reduce((sum, g) => sum + g.n, 0); + + // Rows follow the page's day order from SQL. + const measurements = dayKeys.flatMap((dayKey) => { + const b = buckets.get(dayKey); + if (!b) return []; + return [ + { + // Synthetic id so the list-row key stays stable across pages + // and the row never collides with a real `Measurement.id`. + id: `day:${type}:${dayKey}`, + type, + value: b.total, + unit: b.unit, + source: b.source, + // Anchor the row's timestamp to local noon so the row sorts + // cleanly between same-day spot samples — matches the + // `canonicalDailyTimestamp` convention used by the drain. + measuredAt: canonicalDailyTimestamp(dayKey, tz).toISOString(), + notes: null, + dayKey, + sampleCount: b.sampleCount, + }, + ]; + }); + + annotate({ + action: { name: "measurement.list" }, + meta: { + total, + type, + groupBy: "day", + days: measurements.length, + rowsScanned, + droppedDuplicates: rowsScanned - rowsKept, + }, + }); + return apiSuccess({ + measurements, + meta: { + total, + limit, + offset, + groupBy: "day", + droppedDuplicates: rowsScanned - rowsKept, + }, + }); +} + +/** + * A sleep page holds at most this many nights. The nights' stage rows are + * read and reconstructed in the process, so the bound sits on the nights; + * the applied value comes back as `meta.limit`, and `meta.total` still counts + * every night, so a caller asking for more pages on rather than losing any. + * Until v1.43.6 the list read no more than a year of nights however large + * the limit, so this is a widening, never a new cut. + */ +const SLEEP_NIGHT_PAGE_MAX = 1000; + /** * v1.11.5 — per-night collapse for the SLEEP_DURATION list. Sleep is * stored one row per STAGE per night (minutes); a plain list leaks every @@ -521,11 +617,11 @@ export const GET = apiHandler(async (request: NextRequest) => { * nap count, and awakenings in the row so the UI can caption the night * without a second fetch. * - * The read is bounded to the last year (matching the series route's sleep - * read) so an unbounded list never walks a multi-year per-stage row set. + * The nights are paged in Postgres (`sleepNightPage`), like the days of the + * cumulative list, and only the stage rows of the page's nights are read. + * Until v1.43.6 the read stopped at a year before `to` and the list ended + * there without saying so to the client. */ -const SLEEP_LIST_MAX_DAYS = 365; - async function sleepListResponse( user: { id: string }, opts: { @@ -537,53 +633,74 @@ async function sleepListResponse( dayKey?: string; }, ) { - const { from, to, limit, offset, sortDir, dayKey } = opts; + const { from, to, offset, sortDir, dayKey } = opts; + const limit = Math.min(opts.limit, SLEEP_NIGHT_PAGE_MAX); const [tz, priorityJson] = await Promise.all([ resolveUserTimezone(user.id), loadUserSourcePriority(user.id), ]); - // Bound the read window to a year ending at `to` (or now). An explicit - // `from` narrows it but never widens it: an early `from` used to walk every - // per-stage row back to that date, which on a multi-year, multi-source - // account is the unbounded read this cap exists to stop. Older nights are - // reached by moving `to` back, and `meta.windowFrom` says where a cut - // window started so a client can tell a cut from an empty history. - const windowEnd = to ?? new Date(); - const floor = new Date( - windowEnd.getTime() - SLEEP_LIST_MAX_DAYS * 86_400_000, - ); - const since = from && from > floor ? from : floor; - const windowCut = from !== undefined && from < floor; - const rows = await prisma.measurement.findMany({ - where: { - userId: user.id, - type: "SLEEP_DURATION", - deletedAt: null, - measuredAt: { - gte: since, - ...(to ? { lte: to } : {}), - }, - }, - orderBy: { measuredAt: "asc" }, - select: { - id: true, - value: true, - measuredAt: true, - sleepStage: true, - source: true, - // Writer-level collapse: two HealthKit apps behind one source (watch - // stages vs phone in-bed) must not blend into one night. - deviceType: true, - }, + const page = await sleepNightPage(user.id, tz, { + from, + to, + sortDir, + ...(dayKey ? { night: dayKey, limit: 1, offset: 0 } : { limit, offset }), }); + const nightKeys = page.nights.map((n) => n.night); + const wanted = new Set(nightKeys); + + // The page's sessions lie between the first one's start and the last one's + // end, and every other session is more than the session gap away from that + // span, so this read holds the page's sessions whole and no part of any + // other. + let rows: Array<{ + id: string; + value: number; + measuredAt: Date; + sleepStage: SleepStage | null; + source: MeasurementSource; + deviceType: string | null; + }> = []; + if (page.nights.length > 0) { + let spanStart = page.nights[0].spanStart; + let spanEnd = page.nights[0].spanEnd; + for (const n of page.nights) { + if (n.spanStart < spanStart) spanStart = n.spanStart; + if (n.spanEnd > spanEnd) spanEnd = n.spanEnd; + } + rows = await prisma.measurement.findMany({ + where: { + userId: user.id, + type: "SLEEP_DURATION", + deletedAt: null, + AND: [ + { measuredAt: { gte: spanStart, lte: spanEnd } }, + ...(from ? [{ measuredAt: { gte: from } }] : []), + ...(to ? [{ measuredAt: { lte: to } }] : []), + ], + }, + orderBy: [{ measuredAt: "asc" }, { id: "asc" }], + select: { + id: true, + value: true, + measuredAt: true, + sleepStage: true, + source: true, + // Writer-level collapse: two HealthKit apps behind one source (watch + // stages vs phone in-bed) must not blend into one night. + deviceType: true, + }, + }); + } + const sessions = reconstructSleepSessions(rows, tz, priorityJson).filter( + (s) => wanted.has(s.night), + ); // Drill-down — return the requested night's canonical stage segments. if (dayKey) { - const sessions = reconstructSleepSessions(rows, tz, priorityJson).filter( - (s) => s.night === dayKey, + const { main, naps } = pickMainNightAndNaps( + sessions.filter((s) => s.night === dayKey), ); - const { main, naps } = pickMainNightAndNaps(sessions); const dayUnit = getUnitForType("SLEEP_DURATION"); // One row per stage segment of the main night + each nap, so the // drill-down reveals exactly the rows that summed into the headline. @@ -624,9 +741,7 @@ async function sleepListResponse( // Per-night collapse — one synthetic row per wake day. The headline // value is the MAIN night's TIME ASLEEP (the longest session that day), // per the nap convention: naps are surfaced as a count, never folded - // into the main night's headline. Reconstruct sessions and pick the - // main + naps per wake day. - const sessions = reconstructSleepSessions(rows, tz, priorityJson); + // into the main night's headline. const byDay = new Map(); for (const s of sessions) { const list = byDay.get(s.night) ?? []; @@ -635,80 +750,165 @@ async function sleepListResponse( } const dayUnit = getUnitForType("SLEEP_DURATION"); - const nightRows: Array<{ - id: string; - type: string; - value: number; - unit: string; - source: string; - measuredAt: string; - notes: null; - dayKey: string; - sampleCount: number; - sleepStages: Record; - inBedMinutes: number | null; - awakeMinutes: number | null; - napCount: number; - napAsleepMinutes: number; - awakenings: number; - }> = []; - for (const [day, daySessions] of byDay) { - const { main, naps } = pickMainNightAndNaps(daySessions); - if (!main) continue; - nightRows.push({ - // Synthetic id mirrors the series route's `sleep:` shape. - id: `sleep:${day}`, - type: "SLEEP_DURATION", - value: Math.round(main.asleepMinutes), - unit: dayUnit, - source: main.source ?? "MANUAL", - measuredAt: main.end.toISOString(), - notes: null, - dayKey: day, - // Drives the drill-down chevron — number of stage segments behind - // the night, matching the cumulative-type `sampleCount` contract. - sampleCount: main.segments.length, - sleepStages: main.stages as Record, - inBedMinutes: main.inBedMinutes, - awakeMinutes: main.awakeMinutes, - napCount: naps.length, - napAsleepMinutes: naps.reduce((sum, n) => sum + n.asleepMinutes, 0), - awakenings: main.awakenings, - }); - } - - // Honour offset + limit against the full night set, and report the true - // pre-slice night count as `meta.total` — matching the non-sleep list - // branch's pagination contract so a client can page through nights. - const totalNights = nightRows.length; - const measurements = nightRows - .sort((a, b) => { - const cmp = a.dayKey < b.dayKey ? -1 : a.dayKey > b.dayKey ? 1 : 0; - return sortDir === "asc" ? cmp : -cmp; - }) - .slice(offset, offset + limit); + // Rows follow the page's night order from SQL. + const measurements = nightKeys.flatMap((day) => { + const { main, naps } = pickMainNightAndNaps(byDay.get(day) ?? []); + if (!main) return []; + return [ + { + // Synthetic id mirrors the series route's `sleep:` shape. + id: `sleep:${day}`, + type: "SLEEP_DURATION", + value: Math.round(main.asleepMinutes), + unit: dayUnit, + source: main.source ?? "MANUAL", + measuredAt: main.end.toISOString(), + notes: null, + dayKey: day, + // Drives the drill-down chevron — number of stage segments behind + // the night, matching the cumulative-type `sampleCount` contract. + sampleCount: main.segments.length, + sleepStages: main.stages as Record, + inBedMinutes: main.inBedMinutes, + awakeMinutes: main.awakeMinutes, + napCount: naps.length, + napAsleepMinutes: naps.reduce((sum, n) => sum + n.asleepMinutes, 0), + awakenings: main.awakenings, + }, + ]; + }); annotate({ action: { name: "measurement.list" }, meta: { - total: totalNights, + total: page.total, type: "SLEEP_DURATION", mode: "sleep-per-night", + nights: measurements.length, rowsScanned: rows.length, }, }); return apiSuccess({ measurements, meta: { - total: totalNights, + total: page.total, limit, offset, groupBy: "night", - ...(windowCut ? { windowFrom: since.toISOString() } : {}), }, }); } +/** + * One page of sleep nights, chosen in Postgres. + * + * The stage rows are cut into sessions the way `reconstructSleepSessions` + * cuts them: ordered by start (`measured_at` minus the minutes), a new + * session wherever a row starts more than `SESSION_GAP_MS` after every + * earlier row has ended. A session counts when it holds an asleep row (a + * bare or granular asleep stage, or a stage-less one, with minutes), and its + * night is the local day its last row ends on. Nights are grouped, sorted + * and cut by `limit` / `offset`; `total` is the night count of the whole + * filtered set from the same query. `spanStart` / `spanEnd` bound the + * night's counting sessions, so the caller reads exactly their rows. + * + * The session's night here is taken over every writer, while the JS fold + * takes it over the winning writer only. They part only when a losing writer + * runs past midnight and the winner does not; that night then sits on the + * page and comes back without a row, and `total` counts it. + */ +async function sleepNightPage( + userId: string, + tz: string, + opts: { + from?: Date; + to?: Date; + sortDir: "asc" | "desc"; + limit: number; + offset: number; + night?: string; + }, +): Promise<{ + nights: Array<{ night: string; spanStart: Date; spanEnd: Date }>; + total: number; +}> { + const zone = isValidTimezone(tz) ? tz : DEFAULT_TIMEZONE; + const gapSeconds = SESSION_GAP_MS / 1000; + // Direction comes from the Zod enum, never from raw input. + const direction = Prisma.raw(opts.sortDir === "asc" ? "ASC" : "DESC"); + const nightsCte = Prisma.sql` + WITH r AS ( + SELECT + m."measured_at" AS e, + m."measured_at" - m."value" * interval '1 minute' AS s, + (m."value" > 0 AND (m."sleep_stage" IS NULL + OR m."sleep_stage" IN ('ASLEEP', 'REM', 'CORE', 'DEEP'))) AS asleep + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."type" = 'SLEEP_DURATION'::measurement_type + AND m."deleted_at" IS NULL + ${opts.from ? Prisma.sql`AND m."measured_at" >= ${opts.from}` : Prisma.empty} + ${opts.to ? Prisma.sql`AND m."measured_at" <= ${opts.to}` : Prisma.empty} + ), o AS ( + SELECT r.*, MAX(e) OVER ( + ORDER BY s, e ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING + ) AS prev_end + FROM r + ), g AS ( + SELECT o.*, SUM( + CASE WHEN prev_end IS NULL + OR s - prev_end > make_interval(secs => ${gapSeconds}::double precision) THEN 1 ELSE 0 END + ) OVER (ORDER BY s, e ROWS UNBOUNDED PRECEDING) AS sid + FROM o + ), sess AS ( + SELECT MIN(s) AS s, MAX(e) AS e + FROM g + GROUP BY sid + HAVING BOOL_OR(asleep) + ), nights AS ( + SELECT + to_char((e AT TIME ZONE 'UTC') AT TIME ZONE ${zone}, 'YYYY-MM-DD') AS night, + MIN(s) AS span_start, + MAX(e) AS span_end + FROM sess + GROUP BY 1 + ) + `; + const rows = await prisma.$queryRaw< + Array<{ night: string; span_start: Date; span_end: Date; total: number }> + >` + ${nightsCte} + SELECT night, span_start, span_end, (COUNT(*) OVER ())::int AS total + FROM nights + ${opts.night ? Prisma.sql`WHERE night = ${opts.night}` : Prisma.empty} + ORDER BY night ${direction} + LIMIT ${opts.limit} OFFSET ${opts.offset} + `; + // A page past the end has no row to carry the total; only then is it + // counted on its own. + const total = + rows.length > 0 + ? Number(rows[0].total) + : opts.offset > 0 + ? Number( + ( + await prisma.$queryRaw>` + ${nightsCte} + SELECT COUNT(*)::int AS total FROM nights + ` + )[0]?.total ?? 0, + ) + : 0; + return { + nights: rows.map((r) => ({ + night: r.night, + spanStart: r.span_start, + spanEnd: r.span_end, + })), + total, + }; +} + /** * Does this body attribute a source to any reading — any source at all? * diff --git a/src/components/measurements/measurement-list.tsx b/src/components/measurements/measurement-list.tsx index fe260d0dc..9f0f8ba91 100644 --- a/src/components/measurements/measurement-list.tsx +++ b/src/components/measurements/measurement-list.tsx @@ -581,7 +581,7 @@ export function MeasurementList({ // v1.11.5 — SLEEP_DURATION is collapsed server-side to one row per // night (TIME ASLEEP), so the list paints per-night rows with a // chevron drilling into the night's stage segments. Like the - // cumulative branch it returns a single page of synthesised rows. + // cumulative branch it pages server-side over its synthesised rows. const isSleepFilter = typeFilter === "SLEEP_DURATION"; const isDayGroupedFilter = isCumulativeFilter || isSleepFilter; @@ -634,30 +634,11 @@ export function MeasurementList({ params.set("offset", String((page - 1) * PAGE_SIZE)); params.set("sortBy", sortBy); params.set("sortDir", sortDir); - if (isCumulativeFilter) { - params.set("groupBy", "day"); - // The collapsed mode synthesises one row per day from the - // window of raw samples. Use the server-side ceiling (5000) - // so a chatty pre-drain Apple-Watch account (≈200 per-sample - // chunks/day on the busiest days) still fills several screens - // of daily rows before the nightly drain reduces each day to - // a single `stats:` row. Post-drain the scan is trivially - // short — every day reduces to 1 row. - params.set("limit", "5000"); - // The synthesised rows live entirely in the collapse branch - // — pagination/offset don't apply on the server side. Reset - // the offset so a sort-direction flip doesn't paginate into - // an empty slice. - params.set("offset", "0"); - } else if (isSleepFilter) { - // v1.11.5 — the route auto-collapses SLEEP_DURATION to one row - // per night (no `groupBy` param needed). Request a wide limit - // so the per-night list spans the trailing year the route - // bounds the read to; offset doesn't apply to the synthesised - // rows. - params.set("limit", "366"); - params.set("offset", "0"); - } + // The day-grouped modes page over days, the same 25 per page as + // the per-sample list: `groupBy=day` folds each cumulative day to + // its sum, and SLEEP_DURATION collapses to one row per night + // without a param. + if (isCumulativeFilter) params.set("groupBy", "day"); return apiGet<{ measurements: Measurement[]; meta: { total: number }; @@ -839,14 +820,9 @@ export function MeasurementList({ blocked: updateMutation.isPending || deleteMutation.isPending, }); - // v1.4.37 W7c — the collapsed daily view paints up to 5000 - // synthesised rows in one shot (one row per day for the - // configured window). Pagination would force a second server - // round-trip that re-fires the same scan, so the day-grouped - // path collapses to a single page. Per-sample lists continue to - // paginate at PAGE_SIZE = 25. - const totalPages = - data && !isDayGroupedFilter ? Math.ceil(data.meta.total / PAGE_SIZE) : 0; + // Every mode pages server-side at PAGE_SIZE: the per-sample list over + // rows, the day-grouped modes over days or nights (#1245). + const totalPages = data ? Math.ceil(data.meta.total / PAGE_SIZE) : 0; function startEdit(measurement: Measurement) { if (onEdit) { diff --git a/src/lib/analytics/sleep-night.ts b/src/lib/analytics/sleep-night.ts index cbc3dc4a1..ad1587147 100644 --- a/src/lib/analytics/sleep-night.ts +++ b/src/lib/analytics/sleep-night.ts @@ -252,9 +252,11 @@ function inBedEnvelopeMinutes(rows: readonly SleepStageRow[]): number | null { * Gap (ms) between consecutive stage segments that starts a new sleep * session. Stage segments within one night are contiguous (minutes to a * couple of hours apart); a > 3 h gap separates a nap from the overnight - * block, or two genuine sessions in one day. + * block, or two genuine sessions in one day. The sleep list pages its + * nights in SQL with the same cut (`sleepNightPage` in the measurements + * list route), so this is the one value both read. */ -const SESSION_GAP_MS = 3 * 60 * 60 * 1000; +export const SESSION_GAP_MS = 3 * 60 * 60 * 1000; export interface SleepStageRow { value: number; diff --git a/src/lib/openapi/__tests__/measurement-contract-drift.test.ts b/src/lib/openapi/__tests__/measurement-contract-drift.test.ts index b48088dc4..c55966411 100644 --- a/src/lib/openapi/__tests__/measurement-contract-drift.test.ts +++ b/src/lib/openapi/__tests__/measurement-contract-drift.test.ts @@ -103,7 +103,7 @@ describe("GET /api/measurements — documented response matches the handler", () expect(listResponse.safeParse(stale).success).toBe(false); }); - it("accepts the collapsed day-sum mode (dayKey / sampleCount / partial)", () => { + it("accepts the collapsed day-sum mode (dayKey / sampleCount)", () => { const wire = { data: { measurements: [ @@ -117,7 +117,6 @@ describe("GET /api/measurements — documented response matches the handler", () notes: null, dayKey: "2026-07-18", sampleCount: 12, - partial: true, }, ], meta: { diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index 0138c5937..b435f3850 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -328,12 +328,6 @@ export const measurementResource = z // Synthesised-row extras — present only on the collapsed day-sum mode. dayKey: z.string().optional(), sampleCount: z.number().int().optional(), - partial: z - .boolean() - .optional() - .describe( - "True on the one bucket whose day was cut off by the read cap, so its sum understates the real total.", - ), }) .meta({ id: "MeasurementResource", @@ -400,11 +394,6 @@ const listMeasurementsMeta = z .describe( "Rows discarded by the cross-source canonical picker on the day-collapse mode.", ), - windowFrom: isoDateTime() - .optional() - .describe( - "Set on the per-night SLEEP_DURATION mode when the requested window was longer than 365 days: the instant the read started from. The window is a year ending at `to` (or now); page further back by moving `to`.", - ), }) .meta({ id: "ListMeasurementsMeta" }); @@ -814,7 +803,8 @@ export const measurementPaths: NonNullable = { tags: ["Measurements"], summary: "List measurements", description: - "Filter by type + date range. Response is paged via offset/limit; default limit 100, hard cap 500.", + "Filter by type + date range. Response is paged via offset/limit; default limit 100.\n\n" + + "With `groupBy=day` on a cumulative type the rows are one sum per local day, and `limit` / `offset` page over those days; every returned day is complete. `type=SLEEP_DURATION` returns one row per night, paged the same way over nights. On both, `meta.total` counts the days or nights in the filtered window, and a page holds at most 1000 of them (`meta.limit` echoes the value applied).", requestParams: { query: listMeasurementsSchema, }, diff --git a/src/lib/validations/measurement.ts b/src/lib/validations/measurement.ts index 8b32f2031..24eb63e4e 100644 --- a/src/lib/validations/measurement.ts +++ b/src/lib/validations/measurement.ts @@ -619,12 +619,9 @@ export const listMeasurementsSchema = z // and `sampleCount` = number of per-sample rows behind the bucket. // Omitted = legacy per-sample list behaviour (iOS contract stable). // - // v1.4.37 W10 — the route's groupBy=day branch hard-codes `offset:0` - // in the meta because the collapse runs after the per-sample scan; - // real pagination would require Postgres-side `date_trunc` grouping - // and a separate `prisma.count({ distinct: ["dayKey"] })`. Until - // that lands, reject any caller threading a non-zero offset so the - // pagination contract isn't silently dropped. + // v1.43.6 — `limit` / `offset` page over DAYS on this mode: the route + // picks the page's local days in Postgres and reports the distinct-day + // count as `meta.total` (#1245). groupBy: z.enum(["day"]).optional(), // v1.4.37 W7c — drill-down to per-sample rows for a single day in // the user's IANA timezone. Format `YYYY-MM-DD`; the route resolves @@ -632,9 +629,8 @@ export const listMeasurementsSchema = z // expandable list row to reveal the chunks that contributed to the // collapsed daily total. // - // v1.4.37 W10 — same offset restriction as `groupBy=day`; the - // drill-down branch returns a single bounded page rather than a - // cursor. + // v1.4.37 W10 — a drill-down returns one bounded page rather than a + // cursor, so it refuses a non-zero `offset`. dayKey: z .string() .regex(/^\d{4}-\d{2}-\d{2}$/, "dayKey must be YYYY-MM-DD") @@ -652,15 +648,11 @@ export const listMeasurementsSchema = z ) .optional(), }) - .refine( - ({ offset, groupBy, dayKey }) => - !(offset > 0 && (groupBy === "day" || dayKey != null)), - { - message: - "offset is not supported with groupBy=day or dayKey; use a smaller window or omit offset", - path: ["offset"], - }, - ) + .refine(({ offset, dayKey }) => !(offset > 0 && dayKey != null), { + message: + "offset is not supported with dayKey; a drill-down returns the whole day", + path: ["offset"], + }) // v1.4.38 — surface the drill-down cap on the validator so a caller // asking for more than 1000 rows on a per-day branch sees a 422 // instead of a silent `Math.min(limit, 1000)` clamp inside the diff --git a/tests/integration/measurements-day-grouped-paging.test.ts b/tests/integration/measurements-day-grouped-paging.test.ts new file mode 100644 index 000000000..85d60a3a9 --- /dev/null +++ b/tests/integration/measurements-day-grouped-paging.test.ts @@ -0,0 +1,239 @@ +/** + * #1245 — `GET /api/measurements?type=&groupBy=day` pages over + * local days. + * + * The list used to read the newest `limit` raw rows (the web asked for 5000), + * fold them into days and return every day at once: thousands of table rows + * for a long history, and every day older than the 5000th raw row silently + * missing. The history seeded here is longer than that old cap on purpose, + * with gap days and a second source on some days, and the test walks every + * page to prove each day arrives exactly once, whole, on a small first page. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + cookieJar.clear(); + headerJar.clear(); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +interface DayRow { + id: string; + dayKey: string; + value: number; + sampleCount: number; + source: string; + partial?: boolean; +} + +interface ListEnvelope { + data: { + measurements: DayRow[]; + meta: { total: number; limit: number; offset: number; groupBy?: string }; + }; +} + +const DAYS = 1000; +const SAMPLES_PER_DAY = 6; +const PAGE = 25; + +/** UTC calendar key of `now` shifted back `daysAgo` days. */ +function dayKeyAgo(daysAgo: number): string { + const d = new Date(); + d.setUTCHours(12, 0, 0, 0); + d.setUTCDate(d.getUTCDate() - daysAgo); + return d.toISOString().slice(0, 10); +} + +/** A day is left empty when this holds, so gaps never count as days. */ +const isGap = (daysAgo: number) => daysAgo % 10 === 3; + +async function seedSession(username: string) { + const prisma = getPrismaClient(); + const user = await prisma.user.create({ + data: { + username, + email: `${username}@example.test`, + role: "USER", + timezone: "Europe/Berlin", + }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return user; +} + +/** + * Seeds `DAYS` days of step chunks ending yesterday. Each chunk sits between + * 08:00 and 13:00 UTC, which is the same calendar day in Berlin all year. + * Every fifth day also carries a Withings total that the source ladder must + * drop rather than add. + */ +async function seedHistory(userId: string) { + const rows: Array<{ + userId: string; + type: "ACTIVITY_STEPS"; + value: number; + unit: string; + source: "APPLE_HEALTH" | "WITHINGS"; + measuredAt: Date; + externalId: string; + }> = []; + const expected = new Map(); + for (let daysAgo = 1; daysAgo <= DAYS; daysAgo++) { + if (isGap(daysAgo)) continue; + const key = dayKeyAgo(daysAgo); + let sum = 0; + for (let i = 0; i < SAMPLES_PER_DAY; i++) { + const value = 100 * (i + 1) + (daysAgo % 7); + sum += value; + rows.push({ + userId, + type: "ACTIVITY_STEPS", + value, + unit: "steps", + source: "APPLE_HEALTH", + measuredAt: new Date(`${key}T${String(8 + i).padStart(2, "0")}:00:00Z`), + externalId: `uuid-${key}-${i}`, + }); + } + if (daysAgo % 5 === 0) { + rows.push({ + userId, + type: "ACTIVITY_STEPS", + value: 99_999, + unit: "steps", + source: "WITHINGS", + measuredAt: new Date(`${key}T10:30:00Z`), + externalId: `withings-${key}`, + }); + } + expected.set(key, sum); + } + await getPrismaClient().measurement.createMany({ data: rows }); + return { expected, rawRows: rows.length }; +} + +async function callList(query: string): Promise { + const { GET } = await import("@/app/api/measurements/route"); + return GET(new NextRequest(`http://localhost/api/measurements?${query}`)); +} + +describe("GET /api/measurements?groupBy=day paging (#1245)", () => { + it("pages over every day exactly once, whole, with a small first page", async () => { + const user = await seedSession("day-paging-user"); + const { expected, rawRows } = await seedHistory(user.id); + // The history is longer than the raw-row cap the list used to read. + expect(rawRows).toBeGreaterThan(5000); + + const first = await callList( + `type=ACTIVITY_STEPS&groupBy=day&limit=${PAGE}&offset=0`, + ); + expect(first.status).toBe(200); + const firstText = await first.text(); + const firstBody = JSON.parse(firstText) as ListEnvelope; + expect(firstBody.data.meta.total).toBe(expected.size); + expect(firstBody.data.meta.groupBy).toBe("day"); + expect(firstBody.data.measurements).toHaveLength(PAGE); + // A page is a page, not the history. + expect(firstText.length).toBeLessThan(20_000); + // Newest first: the first row is yesterday. + expect(firstBody.data.measurements[0].dayKey).toBe(dayKeyAgo(1)); + + const seen: string[] = []; + let offset = 0; + let pages = 0; + while (offset < firstBody.data.meta.total) { + const res = await callList( + `type=ACTIVITY_STEPS&groupBy=day&limit=${PAGE}&offset=${offset}`, + ); + expect(res.status).toBe(200); + const body = (await res.json()) as ListEnvelope; + expect(body.data.meta.total).toBe(expected.size); + expect(body.data.meta.offset).toBe(offset); + for (const row of body.data.measurements) { + seen.push(row.dayKey); + // Whole day, canonical source only: the Withings total never adds. + expect(row.value).toBe(expected.get(row.dayKey)); + expect(row.sampleCount).toBe(SAMPLES_PER_DAY); + expect(row.source).toBe("APPLE_HEALTH"); + expect(row.partial).toBeUndefined(); + } + offset += PAGE; + pages += 1; + } + expect(pages).toBe(Math.ceil(expected.size / PAGE)); + expect(new Set(seen).size).toBe(seen.length); + expect([...seen].sort()).toEqual([...expected.keys()].sort()); + // Descending across page boundaries, not just within a page. + expect(seen).toEqual([...seen].sort().reverse()); + + // Past the end: an empty page that still reports the total. + const past = (await ( + await callList( + `type=ACTIVITY_STEPS&groupBy=day&limit=${PAGE}&offset=${expected.size + PAGE}`, + ) + ).json()) as ListEnvelope; + expect(past.data.measurements).toEqual([]); + expect(past.data.meta.total).toBe(expected.size); + }); + + it("starts at the oldest day ascending and counts only the filtered window", async () => { + const user = await seedSession("day-paging-window-user"); + const { expected } = await seedHistory(user.id); + + const asc = (await ( + await callList( + `type=ACTIVITY_STEPS&groupBy=day&limit=${PAGE}&sortDir=asc`, + ) + ).json()) as ListEnvelope; + expect(asc.data.measurements[0].dayKey).toBe(dayKeyAgo(DAYS)); + expect(asc.data.measurements[0].value).toBe(expected.get(dayKeyAgo(DAYS))); + + // A 30-day window: from the start of day 30 ago to the end of yesterday. + const from = `${dayKeyAgo(30)}T00:00:00Z`; + const to = `${dayKeyAgo(1)}T23:59:59Z`; + const windowed = (await ( + await callList( + `type=ACTIVITY_STEPS&groupBy=day&limit=${PAGE}&from=${from}&to=${to}`, + ) + ).json()) as ListEnvelope; + let inWindow = 0; + for (let daysAgo = 1; daysAgo <= 30; daysAgo++) { + if (!isGap(daysAgo)) inWindow += 1; + } + expect(windowed.data.meta.total).toBe(inWindow); + expect(windowed.data.measurements).toHaveLength(PAGE); + }); +}); diff --git a/tests/integration/measurements-sleep-list.test.ts b/tests/integration/measurements-sleep-list.test.ts index 9524bd854..2044d7d74 100644 --- a/tests/integration/measurements-sleep-list.test.ts +++ b/tests/integration/measurements-sleep-list.test.ts @@ -161,60 +161,131 @@ describe("GET /api/measurements?type=SLEEP_DURATION", () => { expect(stages).toEqual(["CORE", "DEEP", "REM"]); }); - it("reads at most a year ending at `to`, however early `from` is", async () => { - // An explicit early `from` used to walk every per-stage row back to it. + it("pages over every night of a long history in SQL, with no year cut", async () => { const prisma = getPrismaClient(); - const user = await seedSession("sleep-window-user"); + const user = await seedSession("sleep-paging-user"); const day = 86_400_000; const morning = new Date(); morning.setUTCHours(4, 0, 0, 0); - // One night of a single CORE stage every day for two years. - await prisma.measurement.createMany({ - data: Array.from({ length: 730 }, (_, i) => ({ + const NIGHTS = 730; + const dayKeyOf = (d: Date) => d.toISOString().slice(0, 10); + // One night of a single CORE stage every day for two years. Every + // seventh day also carries an afternoon nap, which is not a night of its + // own, and every eleventh an in-bed-only evening (more than the session + // gap from both the nap and the night), which holds no sleep and must + // not count as a night either. + const rows: Array<{ + userId: string; + type: "SLEEP_DURATION"; + value: number; + unit: string; + source: "APPLE_HEALTH"; + measuredAt: Date; + externalId: string; + sleepStage: "CORE" | "IN_BED"; + }> = []; + for (let i = 0; i < NIGHTS; i++) { + const wake = new Date(morning.getTime() - i * day); + const base = { userId: user.id, type: "SLEEP_DURATION" as const, - value: 420, unit: "minutes", source: "APPLE_HEALTH" as const, - measuredAt: new Date(morning.getTime() - i * day), + }; + rows.push({ + ...base, + value: 420, + measuredAt: wake, externalId: `uuid-night-${i}`, - sleepStage: "CORE" as const, - })), - }); + sleepStage: "CORE", + }); + if (i % 7 === 0) { + rows.push({ + ...base, + value: 30, + measuredAt: new Date(wake.getTime() + 10 * 3_600_000), + externalId: `uuid-nap-${i}`, + sleepStage: "CORE", + }); + } + if (i % 11 === 5) { + rows.push({ + ...base, + value: 20, + measuredAt: new Date(wake.getTime() - 10.5 * 3_600_000), + externalId: `uuid-inbed-${i}`, + sleepStage: "IN_BED", + }); + } + } + await prisma.measurement.createMany({ data: rows }); + const spy = vi.spyOn(prisma.measurement, "findMany"); - const from = new Date(morning.getTime() - 800 * day).toISOString(); - const res = await callList( - `type=SLEEP_DURATION&from=${encodeURIComponent(from)}&limit=1000`, - ); - expect(res.status).toBe(200); - const body = (await res.json()) as ListEnvelope; - expect(body.data.meta.total).toBeLessThanOrEqual(366); - expect(body.data.meta.total).toBeGreaterThanOrEqual(364); - expect(body.data.meta.windowFrom).toBeDefined(); - const windowFrom = Date.parse(body.data.meta.windowFrom!); - expect(Date.now() - windowFrom).toBeLessThan(366 * day); - expect(Date.now() - windowFrom).toBeGreaterThan(364 * day); - // The read itself was bounded, not just the answer. + const first = await callList("type=SLEEP_DURATION&limit=25"); + expect(first.status).toBe(200); + const firstBody = (await first.json()) as ListEnvelope; + expect(firstBody.data.meta.total).toBe(NIGHTS); + expect(firstBody.data.meta.windowFrom).toBeUndefined(); + expect(firstBody.data.measurements).toHaveLength(25); + expect(firstBody.data.measurements[0].dayKey).toBe(dayKeyOf(morning)); + expect(firstBody.data.measurements[0].value).toBe(420); + expect(firstBody.data.measurements[0].napCount).toBe(1); + // The stage read covered the page's nights, not the history. const read = spy.mock.calls - .map(([args]) => args as { where?: { measuredAt?: { gte?: Date } } }) - .find((args) => args?.where?.measuredAt?.gte); - expect(read?.where?.measuredAt?.gte?.getTime()).toBe(windowFrom); + .map( + ([args]) => + args as { + where?: { AND?: Array<{ measuredAt?: { gte?: Date } }> }; + }, + ) + .find((args) => args?.where?.AND?.[0]?.measuredAt?.gte); + const readFrom = read!.where!.AND![0].measuredAt!.gte!.getTime(); + expect(morning.getTime() - readFrom).toBeLessThan(26 * day); - // Older nights are reached by moving `to` back. - const to = new Date(morning.getTime() - 400 * day).toISOString(); - const older = await callList( - `type=SLEEP_DURATION&from=${encodeURIComponent(from)}&to=${encodeURIComponent(to)}&limit=1000`, - ); - const olderBody = (await older.json()) as ListEnvelope; - expect(olderBody.data.meta.total).toBeGreaterThanOrEqual(329); - expect(olderBody.data.meta.total).toBeLessThanOrEqual(331); + // Walk every page: each night exactly once, newest first, back to the + // oldest one two years ago. + const seen: string[] = []; + for (let offset = 0; offset < NIGHTS; offset += 100) { + const res = await callList( + `type=SLEEP_DURATION&limit=100&offset=${offset}`, + ); + const body = (await res.json()) as ListEnvelope; + expect(body.data.meta.total).toBe(NIGHTS); + for (const row of body.data.measurements) { + seen.push(row.dayKey!); + expect(row.value).toBe(420); + } + } + expect(seen).toHaveLength(NIGHTS); + expect(new Set(seen).size).toBe(NIGHTS); + expect(seen).toEqual([...seen].sort().reverse()); + const oldest = dayKeyOf(new Date(morning.getTime() - (NIGHTS - 1) * day)); + expect(seen[seen.length - 1]).toBe(oldest); - // A window inside the year is untouched and carries no marker. - const recent = await callList( - `type=SLEEP_DURATION&from=${encodeURIComponent(new Date(morning.getTime() - 10.5 * day).toISOString())}`, - ); - const recentBody = (await recent.json()) as ListEnvelope; - expect(recentBody.data.meta.total).toBe(11); - expect(recentBody.data.meta.windowFrom).toBeUndefined(); + // Past the end: an empty page that still reports the total. + const past = (await ( + await callList(`type=SLEEP_DURATION&limit=25&offset=${NIGHTS + 25}`) + ).json()) as ListEnvelope; + expect(past.data.measurements).toEqual([]); + expect(past.data.meta.total).toBe(NIGHTS); + + // Ascending starts at the oldest night; a window counts only its nights. + const asc = (await ( + await callList("type=SLEEP_DURATION&limit=5&sortDir=asc") + ).json()) as ListEnvelope; + expect(asc.data.measurements[0].dayKey).toBe(oldest); + const recent = (await ( + await callList( + `type=SLEEP_DURATION&from=${encodeURIComponent(new Date(morning.getTime() - 10.5 * day).toISOString())}`, + ) + ).json()) as ListEnvelope; + expect(recent.data.meta.total).toBe(11); + + // A night older than a year still drills down. + const drill = (await ( + await callList(`type=SLEEP_DURATION&dayKey=${oldest}`) + ).json()) as ListEnvelope; + expect(drill.data.measurements).toHaveLength(1); + expect(drill.data.measurements[0].sleepStage).toBe("CORE"); }); });