From b2c484e25e7b6ee2d0c826119838c3b00ff129de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 1 Oct 2026 08:52:10 +0200 Subject: [PATCH] fix(series): bucket heart-rate variability and blood oxygen like pulse The dense rule in /api/measurements/series covered pulse and glucose only. Heart-rate variability and blood oxygen read every raw row for any window up to 3650 days, and a watch or ring that writes them every minute makes that the same unbounded read pulse had. Both kinds now fold per local day beyond 90 days and per local hour inside 90 days once the window holds more than 10 000 readings. A bucket keeps the point shape of its kind: only pulse carries valueMin / valueMax. Statistics stay computed over every reading, and a sparse series inside 90 days is unchanged. The OpenAPI description names the four kinds. --- docs/api/openapi.yaml | 19 +- src/app/api/measurements/series/route.ts | 32 +++- src/lib/openapi/routes/measurements.ts | 4 +- .../integration/dense-series-hrv-spo2.test.ts | 175 ++++++++++++++++++ 4 files changed, 210 insertions(+), 20 deletions(-) create mode 100644 tests/integration/dense-series-hrv-spo2.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 20072acf6..ad690c61f 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -41066,9 +41066,9 @@ components: properties: id: type: string - description: Measurement row id; `sleep:` for a sleep night; `day:` for a day bucket (`pulse` / - `glucose` beyond 90 days); `hour:` for an hour bucket (`pulse` / `glucose` within 90 days - when the window holds more than 10 000 readings). + description: Measurement row id; `sleep:` for a sleep night; `day:` for a day bucket (`pulse`, + `glucose`, `heartRateVariability`, `oxygenSaturation` beyond 90 days); `hour:` for an hour + bucket (the same kinds within 90 days when the window holds more than 10 000 readings). at: type: string format: date-time @@ -41141,12 +41141,13 @@ components: - stats additionalProperties: false description: "iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an - explicit `unit` token, and a summary `stats` block. Sample-dense kinds are bucketed server-side: `pulse` and - `glucose` beyond 90 days per local day, and `pulse` / `glucose` within 90 days per local hour when the window - holds more than 10 000 readings (a continuous glucose sensor, a watch recording heart rate every minute; a cuff - or fingerstick meter stays raw). A bucket carries the mean as `value`; a `pulse` bucket also carries its range - as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. `stats` - is always computed over the underlying readings, never over the buckets." + explicit `unit` token, and a summary `stats` block. Sample-dense kinds (`pulse`, `glucose`, + `heartRateVariability`, `oxygenSaturation`) are bucketed server-side: beyond 90 days per local day, and within + 90 days per local hour when the window holds more than 10 000 readings (a continuous glucose sensor, a watch or + ring recording every minute; a cuff, a fingerstick meter or a few readings a night stay raw). A bucket carries + the mean as `value` and otherwise has the shape of a reading of its kind; a `pulse` bucket also carries its + range as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. + `stats` is always computed over the underlying readings, never over the buckets." ListMeasurementsResponse: type: object properties: diff --git a/src/app/api/measurements/series/route.ts b/src/app/api/measurements/series/route.ts index 06d46e07a..f42afb47c 100644 --- a/src/app/api/measurements/series/route.ts +++ b/src/app/api/measurements/series/route.ts @@ -49,11 +49,23 @@ const SLEEP_SERIES_MAX_DAYS = 365; * byte-identical. Sparse kinds (weight, BP, …) are untouched. */ const DENSE_SERIES_RAW_WINDOW_DAYS = 90; -const DENSE_SERIES_KINDS: ReadonlySet = new Set(["glucose", "pulse"]); +/** + * The kinds a device can write at sampling rate. Heart-rate variability and + * blood oxygen joined pulse and glucose once watches and rings began writing + * them every few minutes overnight: their 3650-day "Alle" range read every + * raw row, which is the read this rule exists to stop. + */ +const DENSE_SERIES_KINDS: ReadonlySet = new Set([ + "glucose", + "pulse", + "heartRateVariability", + "oxygenSaturation", +]); /** - * #1023 — inside the raw window, a pulse or glucose series with more rows - * than this is bucketed per local hour in SQL instead of sent raw. + * #1023 — inside the raw window, a dense-kind series (pulse, glucose, HRV, + * blood oxygen) with more rows than this is bucketed per local hour in SQL + * instead of sent raw. * * A watch that records heart rate once a minute puts 43 000 rows in the * default 30-day window and 130 000 in 90 days: a 15 MB response the client @@ -381,7 +393,7 @@ export const GET = apiHandler(async (request: NextRequest) => { }, })) > DENSE_SERIES_RAW_ROW_CAP) ) { - // Day buckets past the raw window; hour buckets for a pulse or CGM + // Day buckets past the raw window; hour buckets for a dense-kind // stream too dense to send raw inside it. const grain = days > DENSE_SERIES_RAW_WINDOW_DAYS ? "day" : "hour"; // v1.28.25 — long-window read of a sample-dense kind (CGM glucose, @@ -489,17 +501,19 @@ export const GET = apiHandler(async (request: NextRequest) => { }; } } else { - // pulse — `value` is the bucket's average; valueMin/valueMax carry the - // bucket's low/high band (folding each hourly import bucket's own + // `value` is the bucket's average. For pulse, valueMin/valueMax carry + // the bucket's low/high band (folding each hourly import bucket's own // spread via the COALESCE above), same band semantics as the per-hour - // shape. + // shape. The other dense kinds keep the bare point shape their raw rows + // have, so a client reads a bucket the way it reads a reading. points = bucketRows.map((r) => ({ id: dayId(r.bucket_start), at: r.bucket_start.toISOString(), value: round2(r.mean), secondary: null, - valueMin: r.min_value, - valueMax: r.max_value, + ...(kind === "pulse" + ? { valueMin: r.min_value, valueMax: r.max_value } + : {}), })); if ( rawAgg !== undefined && diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index 0795e88b1..864baeb4a 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -696,7 +696,7 @@ const seriesPointSchema = z.object({ id: z .string() .describe( - "Measurement row id; `sleep:` for a sleep night; `day:` for a day bucket (`pulse` / `glucose` beyond 90 days); `hour:` for an hour bucket (`pulse` / `glucose` within 90 days when the window holds more than 10 000 readings).", + "Measurement row id; `sleep:` for a sleep night; `day:` for a day bucket (`pulse`, `glucose`, `heartRateVariability`, `oxygenSaturation` beyond 90 days); `hour:` for an hour bucket (the same kinds within 90 days when the window holds more than 10 000 readings).", ), at: z.iso.datetime({ offset: true }).describe("Point timestamp (ISO-8601)."), value: z.number().describe("Primary value in the top-level `unit`."), @@ -751,7 +751,7 @@ const seriesResponse = z .meta({ id: "MeasurementsSeriesResponse", description: - "iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an explicit `unit` token, and a summary `stats` block. Sample-dense kinds are bucketed server-side: `pulse` and `glucose` beyond 90 days per local day, and `pulse` / `glucose` within 90 days per local hour when the window holds more than 10 000 readings (a continuous glucose sensor, a watch recording heart rate every minute; a cuff or fingerstick meter stays raw). A bucket carries the mean as `value`; a `pulse` bucket also carries its range as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. `stats` is always computed over the underlying readings, never over the buckets.", + "iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an explicit `unit` token, and a summary `stats` block. Sample-dense kinds (`pulse`, `glucose`, `heartRateVariability`, `oxygenSaturation`) are bucketed server-side: beyond 90 days per local day, and within 90 days per local hour when the window holds more than 10 000 readings (a continuous glucose sensor, a watch or ring recording every minute; a cuff, a fingerstick meter or a few readings a night stay raw). A bucket carries the mean as `value` and otherwise has the shape of a reading of its kind; a `pulse` bucket also carries its range as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. `stats` is always computed over the underlying readings, never over the buckets.", }); // ── Batched daily series (v1.18.6 dashboard fetch coalescing) ──────── diff --git a/tests/integration/dense-series-hrv-spo2.test.ts b/tests/integration/dense-series-hrv-spo2.test.ts new file mode 100644 index 000000000..e60c70647 --- /dev/null +++ b/tests/integration/dense-series-hrv-spo2.test.ts @@ -0,0 +1,175 @@ +/** + * The per-kind series for heart-rate variability and blood oxygen follows the + * same dense rule as pulse and glucose. + * + * Both used to read every raw row for any window up to 3650 days, and a ring + * or a watch that writes them every minute overnight, or all day, makes that + * the same unbounded read pulse had before #1023. Beyond 90 days they now fold + * per local day; within 90 days a window of more than 10 000 readings folds + * per local hour. A sparse account inside 90 days keeps its raw readings, + * byte for byte, and a bucket has the shape of a reading of its kind (no + * `valueMin` / `valueMax`, which only pulse carries). + */ +import { NextRequest } from "next/server"; +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { cookieJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const TZ = "Europe/Berlin"; +const DENSE = "dense-hrv-spo2-owner"; +const SPARSE = "sparse-hrv-spo2-owner"; +const NOW = new Date(Math.floor(Date.now() / 60_000) * 60_000 - 60_000); +/** Twelve days of one reading a minute: over the 10 000-row cap. */ +const DENSE_ROWS = 12 * 24 * 60; + +const KINDS = [ + { kind: "heartRateVariability", type: "HEART_RATE_VARIABILITY", unit: "ms" }, + { kind: "oxygenSaturation", type: "OXYGEN_SATURATION", unit: "%" }, +] as const; + +beforeAll(async () => { + await truncateAllTables(prisma); + for (const id of [DENSE, SPARSE]) { + await prisma.user.create({ data: { id, username: id, timezone: TZ } }); + invalidateUserTimezone(id); + } + for (const { type, unit } of KINDS) { + await prisma.$executeRawUnsafe( + `INSERT INTO measurements ( + id, user_id, type, value, unit, source, measured_at, + created_at, updated_at, sync_version) + SELECT $4 || lpad(g::text, 7, '0'), $1, $5::measurement_type, + CASE WHEN $5 = 'OXYGEN_SATURATION' THEN 92 + (g % 8) ELSE 30 + (g % 40) * 0.5 END, + $6, 'GOOGLE_HEALTH'::measurement_source, + ($2::timestamptz AT TIME ZONE 'UTC') - (g * interval '1 minute'), + now(), now(), 1 + FROM generate_series(0, $3::int - 1) g`, + DENSE, + NOW.toISOString(), + DENSE_ROWS, + type === "OXYGEN_SATURATION" ? "ds" : "dh", + type, + unit, + ); + // A few readings a night for a month and a half. + await prisma.measurement.createMany({ + data: Array.from({ length: 45 * 3 }, (_, i) => ({ + id: `sp-${type}-${i}`, + userId: SPARSE, + type, + value: type === "OXYGEN_SATURATION" ? 94 + (i % 5) : 35 + (i % 17), + unit, + source: "MANUAL" as const, + measuredAt: new Date( + NOW.getTime() - Math.floor(i / 3) * 86_400_000 - (i % 3) * 7_200_000, + ), + })), + }); + } + await prisma.$executeRawUnsafe(`ANALYZE measurements`); +}, 120_000); + +async function series(userId: string, kind: string, days: number) { + const s = await prisma.session.create({ + data: { userId, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.clear(); + cookieJar.set("healthlog_session", s.id); + const { GET } = await import("@/app/api/measurements/series/route"); + const res = await GET( + new NextRequest( + `http://localhost/api/measurements/series?kind=${kind}&days=${days}`, + ), + ); + expect(res.status).toBe(200); + return ( + (await res.json()) as { + data: { + unit: string; + points: Array & { id: string; value: number }>; + stats: { count: number; mean: number; min: number; max: number }; + }; + } + ).data; +} + +describe.each(KINDS)("$kind series", ({ kind, type, unit }) => { + it("sends a dense stream inside 90 days as hour buckets", async () => { + const data = await series(DENSE, kind, 30); + expect(data.unit).toBe(unit); + expect(data.points.length).toBeLessThanOrEqual(12 * 24 + 2); + expect(data.points.length).toBeGreaterThanOrEqual(12 * 24 - 2); + expect(data.points.every((p) => p.id.startsWith("hour:"))).toBe(true); + // A bucket has the shape of a reading of its kind. + expect( + data.points.every((p) => !("valueMin" in p) && !("valueMax" in p)), + ).toBe(true); + // Statistics stay over every reading. + const agg = await prisma.measurement.aggregate({ + where: { + userId: DENSE, + type, + measuredAt: { gte: new Date(Date.now() - 30 * 86_400_000) }, + }, + _count: true, + _min: { value: true }, + _max: { value: true }, + }); + expect(data.stats.count).toBe(agg._count); + expect(data.stats.min).toBe(agg._min.value); + expect(data.stats.max).toBe(agg._max.value); + }); + + it("folds the long ranges per local day", async () => { + const data = await series(SPARSE, kind, 3650); + expect(data.points.length).toBeGreaterThan(40); + expect(data.points.length).toBeLessThanOrEqual(46); + expect(data.points.every((p) => p.id.startsWith("day:"))).toBe(true); + expect(data.stats.count).toBe(45 * 3); + }); + + it("keeps a sparse series inside 90 days raw", async () => { + const data = await series(SPARSE, kind, 30); + const raw = await prisma.measurement.findMany({ + where: { + userId: SPARSE, + type, + measuredAt: { gte: new Date(Date.now() - 30 * 86_400_000) }, + }, + orderBy: { measuredAt: "asc" }, + select: { id: true, value: true, measuredAt: true }, + }); + expect(data.points).toEqual( + raw.map((r) => ({ + id: r.id, + at: r.measuredAt.toISOString(), + value: r.value, + secondary: null, + })), + ); + }); +});