mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 20:42:43 +00:00
fix(measurements): hour-bucket a CGM glucose series inside the 90-day window
A sensor reading every five minutes puts about 26 000 glucose readings in a 90-day window, and the per-kind series sent every one of them raw. The hourly bucketing that already applies to a dense heart-rate stream now applies to glucose too, with the same 10 000-reading cap: the hour's mean converted once to the user's unit, statistics over every raw reading. A fingerstick meter stays far under the cap and keeps its raw points. The doctor report keeps glucose raw inside 90 days. Its clinical panel is computed from the raw readings of every source, while the day path works from canonical-source day buckets, so the two do not give the same figures when a meter and a sensor overlap.
This commit is contained in:
1 parent
ca10f2e5df
commit
731e073284
5 files changed
+340
-18
No files matched your search
@@ -40910,8 +40910,8 @@ components:
|
||||
id:
|
||||
type: string
|
||||
description: Measurement row id; `sleep:<wake-day>` for a sleep night; `day:<local-day>` for a day bucket (`pulse` /
|
||||
`glucose` beyond 90 days); `hour:<ISO start>` for an hour bucket (`pulse` within 90 days when the
|
||||
window holds more than 10 000 readings).
|
||||
`glucose` beyond 90 days); `hour:<ISO start>` for an hour bucket (`pulse` / `glucose` within 90 days
|
||||
when the window holds more than 10 000 readings).
|
||||
at:
|
||||
type: string
|
||||
format: date-time
|
||||
@@ -40985,9 +40985,11 @@ components:
|
||||
additionalProperties: false
|
||||
description: "iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an
|
||||
explicit `unit` token, and a summary `stats` block. Sample-dense kinds are bucketed server-side: `pulse` and
|
||||
`glucose` beyond 90 days per local day, and `pulse` within 90 days per local hour when the window holds more
|
||||
than 10 000 readings (a bucket carries the mean as `value` and its range as `valueMin` / `valueMax`). `stats` is
|
||||
always computed over the underlying readings, never over the buckets."
|
||||
`glucose` beyond 90 days per local day, and `pulse` / `glucose` within 90 days per local hour when the window
|
||||
holds more than 10 000 readings (a continuous glucose sensor, a watch recording heart rate every minute; a cuff
|
||||
or fingerstick meter stays raw). A bucket carries the mean as `value`; a `pulse` bucket also carries its range
|
||||
as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. `stats`
|
||||
is always computed over the underlying readings, never over the buckets."
|
||||
ListMeasurementsResponse:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
@@ -631,6 +631,85 @@ describe("GET /api/measurements/series — dense-kind day-bucketing (v1.28.25)",
|
||||
expect(bucketSql.slice(1)).toContain("hour");
|
||||
});
|
||||
|
||||
it("hour-buckets a CGM glucose window too dense to send raw, in the user's unit", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({
|
||||
glucoseUnit: "mmol/L",
|
||||
} as never);
|
||||
// A sensor reading every five minutes for 90 days.
|
||||
vi.mocked(prisma.measurement.count).mockResolvedValue(25_920 as never);
|
||||
vi.mocked(prisma.$queryRaw)
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
bucket_start: new Date("2026-06-01T08:00:00Z"),
|
||||
mean: 100,
|
||||
min_value: 90,
|
||||
max_value: 112,
|
||||
},
|
||||
] as never)
|
||||
.mockResolvedValueOnce([
|
||||
{ n: 25_920, mean: 118.004, min: 48, max: 260, sd: 36.004 },
|
||||
] as never);
|
||||
|
||||
const res = await GET(req("kind=glucose&days=90"));
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
data: {
|
||||
unit: string;
|
||||
points: Array<Record<string, unknown>>;
|
||||
stats: {
|
||||
count: number;
|
||||
mean: number;
|
||||
min: number;
|
||||
max: number;
|
||||
stdDev: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
expect(prisma.measurement.findMany).not.toHaveBeenCalled();
|
||||
expect(vi.mocked(prisma.measurement.count).mock.calls[0]?.[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
where: expect.objectContaining({ type: "BLOOD_GLUCOSE" }),
|
||||
}),
|
||||
);
|
||||
expect(body.data.unit).toBe("mmol/L");
|
||||
// Converted once at serialization; no band on a glucose point.
|
||||
expect(body.data.points).toEqual([
|
||||
{
|
||||
id: "hour:2026-06-01T08:00:00.000Z",
|
||||
at: "2026-06-01T08:00:00.000Z",
|
||||
value: 5.5,
|
||||
secondary: null,
|
||||
},
|
||||
]);
|
||||
// Stats describe every reading (mg/dL aggregate, converted once).
|
||||
expect(body.data.stats).toEqual({
|
||||
count: 25_920,
|
||||
mean: 6.5,
|
||||
min: 2.7,
|
||||
max: 14.4,
|
||||
stdDev: 2,
|
||||
});
|
||||
const bucketSql = vi.mocked(prisma.$queryRaw).mock.calls[0] as unknown[];
|
||||
expect(bucketSql.slice(1)).toContain("hour");
|
||||
});
|
||||
|
||||
it("keeps fingerstick glucose at or under the row cap raw", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.measurement.count).mockResolvedValue(360 as never);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
{ id: "g1", value: 104, measuredAt: new Date("2026-06-01T07:12:00Z") },
|
||||
] as never);
|
||||
const res = await GET(req("kind=glucose&days=90"));
|
||||
const body = (await res.json()) as {
|
||||
data: { points: Array<Record<string, unknown>> };
|
||||
};
|
||||
expect(prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
expect(body.data.points).toEqual([
|
||||
{ id: "g1", at: "2026-06-01T07:12:00.000Z", value: 104, secondary: null },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps a pulse window at or under the row cap raw", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.measurement.count).mockResolvedValue(10_000 as never);
|
||||
|
||||
@@ -52,17 +52,20 @@ const DENSE_SERIES_RAW_WINDOW_DAYS = 90;
|
||||
const DENSE_SERIES_KINDS: ReadonlySet<string> = new Set(["glucose", "pulse"]);
|
||||
|
||||
/**
|
||||
* #1023 — inside the raw window, a pulse series with more rows than this is
|
||||
* bucketed per local hour in SQL instead of sent raw.
|
||||
* #1023 — inside the raw window, a pulse or glucose series with more rows
|
||||
* than this is bucketed per local hour in SQL instead of sent raw.
|
||||
*
|
||||
* A watch that records heart rate once a minute puts 43 000 rows in the
|
||||
* default 30-day window and 130 000 in 90 days: a 15 MB response the client
|
||||
* could not draw at that resolution anyway, built from as many objects on the
|
||||
* server. Hourly buckets carry the hour's mean with its low/high band, the
|
||||
* shape an hourly-bucket import already has, and cap the answer at 24 points
|
||||
* a day. The cap sits far above what sparse sources produce (a cuff, hourly
|
||||
* buckets over 90 days, a watch sampling every five minutes over 30 days), so
|
||||
* those series stay raw and unchanged.
|
||||
* server. A CGM reading every five minutes puts 26 000 glucose rows in 90
|
||||
* days, the same problem at a smaller scale. Hourly buckets carry the hour's
|
||||
* mean (pulse also its low/high band, the shape an hourly-bucket import
|
||||
* already has) and cap the answer at 24 points a day; the statistics stay
|
||||
* computed over every raw reading. The cap sits far above what sparse sources
|
||||
* produce (a cuff, fingerstick glucose, hourly buckets over 90 days, a watch
|
||||
* sampling every five minutes over 30 days), so those series stay raw and
|
||||
* unchanged.
|
||||
*/
|
||||
const DENSE_SERIES_RAW_ROW_CAP = 10_000;
|
||||
|
||||
@@ -368,18 +371,18 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
});
|
||||
} else if (
|
||||
(DENSE_SERIES_KINDS.has(kind) && days > DENSE_SERIES_RAW_WINDOW_DAYS) ||
|
||||
(kind === "pulse" &&
|
||||
(DENSE_SERIES_KINDS.has(kind) &&
|
||||
(await prisma.measurement.count({
|
||||
where: {
|
||||
userId: user.id,
|
||||
type: "PULSE",
|
||||
type: KIND_TO_TYPE[kind],
|
||||
measuredAt: { gte: since },
|
||||
deletedAt: null,
|
||||
},
|
||||
})) > DENSE_SERIES_RAW_ROW_CAP)
|
||||
) {
|
||||
// Day buckets past the raw window; hour buckets for a pulse stream too
|
||||
// dense to send raw inside it.
|
||||
// Day buckets past the raw window; hour buckets for a pulse or CGM
|
||||
// stream too dense to send raw inside it.
|
||||
const grain = days > DENSE_SERIES_RAW_WINDOW_DAYS ? "day" : "hour";
|
||||
// v1.28.25 — long-window read of a sample-dense kind (CGM glucose,
|
||||
// per-sample / hourly pulse). Day-bucket in SQL instead of walking
|
||||
|
||||
@@ -696,7 +696,7 @@ const seriesPointSchema = z.object({
|
||||
id: z
|
||||
.string()
|
||||
.describe(
|
||||
"Measurement row id; `sleep:<wake-day>` for a sleep night; `day:<local-day>` for a day bucket (`pulse` / `glucose` beyond 90 days); `hour:<ISO start>` for an hour bucket (`pulse` within 90 days when the window holds more than 10 000 readings).",
|
||||
"Measurement row id; `sleep:<wake-day>` for a sleep night; `day:<local-day>` for a day bucket (`pulse` / `glucose` beyond 90 days); `hour:<ISO start>` for an hour bucket (`pulse` / `glucose` within 90 days when the window holds more than 10 000 readings).",
|
||||
),
|
||||
at: z.iso.datetime({ offset: true }).describe("Point timestamp (ISO-8601)."),
|
||||
value: z.number().describe("Primary value in the top-level `unit`."),
|
||||
@@ -751,7 +751,7 @@ const seriesResponse = z
|
||||
.meta({
|
||||
id: "MeasurementsSeriesResponse",
|
||||
description:
|
||||
"iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an explicit `unit` token, and a summary `stats` block. Sample-dense kinds are bucketed server-side: `pulse` and `glucose` beyond 90 days per local day, and `pulse` within 90 days per local hour when the window holds more than 10 000 readings (a bucket carries the mean as `value` and its range as `valueMin` / `valueMax`). `stats` is always computed over the underlying readings, never over the buckets.",
|
||||
"iOS-friendly per-kind time series: one point per reading (or per reconstructed night for `sleep`), an explicit `unit` token, and a summary `stats` block. Sample-dense kinds are bucketed server-side: `pulse` and `glucose` beyond 90 days per local day, and `pulse` / `glucose` within 90 days per local hour when the window holds more than 10 000 readings (a continuous glucose sensor, a watch recording heart rate every minute; a cuff or fingerstick meter stays raw). A bucket carries the mean as `value`; a `pulse` bucket also carries its range as `valueMin` / `valueMax`, and a `glucose` bucket is converted to the user's unit like a raw reading. `stats` is always computed over the underlying readings, never over the buckets.",
|
||||
});
|
||||
|
||||
// ── Batched daily series (v1.18.6 dashboard fetch coalescing) ────────
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
/**
|
||||
* The per-kind glucose series inside the 90-day raw window, on a continuous
|
||||
* sensor stream.
|
||||
*
|
||||
* Past 90 days glucose already folded per day in SQL. Inside it, a sensor
|
||||
* reading every five minutes sent every row raw: 26 000 points in 90 days,
|
||||
* the same class as the per-minute heart-rate stream in
|
||||
* `dense-pulse-raw-window-reads.test.ts`. Over the row cap the series now
|
||||
* folds per local hour; a fingerstick meter stays far under it and keeps its
|
||||
* exact old shape.
|
||||
*
|
||||
* The doctor report deliberately keeps glucose raw inside 90 days: its
|
||||
* clinical panel (time in range, GMI, CV and the risk indices) is computed
|
||||
* from the raw readings of every source, and the day path computes it from
|
||||
* canonical-source day buckets, which is not the same figure when a meter and
|
||||
* a sensor overlap. The last case pins that the panel is still the raw
|
||||
* engine's answer.
|
||||
*/
|
||||
import { NextRequest } from "next/server";
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { cookieJar } from "./mock-next-headers";
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
import { computeGlucoseClinicalMetrics } from "@/lib/analytics/glucose-metrics";
|
||||
import { collectDoctorReportData } from "@/lib/doctor-report-data";
|
||||
import { convertGlucose } from "@/lib/glucose";
|
||||
import { selectionFromLeaves } from "@/lib/report-selection/selection";
|
||||
import { invalidateUserTimezone } from "@/lib/tz/resolver";
|
||||
|
||||
vi.mock("next/headers", async () => {
|
||||
const { cookieJar, headerJar } = await import("./mock-next-headers");
|
||||
return {
|
||||
headers: vi.fn(async () => ({
|
||||
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
|
||||
})),
|
||||
cookies: vi.fn(async () => ({
|
||||
get: (name: string) => {
|
||||
const value = cookieJar.get(name);
|
||||
return value ? { name, value } : undefined;
|
||||
},
|
||||
set: (name: string, value: string) => {
|
||||
cookieJar.set(name, value);
|
||||
},
|
||||
delete: (name: string) => {
|
||||
cookieJar.delete(name);
|
||||
},
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("@/lib/db-compat", () => ({
|
||||
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const prisma = getPrismaClient();
|
||||
const TZ = "America/New_York";
|
||||
const DENSE = "dense-glucose-owner";
|
||||
const SPARSE = "sparse-glucose-owner";
|
||||
/** Forty days of one reading every five minutes: over the raw-window cap. */
|
||||
const DENSE_ROWS = 40 * 288;
|
||||
const NOW = new Date(Math.floor(Date.now() / 300_000) * 300_000 - 300_000);
|
||||
|
||||
/** A fingerstick meter: a handful of readings a day at irregular times. */
|
||||
function sparseReadings(): Array<{ at: Date; value: number }> {
|
||||
const out: Array<{ at: Date; value: number }> = [];
|
||||
for (let day = 1; day <= 30; day++) {
|
||||
const perDay = 2 + (day % 4);
|
||||
for (let i = 0; i < perDay; i++) {
|
||||
out.push({
|
||||
at: new Date(
|
||||
NOW.getTime() - day * 86_400_000 + (i * 211 + day * 17) * 60_000,
|
||||
),
|
||||
value: 78 + ((day * 13 + i * 29) % 120),
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function session(userId: string): Promise<void> {
|
||||
const s = await prisma.session.create({
|
||||
data: { userId, expiresAt: new Date(Date.now() + 3_600_000) },
|
||||
});
|
||||
cookieJar.clear();
|
||||
cookieJar.set("healthlog_session", s.id);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
await truncateAllTables(prisma);
|
||||
await prisma.user.create({
|
||||
data: { id: DENSE, username: DENSE, timezone: TZ, glucoseUnit: "mmol/L" },
|
||||
});
|
||||
await prisma.user.create({
|
||||
data: { id: SPARSE, username: SPARSE, timezone: TZ },
|
||||
});
|
||||
invalidateUserTimezone(DENSE);
|
||||
invalidateUserTimezone(SPARSE);
|
||||
await prisma.$executeRawUnsafe(
|
||||
`INSERT INTO measurements (
|
||||
id, user_id, type, value, unit, source, measured_at,
|
||||
created_at, updated_at, sync_version)
|
||||
SELECT 'cg' || lpad(g::text, 7, '0'), $1, 'BLOOD_GLUCOSE'::measurement_type,
|
||||
62 + ((g * 37) % 170), 'mg/dL', 'NIGHTSCOUT'::measurement_source,
|
||||
($2::timestamptz AT TIME ZONE 'UTC') - (g * interval '5 minutes'),
|
||||
now(), now(), 1
|
||||
FROM generate_series(0, $3::int - 1) g`,
|
||||
DENSE,
|
||||
NOW.toISOString(),
|
||||
DENSE_ROWS,
|
||||
);
|
||||
await prisma.measurement.createMany({
|
||||
data: sparseReadings().map((r, i) => ({
|
||||
id: `fs${String(i).padStart(4, "0")}`,
|
||||
userId: SPARSE,
|
||||
type: "BLOOD_GLUCOSE" as const,
|
||||
value: r.value,
|
||||
unit: "mg/dL",
|
||||
source: "MANUAL" as const,
|
||||
measuredAt: r.at,
|
||||
})),
|
||||
});
|
||||
await prisma.$executeRawUnsafe(`ANALYZE measurements`);
|
||||
}, 120_000);
|
||||
|
||||
async function series(userId: string, days: number) {
|
||||
await session(userId);
|
||||
const { GET } = await import("@/app/api/measurements/series/route");
|
||||
const res = await GET(
|
||||
new NextRequest(
|
||||
`http://localhost/api/measurements/series?kind=glucose&days=${days}`,
|
||||
),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
return (
|
||||
(await res.json()) as {
|
||||
data: {
|
||||
unit: string;
|
||||
points: Array<Record<string, unknown> & { id: string; at: string }>;
|
||||
stats: {
|
||||
count: number;
|
||||
mean: number;
|
||||
min: number;
|
||||
max: number;
|
||||
stdDev: number;
|
||||
};
|
||||
};
|
||||
}
|
||||
).data;
|
||||
}
|
||||
|
||||
describe("per-kind glucose series inside the raw window", () => {
|
||||
it("sends a sensor stream as hour buckets in the user's unit, statistics over every reading", async () => {
|
||||
const data = await series(DENSE, 90);
|
||||
const rows = await prisma.measurement.findMany({
|
||||
where: { userId: DENSE, type: "BLOOD_GLUCOSE" },
|
||||
select: { value: true, measuredAt: true },
|
||||
});
|
||||
expect(rows).toHaveLength(DENSE_ROWS);
|
||||
|
||||
// Forty days of hours, give or take the partial hours at either end.
|
||||
expect(data.points.length).toBeLessThanOrEqual(40 * 24 + 2);
|
||||
expect(data.points.length).toBeGreaterThanOrEqual(40 * 24 - 2);
|
||||
expect(data.points.every((p) => p.id.startsWith("hour:"))).toBe(true);
|
||||
expect(data.unit).toBe("mmol/L");
|
||||
|
||||
// Every bucket is the mean of its hour's readings, converted once. New
|
||||
// York sits a whole number of hours off UTC, so its local hours are UTC
|
||||
// hours.
|
||||
const byHour = new Map<number, number[]>();
|
||||
for (const r of rows) {
|
||||
const hour = Math.floor(r.measuredAt.getTime() / 3_600_000) * 3_600_000;
|
||||
(byHour.get(hour) ?? byHour.set(hour, []).get(hour)!).push(r.value);
|
||||
}
|
||||
for (const p of data.points) {
|
||||
const values = byHour.get(new Date(p.at).getTime());
|
||||
expect(values, p.at).toBeDefined();
|
||||
const mean = values!.reduce((a, b) => a + b, 0) / values!.length;
|
||||
expect(p).toEqual({
|
||||
id: `hour:${p.at}`,
|
||||
at: p.at,
|
||||
value: convertGlucose(mean, "mmol/L"),
|
||||
secondary: null,
|
||||
});
|
||||
}
|
||||
|
||||
const values = rows.map((r) => r.value);
|
||||
const mean = values.reduce((a, b) => a + b, 0) / values.length;
|
||||
expect(data.stats.count).toBe(DENSE_ROWS);
|
||||
expect(data.stats.min).toBe(convertGlucose(Math.min(...values), "mmol/L"));
|
||||
expect(data.stats.max).toBe(convertGlucose(Math.max(...values), "mmol/L"));
|
||||
expect(data.stats.mean).toBe(
|
||||
convertGlucose(Math.round(mean * 100) / 100, "mmol/L"),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps fingerstick readings raw, one point per reading", async () => {
|
||||
const data = await series(SPARSE, 90);
|
||||
const expected = sparseReadings()
|
||||
.map((r, i) => ({
|
||||
id: `fs${String(i).padStart(4, "0")}`,
|
||||
at: r.at.toISOString(),
|
||||
value: r.value,
|
||||
secondary: null,
|
||||
}))
|
||||
.sort((a, b) => a.at.localeCompare(b.at));
|
||||
expect(data.unit).toBe("mg/dL");
|
||||
expect(data.points).toEqual(expected);
|
||||
expect(data.stats.count).toBe(expected.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe("doctor report keeps glucose raw inside the raw window", () => {
|
||||
it("computes the clinical panel from every raw reading", async () => {
|
||||
const end = new Date();
|
||||
const start = new Date(end.getTime() - 60 * 86_400_000);
|
||||
const data = await collectDoctorReportData(
|
||||
DENSE,
|
||||
{ start, end, days: 60 },
|
||||
selectionFromLeaves(["BLOOD_GLUCOSE", "GLUCOSE_PANEL"]),
|
||||
);
|
||||
const rows = await prisma.measurement.findMany({
|
||||
where: {
|
||||
userId: DENSE,
|
||||
type: "BLOOD_GLUCOSE",
|
||||
measuredAt: { gte: start, lte: end },
|
||||
},
|
||||
orderBy: { measuredAt: "asc" },
|
||||
select: { value: true, measuredAt: true },
|
||||
});
|
||||
expect(data.measurements.BLOOD_GLUCOSE).toHaveLength(rows.length);
|
||||
expect(data.glucoseClinical).toEqual(
|
||||
computeGlucoseClinicalMetrics(
|
||||
rows.map((r) => ({ measuredAt: r.measuredAt, mgdl: r.value })),
|
||||
{ windowDays: 60, now: end },
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user