mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 07:13:29 +00:00
fix(measurements): fold a dense pulse stream inside the 90-day raw window
Past 90 days every pulse reader already folded in SQL. Inside it, three still read every raw heart-rate row, which on a watch that records a reading a minute is 130 000 rows per read (#1023): - The insights page's resting-pulse proxy, used when an account has no resting heart rate, walked the whole window to take one percentile per day and held the event loop for over a second per view. It now folds in Postgres (`readRestingPulseProxy`) with the same percentile, rounding, minimum-sample rule and day anchor. - The per-kind series sent every reading: 14.7 MB for 90 days. A pulse window with more than 10 000 readings is now bucketed per local hour, mean plus low/high band, with stats still over the readings. - The doctor report drew every reading into one sparkline. A pulse window over the same cap now takes the per-day path longer reports already use. Sparse pulse (a cuff, hourly imports, a watch sampling every few minutes) stays under the cap and keeps its raw shape.
This commit is contained in:
1 parent
821c2eb884
commit
0e991c3640
10 files changed
+474
-32
No files matched your search
@@ -48,6 +48,12 @@ vi.mock("@/lib/rollups/mood-rollups", async () => {
|
||||
|
||||
vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() }));
|
||||
|
||||
// The raw-PULSE resting proxy folds in SQL; against the mocked client it is an
|
||||
// account with no pulse.
|
||||
vi.mock("@/lib/analytics/resting-pulse-read", () => ({
|
||||
readRestingPulseProxy: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/auth/audit", () => ({
|
||||
auditLog: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
@@ -107,6 +113,7 @@ import { getSession } from "@/lib/auth/session";
|
||||
import { requireModuleEnabled } from "@/lib/modules/gate";
|
||||
import { aiCapabilityToServe } from "@/lib/ai/capabilities/gate";
|
||||
import { probeProviderPresence } from "@/lib/ai/provider";
|
||||
import { readRestingPulseProxy } from "@/lib/analytics/resting-pulse-read";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { buildComprehensiveAggregate } from "@/lib/insights/comprehensive-aggregator";
|
||||
import { checkAnalyticsReadRateLimit } from "@/lib/rate-limit";
|
||||
@@ -155,8 +162,9 @@ beforeEach(() => {
|
||||
dateOfBirth: new Date("1985-01-01"),
|
||||
});
|
||||
(prisma.moodEntry.findMany as ReturnType<typeof vi.fn>).mockResolvedValue([]);
|
||||
// The resting-pulse pairing (M-CS2) reads RESTING_HEART_RATE rows (and the
|
||||
// raw-PULSE proxy only when none exist) straight from the measurement table.
|
||||
vi.mocked(readRestingPulseProxy).mockResolvedValue([]);
|
||||
// The resting-pulse pairing (M-CS2) reads RESTING_HEART_RATE rows straight
|
||||
// from the measurement table (the raw-PULSE proxy is mocked above).
|
||||
(prisma.measurement.findMany as ReturnType<typeof vi.fn>).mockResolvedValue(
|
||||
[],
|
||||
);
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
} from "@/lib/analytics/compliance";
|
||||
import { getMedicationCategories } from "@/lib/medication-category";
|
||||
import { resolveRestingPulseSeries } from "@/lib/analytics/resting-pulse";
|
||||
import { readRestingPulseProxy } from "@/lib/analytics/resting-pulse-read";
|
||||
import {
|
||||
apiHandler,
|
||||
requireRecordAuth,
|
||||
@@ -341,24 +342,24 @@ export async function buildComprehensiveResponse(user: AuthedUser) {
|
||||
},
|
||||
select: { measuredAt: true, value: true },
|
||||
});
|
||||
const rawPulseRowsForResting =
|
||||
restingPulseRows.length === 0
|
||||
? await prisma.measurement.findMany({
|
||||
where: {
|
||||
userId,
|
||||
type: "PULSE",
|
||||
deletedAt: null,
|
||||
measuredAt: { gte: ninetyDaysAgo },
|
||||
},
|
||||
select: { measuredAt: true, value: true },
|
||||
})
|
||||
: [];
|
||||
//
|
||||
// #1023 — the proxy folds in Postgres. It needs one percentile per day, and
|
||||
// reading the raw samples to get it walked 130 000 rows per page view on a
|
||||
// watch that records once a minute, blocking the event loop for over a
|
||||
// second. `readRestingPulseProxy` returns the same series one row per day.
|
||||
const localDayKey = (d: Date) => userDayKey(d, userTz);
|
||||
const { series: restingPulseSeries } = resolveRestingPulseSeries({
|
||||
restingSamples: restingPulseRows,
|
||||
pulseSamples: rawPulseRowsForResting,
|
||||
dayKeyOf: localDayKey,
|
||||
});
|
||||
const restingPulseSeries =
|
||||
restingPulseRows.length > 0
|
||||
? resolveRestingPulseSeries({
|
||||
restingSamples: restingPulseRows,
|
||||
pulseSamples: [],
|
||||
dayKeyOf: localDayKey,
|
||||
}).series
|
||||
: await readRestingPulseProxy({
|
||||
userId,
|
||||
since: ninetyDaysAgo,
|
||||
timeZone: userTz,
|
||||
});
|
||||
const restingByDay = new Map<string, { sum: number; count: number }>();
|
||||
for (const s of restingPulseSeries) {
|
||||
const key = localDayKey(s.measuredAt);
|
||||
|
||||
@@ -3,7 +3,7 @@ import { NextRequest } from "next/server";
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
measurement: { findMany: vi.fn() },
|
||||
measurement: { findMany: vi.fn(), count: vi.fn() },
|
||||
user: { findUnique: vi.fn() },
|
||||
auditLog: { create: vi.fn() },
|
||||
// v1.28.25 — dense kinds (glucose / pulse) day-bucket in SQL for
|
||||
@@ -68,6 +68,7 @@ function req(query: string): NextRequest {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.measurement.count).mockResolvedValue(0 as never);
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never);
|
||||
vi.mocked(prisma.auditLog.create).mockResolvedValue({} as never);
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({
|
||||
@@ -586,6 +587,72 @@ describe("GET /api/measurements/series — dense-kind day-bucketing (v1.28.25)",
|
||||
expect(body.data.points[0].id).toBe("g1");
|
||||
});
|
||||
|
||||
it("hour-buckets a pulse window too dense to send raw (#1023)", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
// One reading a minute for 30 days.
|
||||
vi.mocked(prisma.measurement.count).mockResolvedValue(43_200 as never);
|
||||
vi.mocked(prisma.$queryRaw)
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
bucket_start: new Date("2026-06-01T08:00:00Z"),
|
||||
mean: 71.666,
|
||||
min_value: 58,
|
||||
max_value: 96,
|
||||
},
|
||||
] as never)
|
||||
.mockResolvedValueOnce([
|
||||
{ n: 43_200, mean: 72.1, min: 50, max: 150, sd: 11.5 },
|
||||
] as never);
|
||||
|
||||
const res = await GET(req("kind=pulse&days=30"));
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
data: {
|
||||
points: Array<Record<string, unknown>>;
|
||||
stats: { count: number; mean: number };
|
||||
};
|
||||
};
|
||||
expect(prisma.measurement.findMany).not.toHaveBeenCalled();
|
||||
expect(body.data.points).toEqual([
|
||||
{
|
||||
id: "hour:2026-06-01T08:00:00.000Z",
|
||||
at: "2026-06-01T08:00:00.000Z",
|
||||
value: 71.67,
|
||||
secondary: null,
|
||||
valueMin: 58,
|
||||
valueMax: 96,
|
||||
},
|
||||
]);
|
||||
// Stats describe the readings, not the buckets.
|
||||
expect(body.data.stats.count).toBe(43_200);
|
||||
expect(body.data.stats.mean).toBe(72.1);
|
||||
// The bucket read folds per hour, not per day.
|
||||
const bucketSql = vi.mocked(prisma.$queryRaw).mock.calls[0] as unknown[];
|
||||
expect(bucketSql.slice(1)).toContain("hour");
|
||||
});
|
||||
|
||||
it("keeps a pulse window at or under the row cap raw", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.measurement.count).mockResolvedValue(10_000 as never);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
{
|
||||
id: "p1",
|
||||
value: 64,
|
||||
measuredAt: new Date("2026-06-01T08:03:00Z"),
|
||||
valueMin: null,
|
||||
valueMax: null,
|
||||
},
|
||||
] as never);
|
||||
const res = await GET(req("kind=pulse&days=90"));
|
||||
const body = (await res.json()) as {
|
||||
data: { points: Array<{ id: string; at: string }> };
|
||||
};
|
||||
expect(prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
expect(body.data.points).toEqual([
|
||||
expect.objectContaining({ id: "p1", at: "2026-06-01T08:03:00.000Z" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps sparse kinds raw even on the ten-year window", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue([
|
||||
|
||||
@@ -51,6 +51,21 @@ const SLEEP_SERIES_MAX_DAYS = 365;
|
||||
const DENSE_SERIES_RAW_WINDOW_DAYS = 90;
|
||||
const DENSE_SERIES_KINDS: ReadonlySet<string> = new Set(["glucose", "pulse"]);
|
||||
|
||||
/**
|
||||
* #1023 — inside the raw window, a pulse series with more rows than this is
|
||||
* bucketed per local hour in SQL instead of sent raw.
|
||||
*
|
||||
* A watch that records heart rate once a minute puts 43 000 rows in the
|
||||
* default 30-day window and 130 000 in 90 days: a 15 MB response the client
|
||||
* could not draw at that resolution anyway, built from as many objects on the
|
||||
* server. Hourly buckets carry the hour's mean with its low/high band, the
|
||||
* shape an hourly-bucket import already has, and cap the answer at 24 points
|
||||
* a day. The cap sits far above what sparse sources produce (a cuff, hourly
|
||||
* buckets over 90 days, a watch sampling every five minutes over 30 days), so
|
||||
* those series stay raw and unchanged.
|
||||
*/
|
||||
const DENSE_SERIES_RAW_ROW_CAP = 10_000;
|
||||
|
||||
const kindEnum = z.enum([
|
||||
"weight",
|
||||
"bloodPressure",
|
||||
@@ -352,9 +367,20 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
};
|
||||
});
|
||||
} else if (
|
||||
DENSE_SERIES_KINDS.has(kind) &&
|
||||
days > DENSE_SERIES_RAW_WINDOW_DAYS
|
||||
(DENSE_SERIES_KINDS.has(kind) && days > DENSE_SERIES_RAW_WINDOW_DAYS) ||
|
||||
(kind === "pulse" &&
|
||||
(await prisma.measurement.count({
|
||||
where: {
|
||||
userId: user.id,
|
||||
type: "PULSE",
|
||||
measuredAt: { gte: since },
|
||||
deletedAt: null,
|
||||
},
|
||||
})) > DENSE_SERIES_RAW_ROW_CAP)
|
||||
) {
|
||||
// Day buckets past the raw window; hour buckets for a pulse stream too
|
||||
// dense to send raw inside it.
|
||||
const grain = days > DENSE_SERIES_RAW_WINDOW_DAYS ? "day" : "hour";
|
||||
// v1.28.25 — long-window read of a sample-dense kind (CGM glucose,
|
||||
// per-sample / hourly pulse). Day-bucket in SQL instead of walking
|
||||
// every raw row into JS: one aggregate pass in Postgres returns at
|
||||
@@ -377,7 +403,7 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
m."value_min",
|
||||
m."value_max",
|
||||
date_trunc(
|
||||
'day',
|
||||
${grain},
|
||||
(m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${userTz}
|
||||
) AS local_day
|
||||
FROM measurements m
|
||||
@@ -422,7 +448,10 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
AND m."deleted_at" IS NULL
|
||||
`;
|
||||
const round2 = (v: number) => Math.round(v * 100) / 100;
|
||||
const dayId = (d: Date) => `day:${userDayKey(d, userTz)}`;
|
||||
const dayId = (d: Date) =>
|
||||
grain === "day"
|
||||
? `day:${userDayKey(d, userTz)}`
|
||||
: `hour:${d.toISOString()}`;
|
||||
if (kind === "glucose") {
|
||||
// Same unit engine as the raw glucose branch: canonical mg/dL in
|
||||
// the DB, converted ONCE at serialization to the user's preference.
|
||||
@@ -457,9 +486,10 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
};
|
||||
}
|
||||
} else {
|
||||
// pulse — `value` is the day's average; valueMin/valueMax carry the
|
||||
// day's low/high band (folding each hourly bucket's own spread via
|
||||
// the COALESCE above), same band semantics as the per-hour shape.
|
||||
// pulse — `value` is the bucket's average; valueMin/valueMax carry the
|
||||
// bucket's low/high band (folding each hourly import bucket's own
|
||||
// spread via the COALESCE above), same band semantics as the per-hour
|
||||
// shape.
|
||||
points = bucketRows.map((r) => ({
|
||||
id: dayId(r.bucket_start),
|
||||
at: r.bucket_start.toISOString(),
|
||||
|
||||
Reference in new issue
Block a user