feat(api): batched rollup-backed series endpoint to coalesce dashboard fetches

Add GET /api/measurements/series-batch returning every requested
MeasurementType's daily series in one response, so the dashboard chart
row stops issuing one self-fetch per chart. Extract the rollup-daily read
(per-source collapse, coverage probe + inline fold, live-SQL fallback)
into a shared readDailySeries reader so the single-type route and the
batch route read through the same path with no SQL duplication; the
single route now delegates to it. Cap the batch's per-type reads with
p-limit to protect the shared pool. Sleep stays on its per-night /series
adapter and is not batched.

HealthChart gains an optional preloadedSeries prop: when the dashboard
hands it a covered slice for the daily window it folds those rows instead
of self-fetching, keyed so a range-tab change re-fetches cleanly. Non-
dashboard mounts are unchanged (empty discriminator keeps the cache tuple
byte-identical). Register the endpoint in the OpenAPI table and regen the
spec; add unit + parity coverage for the batch route.
This commit is contained in:
Marc-André Bombeck committed 2026-06-18 15:29:20 +02:00
1 parent bd80557749
commit 143aefc709
9 files changed
+1016 -208

No files matched your search

+21 -186
View File
@@ -33,12 +33,9 @@ import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy";
import {
recomputeBucketsForMeasurement,
collapseToTypeDayKeys,
recomputeUserRollups,
} from "@/lib/rollups/measurement-rollups";
import {
collapseRollupRowsBySource,
loadUserSourcePriority,
} from "@/lib/rollups/measurement-read";
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
import { readDailySeries } from "@/lib/measurements/daily-series-read";
import {
reconstructSleepSessions,
pickMainNightAndNaps,
@@ -350,186 +347,23 @@ export const GET = apiHandler(async (request: NextRequest) => {
// populator hasn't caught up still sees a correct chart on its
// first render.
if (source === "rollup" && aggregate === "daily" && type && from && to) {
// v1.18.6 — the rollup-daily read (per-source collapse + coverage
// probe + inline fold + live-SQL fallback) moved to the shared
// `readDailySeries` reader so the batched series endpoint
// (`/api/measurements/series-batch`) reads through the SAME path
// instead of duplicating this SQL. Behaviour is unchanged: the rows
// returned here are byte-identical with the pre-extraction branch.
const cap = Math.min(limit, BUCKET_CAP.daily);
// v1.11.1 — rollup rows are per source; collapse overlapping sources to the
// ladder-canonical reading per day before the chart consumes them, so a
// dual-source vital paints one line, not two stacked points per day.
const priorityJson = await loadUserSourcePriority(user.id);
const rollupRows = collapseRollupRowsBySource(
await prisma.measurementRollup.findMany({
where: {
userId: user.id,
type: type as MeasurementType,
granularity: "DAY",
bucketStart: { gte: from, lte: to },
},
orderBy: { bucketStart: "asc" },
// Fetch beyond the bucket cap pre-collapse — N sources per day can
// exceed `cap` rows before the per-day collapse reduces them.
select: {
type: true,
source: true,
bucketStart: true,
mean: true,
count: true,
// v1.4.39 W-SUM — the writer now populates `sum_value` on
// every fold. The cumulative metric path consumes the column
// directly instead of reconstructing `mean * count`; the
// legacy fallback below covers the boot-backfill convergence
// window when an existing bucket pre-dates v1.4.39.
sumValue: true,
minValue: true,
maxValue: true,
},
}),
type as MeasurementType,
const measurements = await readDailySeries({
userId: user.id,
type: type as MeasurementType,
from,
to,
limit,
priorityJson,
).slice(0, cap);
// v1.4.39.2 — coverage-mismatch fallback. v1.4.39.1 wired the
// rollup write hook into the previously-bypassed Withings sync,
// /api/import, and admin-restore paths, and widened the boot-time
// backfill discovery to per-(user, type, day). Together those
// changes ensure new writes and post-deploy restarts converge the
// rollup tier for every measurement type the user has logged. They
// do NOT, however, guarantee convergence inside a single request
// when the boot backfill has not yet finished folding the user's
// historic rows: a chart query lands first, the rollup table still
// carries 2 rows for a 30-day BP_SYS window, and the route's pre-
// v1.4.39.2 "rollup wins on length > 0" short-circuit returns the
// sparse rollup. The chart trips its `< 3 daily points` empty-state
// even though `measurements` holds the full 30 days.
//
// This branch closes that gap. When the rollup row count is
// suspiciously low for the requested window (≥ 7 days asked and
// < 3 rollup rows present), we probe `measurements` for the actual
// distinct-day count over the same window. If `measurements`
// carries strictly more days than the rollup, the rollup is
// under-converged: we fold the (user, type, DAY, [from, to])
// partition inline, re-read the rollup, and return the refreshed
// rows. The inline fold is bounded by the window + type so it is
// small (~30 buckets × one type for the dashboard chart's 30-day
// window) and finishes well inside the request budget.
//
// The probe runs only on suspicious rollups so the covered-tenant
// happy path stays a single indexed read. The maintainer's tenant on a 30-day
// BP_SYS window pre-fix returned 2 rollup rows and hit `total: 2`
// in the live trace; post-fix the probe finds ~30 distinct days in
// `measurements`, folds them, and the chart paints all 30.
const windowMs = to.getTime() - from.getTime();
const windowDays = Math.ceil(windowMs / 86_400_000);
const SUSPICIOUS_ROW_FLOOR = 3;
const SUSPICIOUS_WINDOW_DAYS = 7;
let effectiveRows = rollupRows;
let coverageFallbackFired = false;
if (
rollupRows.length > 0 &&
rollupRows.length < SUSPICIOUS_ROW_FLOOR &&
windowDays >= SUSPICIOUS_WINDOW_DAYS
) {
// Cheap probe — one indexed range scan over
// `(user_id, type, measured_at)`. Returns the count of distinct
// calendar days the live table holds for the window.
const liveDayCountRows = await prisma.$queryRaw<Array<{ days: bigint }>>`
SELECT COUNT(DISTINCT date_trunc('day', m."measured_at"))::bigint AS days
FROM measurements m
WHERE m."user_id" = ${user.id}
AND m."type" = ${type}::measurement_type
AND m."measured_at" >= ${from}
AND m."measured_at" <= ${to}
AND m."deleted_at" IS NULL
`;
const liveDays = Number(liveDayCountRows[0]?.days ?? BigInt(0));
if (liveDays > rollupRows.length) {
coverageFallbackFired = true;
try {
// Inline fold of the (user, type, DAY, window) partition.
// Bounded by the window so it stays small even on a
// power-user account; the upsert is idempotent against the
// composite primary key.
await recomputeUserRollups(user.id, {
types: [type as MeasurementType],
granularities: ["DAY"],
from,
to,
});
// Re-read so this request returns the converged rows. The
// next chart paint hits the same warm rollup without paying
// the fallback cost. v1.11.1 — collapse per-source rows to the
// canonical reading per day, same as the initial read.
effectiveRows = collapseRollupRowsBySource(
await prisma.measurementRollup.findMany({
where: {
userId: user.id,
type: type as MeasurementType,
granularity: "DAY",
bucketStart: { gte: from, lte: to },
},
orderBy: { bucketStart: "asc" },
select: {
type: true,
source: true,
bucketStart: true,
mean: true,
count: true,
sumValue: true,
minValue: true,
maxValue: true,
},
}),
type as MeasurementType,
priorityJson,
).slice(0, cap);
} catch (err) {
// Best-effort — fall back to the legacy live aggregate path
// below if the fold itself failed so the chart paints
// something sensible regardless. `annotate` carries the
// surface so ops can spot a silent populator regression.
const message = err instanceof Error ? err.message : String(err);
annotate({
meta: {
rollup_coverage_fallback_failed: true,
rollup_coverage_fallback_error: message,
type,
},
});
// Drop into the live `date_trunc` branch below.
effectiveRows = [];
}
}
}
if (effectiveRows.length > 0) {
// Cumulative metrics (steps, active energy, distance, flights,
// daylight) store `mean` per bucket but the chart needs the
// daily SUM. v1.4.39 W-SUM — read `sumValue` directly; fall back
// to `mean * count` only when the legacy NULL surfaces (pre-
// v1.4.39 rows still waiting for the boot-time backfill to
// converge). `mean * count` is algebraically equivalent to
// SUM(value) for a single-source day because AVG = SUM / COUNT.
const useSum =
type != null && CUMULATIVE_HK_TYPES.has(type as MeasurementType);
const measurements = effectiveRows.map((r) => ({
type: r.type,
value: useSum ? (r.sumValue ?? r.mean * r.count) : r.mean,
measuredAt: r.bucketStart.toISOString(),
count: r.count,
// v1.8.5 — per-day min / max so the chart can shade an
// Apple-Health-style range band around the daily mean line.
// Omitted for cumulative metrics (steps, distance, …) where the
// daily value is a SUM, not a central tendency, so an intra-day
// spread band carries no meaning.
minValue: useSum ? undefined : r.minValue,
maxValue: useSum ? undefined : r.maxValue,
}));
annotate({
action: { name: "measurement.list" },
meta: {
total: measurements.length,
type,
aggregate: "daily",
source: "rollup",
coverage_fallback: coverageFallbackFired,
},
});
});
if (measurements.length > 0) {
return apiSuccess({
measurements,
meta: {
@@ -540,10 +374,11 @@ export const GET = apiHandler(async (request: NextRequest) => {
},
});
}
// Fall through to the live aggregate path when the rollup is
// empty for the requested window — covers brand-new accounts +
// the race between a recent write hook and the read, and the
// coverage-fallback inline-fold-failed branch above.
// `readDailySeries` already falls back to the live `date_trunc`
// aggregate on a rollup-coverage miss, so an empty result here means
// the window genuinely holds no rows. Fall through to the shared
// live aggregate branch below to keep the response shape identical
// for the truly-empty case.
}
if (aggregate && aggregate !== "raw" && from && to) {
@@ -0,0 +1,120 @@
/**
* v1.18.6 — `GET /api/measurements/series-batch` unit + parity tests.
*
* The endpoint reads each requested type through the SHARED
* `readDailySeries` reader — the exact same reader the single-type
* `GET /api/measurements?aggregate=daily&source=rollup` route delegates
* to (see `src/app/api/measurements/route.ts`). The parity guarantee is
* therefore structural: both routes call `readDailySeries`, so the rows
* are identical by construction. These tests pin that the batch route
* (a) validates its query, (b) calls the reader once per requested type
* with the right window, and (c) returns each type's reader output
* verbatim under `series[type]`.
*/
import { describe, it, expect, vi, beforeEach } from "vitest";
import { NextRequest } from "next/server";
vi.mock("@/lib/auth/session", () => ({ getSession: vi.fn() }));
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() }));
vi.mock("@/lib/logging/context", async (importOriginal) => {
const actual =
await importOriginal<typeof import("@/lib/logging/context")>();
return { ...actual, annotate: vi.fn() };
});
vi.mock("next/headers", () => ({
headers: vi.fn(async () => ({ get: () => null })),
cookies: vi.fn(async () => ({
get: () => undefined,
set: () => {},
delete: () => {},
})),
}));
vi.mock("@/lib/rollups/measurement-read", () => ({
loadUserSourcePriority: vi.fn(async () => null),
}));
vi.mock("@/lib/measurements/daily-series-read", () => ({
readDailySeries: vi.fn(),
}));
import { GET } from "../route";
import { getSession } from "@/lib/auth/session";
import { readDailySeries } from "@/lib/measurements/daily-series-read";
const SESSION_OK = {
session: { id: "s1", expiresAt: new Date(Date.now() + 3_600_000) },
user: { id: "user-1", username: "u", role: "USER" as const },
};
const FROM = "2026-05-01T00:00:00.000Z";
const TO = "2026-05-31T23:59:59.000Z";
function req(query: string): NextRequest {
return new NextRequest(
`http://localhost/api/measurements/series-batch?${query}`,
);
}
beforeEach(() => {
vi.resetAllMocks();
});
describe("GET /api/measurements/series-batch", () => {
it("returns 401 when unauthenticated", async () => {
vi.mocked(getSession).mockResolvedValue(null);
const res = await GET(req(`types=WEIGHT&from=${FROM}&to=${TO}`));
expect(res.status).toBe(401);
});
it("returns 422 when types is missing", async () => {
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
const res = await GET(req(`from=${FROM}&to=${TO}`));
expect(res.status).toBe(422);
});
it("returns 422 for an unknown type", async () => {
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
const res = await GET(req(`types=GARBAGE&from=${FROM}&to=${TO}`));
expect(res.status).toBe(422);
});
it("reads each type through readDailySeries and keys the result by type", async () => {
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
const weightRows = [
{ type: "WEIGHT", value: 80, measuredAt: FROM, count: 1 },
];
const bpRows = [
{ type: "BLOOD_PRESSURE_SYS", value: 126, measuredAt: FROM, count: 1 },
];
vi.mocked(readDailySeries).mockImplementation(async ({ type }) => {
if (type === "WEIGHT") return weightRows as never;
if (type === "BLOOD_PRESSURE_SYS") return bpRows as never;
return [] as never;
});
const res = await GET(
req(`types=WEIGHT,BLOOD_PRESSURE_SYS&from=${FROM}&to=${TO}`),
);
expect(res.status).toBe(200);
const body = await res.json();
expect(body.data.series.WEIGHT).toEqual(weightRows);
expect(body.data.series.BLOOD_PRESSURE_SYS).toEqual(bpRows);
// One reader call per unique type, with the parsed window.
expect(vi.mocked(readDailySeries)).toHaveBeenCalledTimes(2);
const call = vi.mocked(readDailySeries).mock.calls[0][0];
expect(call.userId).toBe("user-1");
expect(call.from.toISOString()).toBe(FROM);
expect(call.to.toISOString()).toBe(TO);
});
it("de-dupes a repeated type to one reader call", async () => {
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
vi.mocked(readDailySeries).mockResolvedValue([] as never);
const res = await GET(req(`types=WEIGHT,WEIGHT&from=${FROM}&to=${TO}`));
expect(res.status).toBe(200);
expect(vi.mocked(readDailySeries)).toHaveBeenCalledTimes(1);
});
});
@@ -0,0 +1,99 @@
/**
* v1.18.6 — GET /api/measurements/series-batch
*
* Batched, rollup-backed daily-series endpoint. Returns ALL requested
* measurement types in ONE response so the dashboard's chart row does a
* single round-trip instead of one self-fetch per chart (audit finding
* #2 — the per-chart fan-out).
*
* Query: `types` (comma-list of `MeasurementType`), `from`, `to`. Always
* reads `aggregate=daily` through the rollup tier (`source=rollup`) via
* the shared `readDailySeries` reader — the SAME path
* `GET /api/measurements?aggregate=daily&source=rollup` uses, so the
* per-type series are byte-identical between the two endpoints. Internal
* per-type reads are capped with `p-limit` to protect the shared Prisma
* pool.
*
* Sleep is deliberately NOT included: SLEEP_DURATION rides a per-night
* stage reconstruction (`/api/measurements/series?kind=sleep`), not the
* rollup tier, so the dashboard sleep chart stays on its existing
* `/series` call. Passing `SLEEP_DURATION` here yields an empty series
* for that key (the rollup carries no useful per-night collapse) — the
* client keeps sleep on the dedicated route.
*/
import { NextRequest } from "next/server";
import pLimit from "p-limit";
import { apiHandler, requireAuth } from "@/lib/api-handler";
import { returnAllZodIssues } from "@/lib/api-response";
import { apiSuccess } from "@/lib/api-response";
import { annotate } from "@/lib/logging/context";
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
import {
readDailySeries,
type DailySeriesRow,
} from "@/lib/measurements/daily-series-read";
import {
seriesBatchQuerySchema,
SERIES_BATCH_MAX_TYPES,
} from "@/lib/validations/series-batch";
import type { MeasurementType } from "@/generated/prisma/client";
/**
* Concurrency cap for the per-type rollup reads. Mirrors the W-POOL
* `p-limit(4)` discipline applied across the analytics fan-out so a
* wide dashboard never packs the shared Prisma pool.
*/
const SERIES_BATCH_CONCURRENCY = 4;
export const GET = apiHandler(async (request: NextRequest) => {
const { user } = await requireAuth();
const params = Object.fromEntries(request.nextUrl.searchParams);
const parsed = seriesBatchQuerySchema.safeParse(params);
if (!parsed.success) {
return returnAllZodIssues(parsed.error, 422);
}
const { types, from, to } = parsed.data;
// De-dupe while preserving order so a caller repeating a type doesn't
// double-read it.
const uniqueTypes = Array.from(new Set(types)) as MeasurementType[];
// Load the source-priority ladder ONCE and thread it across every type
// so the batched reader doesn't re-query it per type.
const priorityJson = await loadUserSourcePriority(user.id);
const limit = pLimit(SERIES_BATCH_CONCURRENCY);
const entries = await Promise.all(
uniqueTypes.map((type) =>
limit(async () => {
const rows = await readDailySeries({
userId: user.id,
type,
from,
to,
priorityJson,
});
return [type, rows] as const;
}),
),
);
const series: Record<string, DailySeriesRow[]> = {};
for (const [type, rows] of entries) {
series[type] = rows;
}
annotate({
action: { name: "measurement.series.batch" },
meta: {
type_count: uniqueTypes.length,
total: entries.reduce((sum, [, rows]) => sum + rows.length, 0),
},
});
return apiSuccess({ series });
});
// Re-export the cap so the OpenAPI table + tests share the constant.
export { SERIES_BATCH_MAX_TYPES };