mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 20:42:43 +00:00
fix(measurements): cut chart day buckets at the user's midnight
Switching a chart from 7 to 30 days moved steps, active energy, flights and distance totals onto neighbouring days. The 7-day range buckets raw rows by the user's calendar day; the 30-day range read the DAY rollup tier, whose buckets are UTC days, and the chart labelled each bucket's UTC-midnight start in the user's zone. East of UTC a sample shortly after local midnight counted on the previous day; west of UTC every bucket read one day early. The daily series (the chart read and the batched dashboard read) and the aggregate=daily|weekly|monthly list now fold live in the user's zone and return the instant of local midnight, so every range and every summed metric shares one day boundary. The fold runs in Postgres and returns one row per day; about 0.1 s for 90 days of per-minute samples. Windows longer than the DAY cap still step up to the coarser rollup tiers. Refs #1026
This commit is contained in:
1 parent
01cd755de6
commit
88cea09c7b
6 files changed
+365
-702
No files matched your search
@@ -32,9 +32,6 @@ vi.mock("@/lib/db", () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
// v1.4.39.2 — the rollup branch's coverage-mismatch fallback may call
|
||||
// `recomputeUserRollups` inline. Mock it so tests can assert whether
|
||||
// the inline fold fired without exercising the real populator.
|
||||
vi.mock("@/lib/rollups/measurement-rollups", async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("@/lib/rollups/measurement-rollups")
|
||||
@@ -66,7 +63,7 @@ vi.mock("next/headers", () => ({
|
||||
import { GET } from "../route";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { getSession } from "@/lib/auth/session";
|
||||
import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
|
||||
import { invalidateUserTimezone } from "@/lib/tz/resolver";
|
||||
|
||||
const SESSION_OK = {
|
||||
session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) },
|
||||
@@ -221,318 +218,50 @@ describe("GET /api/measurements — all-time semantics (SD-H1)", () => {
|
||||
expect(months.size).toBe(24);
|
||||
});
|
||||
|
||||
it("source=rollup + aggregate=daily reads measurement_rollups without firing the heavy date_trunc query", async () => {
|
||||
// v1.4.36 W1 — Insights trends row routes the three daily chart
|
||||
// fetches through the persistent DAY buckets via `source=rollup`.
|
||||
// The heavy `date_trunc` $queryRaw must NOT fire on the happy
|
||||
// path where the rollup has rows for the requested window.
|
||||
const buckets = Array.from({ length: 30 }, (_, i) => ({
|
||||
type: "WEIGHT",
|
||||
bucketStart: new Date(Date.UTC(2026, 3, 16 + i, 0, 0, 0)),
|
||||
mean: 81 + i * 0.05,
|
||||
count: 5,
|
||||
// v1.11.1 — single source per day → collapse is identity.
|
||||
source: "APPLE_HEALTH",
|
||||
}));
|
||||
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue(
|
||||
buckets as never,
|
||||
);
|
||||
it("source=rollup + aggregate=daily folds live in the user's zone, never the UTC-day rollup (#1026)", async () => {
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValue({
|
||||
timezone: "Asia/Kolkata",
|
||||
sourcePriorityJson: null,
|
||||
} as never);
|
||||
// The resolver caches per user; drop what earlier cases left behind.
|
||||
invalidateUserTimezone("user-1");
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue([
|
||||
{
|
||||
type: "ACTIVITY_STEPS",
|
||||
bucket_start: new Date("2026-05-09T18:30:00.000Z"),
|
||||
avg: 8123,
|
||||
cnt: 96,
|
||||
min_value: 0,
|
||||
max_value: 400,
|
||||
},
|
||||
] as never);
|
||||
|
||||
const res = await GET(
|
||||
getRequest(
|
||||
"type=WEIGHT&from=2026-04-15T00:00:00Z&to=2026-05-15T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
|
||||
"type=ACTIVITY_STEPS&from=2026-04-15T00:00:00Z&to=2026-05-15T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
|
||||
),
|
||||
);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const json = (await res.json()) as {
|
||||
data: {
|
||||
measurements: Array<{ type: string; value: number; count: number }>;
|
||||
meta: { aggregate?: string };
|
||||
measurements: Array<{
|
||||
measuredAt: string;
|
||||
value: number;
|
||||
minValue?: number;
|
||||
}>;
|
||||
};
|
||||
};
|
||||
expect(prisma.measurementRollup.findMany).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
expect(json.data.measurements.length).toBe(30);
|
||||
// First bucket reflects the rollup mean — round-trip-stable.
|
||||
expect(json.data.measurements[0].type).toBe("WEIGHT");
|
||||
expect(json.data.measurements[0].value).toBeCloseTo(81, 5);
|
||||
expect(json.data.measurements[0].count).toBe(5);
|
||||
expect(json.data.meta.aggregate).toBe("daily");
|
||||
});
|
||||
|
||||
it("source=rollup cumulative path reads sum_value directly (v1.4.39 W-SUM)", async () => {
|
||||
// ACTIVITY_STEPS rollup row with sum_value populated. The route
|
||||
// must consume sumValue directly — NOT reconstruct from mean *
|
||||
// count. Use a sum that is NOT mean × count so the test
|
||||
// distinguishes the two paths.
|
||||
const buckets = Array.from({ length: 3 }, (_, i) => ({
|
||||
type: "ACTIVITY_STEPS",
|
||||
bucketStart: new Date(Date.UTC(2026, 4, 1 + i, 0, 0, 0)),
|
||||
mean: 2000, // would yield 10000 if multiplied by count=5
|
||||
count: 5,
|
||||
sumValue: 11000 + i * 250, // distinct from mean × count
|
||||
source: "APPLE_HEALTH",
|
||||
}));
|
||||
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue(
|
||||
buckets as never,
|
||||
);
|
||||
|
||||
const res = await GET(
|
||||
getRequest(
|
||||
"type=ACTIVITY_STEPS&from=2026-05-01T00:00:00Z&to=2026-05-05T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
|
||||
),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const json = (await res.json()) as {
|
||||
data: { measurements: Array<{ value: number }> };
|
||||
};
|
||||
expect(prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
expect(json.data.measurements.map((m) => m.value)).toEqual([
|
||||
11000, 11250, 11500,
|
||||
]);
|
||||
});
|
||||
|
||||
it("source=rollup cumulative path falls back to mean*count when sum_value is NULL (v1.4.39 W-SUM)", async () => {
|
||||
// Pre-v1.4.39 row — boot-backfill hasn't converged yet. The
|
||||
// route falls back to mean × count so the chart never paints
|
||||
// a hole during the convergence window.
|
||||
const buckets = [
|
||||
{
|
||||
type: "ACTIVITY_STEPS",
|
||||
bucketStart: new Date(Date.UTC(2026, 4, 1)),
|
||||
mean: 2000,
|
||||
count: 5,
|
||||
sumValue: null,
|
||||
source: "APPLE_HEALTH",
|
||||
},
|
||||
];
|
||||
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue(
|
||||
buckets as never,
|
||||
);
|
||||
|
||||
const res = await GET(
|
||||
getRequest(
|
||||
"type=ACTIVITY_STEPS&from=2026-05-01T00:00:00Z&to=2026-05-05T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
|
||||
),
|
||||
);
|
||||
const json = (await res.json()) as {
|
||||
data: { measurements: Array<{ value: number }> };
|
||||
};
|
||||
expect(json.data.measurements[0].value).toBe(10000);
|
||||
});
|
||||
|
||||
it("falls back to live date_trunc when source=rollup returns zero buckets", async () => {
|
||||
// Empty rollup ⇒ heavy aggregate runs so brand-new accounts still
|
||||
// see a correct chart on their first render.
|
||||
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue([] as never);
|
||||
const liveBuckets = Array.from({ length: 7 }, (_, i) => ({
|
||||
type: "WEIGHT",
|
||||
bucket_start: new Date(Date.UTC(2026, 4, 8 + i, 0, 0, 0)),
|
||||
avg: 81 + i * 0.05,
|
||||
cnt: 2,
|
||||
}));
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValue(liveBuckets as never);
|
||||
|
||||
const res = await GET(
|
||||
getRequest(
|
||||
"type=WEIGHT&from=2026-05-08T00:00:00Z&to=2026-05-15T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
|
||||
),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(prisma.measurementRollup.findMany).toHaveBeenCalledTimes(1);
|
||||
expect(prisma.$queryRaw).toHaveBeenCalledTimes(1);
|
||||
const json = (await res.json()) as {
|
||||
data: { measurements: Array<unknown> };
|
||||
};
|
||||
expect(json.data.measurements.length).toBe(7);
|
||||
});
|
||||
|
||||
it("source=rollup folds the partition inline and re-reads when the rollup is under-converged (v1.4.39.2)", async () => {
|
||||
// The maintainer's pre-fix production trace: 30-day BP_SYS window returned
|
||||
// `total: 2` from the rollup branch even though `measurements`
|
||||
// carried ~30 distinct days. The chart tripped its `< 3 daily
|
||||
// points` empty-state because the route short-circuited on
|
||||
// `rollupRows.length > 0` and never reconciled with the live
|
||||
// table. The coverage-mismatch fallback runs an inline
|
||||
// `recomputeUserRollups` for the (user, type, DAY, window)
|
||||
// partition and re-reads so the request returns the converged
|
||||
// rows.
|
||||
const sparseRollup = [
|
||||
{
|
||||
type: "BLOOD_PRESSURE_SYS",
|
||||
bucketStart: new Date(Date.UTC(2026, 4, 1, 0, 0, 0)),
|
||||
mean: 128,
|
||||
count: 1,
|
||||
sumValue: null,
|
||||
source: "APPLE_HEALTH",
|
||||
},
|
||||
{
|
||||
type: "BLOOD_PRESSURE_SYS",
|
||||
bucketStart: new Date(Date.UTC(2026, 4, 10, 0, 0, 0)),
|
||||
mean: 132,
|
||||
count: 1,
|
||||
sumValue: null,
|
||||
source: "APPLE_HEALTH",
|
||||
},
|
||||
];
|
||||
const convergedRollup = Array.from({ length: 28 }, (_, i) => ({
|
||||
type: "BLOOD_PRESSURE_SYS",
|
||||
bucketStart: new Date(Date.UTC(2026, 4, 1 + i, 0, 0, 0)),
|
||||
mean: 128 + (i % 5),
|
||||
count: 2,
|
||||
sumValue: null,
|
||||
source: "APPLE_HEALTH",
|
||||
}));
|
||||
// First read returns the sparse rows; second (post-fold) read
|
||||
// returns the converged rows.
|
||||
vi.mocked(prisma.measurementRollup.findMany)
|
||||
.mockResolvedValueOnce(sparseRollup as never)
|
||||
.mockResolvedValueOnce(convergedRollup as never);
|
||||
// Coverage probe — live `measurements` has 28 distinct days.
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([
|
||||
{ days: BigInt(28) },
|
||||
] as never);
|
||||
|
||||
const res = await GET(
|
||||
getRequest(
|
||||
"type=BLOOD_PRESSURE_SYS&from=2026-04-30T00:00:00Z&to=2026-05-30T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
|
||||
),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const json = (await res.json()) as {
|
||||
data: { measurements: Array<unknown> };
|
||||
};
|
||||
// The inline fold fired exactly once with the (type, DAY, window)
|
||||
// bound so the partition write stays small.
|
||||
expect(recomputeUserRollups).toHaveBeenCalledTimes(1);
|
||||
expect(recomputeUserRollups).toHaveBeenCalledWith(
|
||||
"user-1",
|
||||
expect.objectContaining({
|
||||
types: ["BLOOD_PRESSURE_SYS"],
|
||||
granularities: ["DAY"],
|
||||
}),
|
||||
);
|
||||
// The route re-read the rollup after the fold so the response
|
||||
// carries the converged 28 buckets — well above the chart's
|
||||
// `< 3 daily points` empty-state threshold.
|
||||
expect(prisma.measurementRollup.findMany).toHaveBeenCalledTimes(2);
|
||||
expect(json.data.measurements.length).toBe(28);
|
||||
});
|
||||
|
||||
it("source=rollup skips the coverage probe on a covered window (v1.4.39.2 hot-path)", async () => {
|
||||
// Happy case: 30-day window, 28 rollup rows present. The route
|
||||
// must short-circuit on the first read and skip the probe so the
|
||||
// covered-tenant hot path stays a single indexed read.
|
||||
const buckets = Array.from({ length: 28 }, (_, i) => ({
|
||||
type: "WEIGHT",
|
||||
bucketStart: new Date(Date.UTC(2026, 3, 16 + i, 0, 0, 0)),
|
||||
mean: 81 + i * 0.05,
|
||||
count: 2,
|
||||
sumValue: null,
|
||||
source: "APPLE_HEALTH",
|
||||
}));
|
||||
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue(
|
||||
buckets as never,
|
||||
);
|
||||
|
||||
const res = await GET(
|
||||
getRequest(
|
||||
"type=WEIGHT&from=2026-04-15T00:00:00Z&to=2026-05-15T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
|
||||
),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(prisma.measurementRollup.findMany).toHaveBeenCalledTimes(1);
|
||||
expect(recomputeUserRollups).not.toHaveBeenCalled();
|
||||
expect(prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("source=rollup skips the coverage fallback on short windows (v1.4.39.2)", async () => {
|
||||
// A 5-day window with 2 rollup rows is plausibly genuine sparsity
|
||||
// for a user who measures BP every other day. The coverage
|
||||
// fallback gates on `windowDays >= 7` so the probe + inline fold
|
||||
// never fire on these short windows.
|
||||
const sparseRollup = [
|
||||
{
|
||||
type: "BLOOD_PRESSURE_SYS",
|
||||
bucketStart: new Date(Date.UTC(2026, 4, 1, 0, 0, 0)),
|
||||
mean: 128,
|
||||
count: 1,
|
||||
sumValue: null,
|
||||
source: "APPLE_HEALTH",
|
||||
},
|
||||
{
|
||||
type: "BLOOD_PRESSURE_SYS",
|
||||
bucketStart: new Date(Date.UTC(2026, 4, 3, 0, 0, 0)),
|
||||
mean: 132,
|
||||
count: 1,
|
||||
sumValue: null,
|
||||
source: "APPLE_HEALTH",
|
||||
},
|
||||
];
|
||||
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue(
|
||||
sparseRollup as never,
|
||||
);
|
||||
|
||||
const res = await GET(
|
||||
getRequest(
|
||||
"type=BLOOD_PRESSURE_SYS&from=2026-05-01T00:00:00Z&to=2026-05-06T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
|
||||
),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(recomputeUserRollups).not.toHaveBeenCalled();
|
||||
expect(prisma.$queryRaw).not.toHaveBeenCalled();
|
||||
const json = (await res.json()) as {
|
||||
data: { measurements: Array<unknown> };
|
||||
};
|
||||
expect(json.data.measurements.length).toBe(2);
|
||||
});
|
||||
|
||||
it("source=rollup keeps the sparse rollup when live `measurements` agrees (v1.4.39.2)", async () => {
|
||||
// Genuine sparsity: the user measured BP twice in 30 days. The
|
||||
// probe finds the same 2 distinct days in `measurements` so the
|
||||
// route returns the 2 rollup rows without firing the inline
|
||||
// fold. The chart's `< 3 daily points` empty-state is the right
|
||||
// surface for this case — the data really is sparse.
|
||||
const sparseRollup = [
|
||||
{
|
||||
type: "BLOOD_PRESSURE_SYS",
|
||||
bucketStart: new Date(Date.UTC(2026, 4, 1, 0, 0, 0)),
|
||||
mean: 128,
|
||||
count: 1,
|
||||
sumValue: null,
|
||||
source: "APPLE_HEALTH",
|
||||
},
|
||||
{
|
||||
type: "BLOOD_PRESSURE_SYS",
|
||||
bucketStart: new Date(Date.UTC(2026, 4, 10, 0, 0, 0)),
|
||||
mean: 132,
|
||||
count: 1,
|
||||
sumValue: null,
|
||||
source: "APPLE_HEALTH",
|
||||
},
|
||||
];
|
||||
vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue(
|
||||
sparseRollup as never,
|
||||
);
|
||||
// Probe agrees — 2 distinct days in `measurements` too.
|
||||
vi.mocked(prisma.$queryRaw).mockResolvedValueOnce([
|
||||
{ days: BigInt(2) },
|
||||
] as never);
|
||||
|
||||
const res = await GET(
|
||||
getRequest(
|
||||
"type=BLOOD_PRESSURE_SYS&from=2026-04-30T00:00:00Z&to=2026-05-30T00:00:00Z&aggregate=daily&source=rollup&limit=5000",
|
||||
),
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
// Probe fired (suspicious-low rollup) but the inline fold did
|
||||
// not (live and rollup agree).
|
||||
expect(prisma.$queryRaw).toHaveBeenCalledTimes(1);
|
||||
expect(recomputeUserRollups).not.toHaveBeenCalled();
|
||||
const json = (await res.json()) as {
|
||||
data: { measurements: Array<unknown> };
|
||||
};
|
||||
expect(json.data.measurements.length).toBe(2);
|
||||
expect(prisma.measurementRollup.findMany).not.toHaveBeenCalled();
|
||||
// The zone travels as a bound parameter of the fold.
|
||||
const call = vi.mocked(prisma.$queryRaw).mock.calls[0] as unknown[];
|
||||
expect(call.slice(1)).toContain("Asia/Kolkata");
|
||||
// Local midnight of 10 May in Kolkata; a SUM carries no spread band.
|
||||
expect(json.data.measurements[0]).toMatchObject({
|
||||
measuredAt: "2026-05-09T18:30:00.000Z",
|
||||
value: 8123,
|
||||
});
|
||||
expect(json.data.measurements[0].minValue).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns a weekly series when the all-time window is under two years", async () => {
|
||||
|
||||
@@ -45,20 +45,21 @@ import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy";
|
||||
import { runSafetyFloorCheck } from "@/lib/illness/safety-floor-check";
|
||||
import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation";
|
||||
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
|
||||
import { readDailySeries } from "@/lib/measurements/daily-series-read";
|
||||
import {
|
||||
readDailySeries,
|
||||
readLiveBuckets,
|
||||
} from "@/lib/measurements/daily-series-read";
|
||||
import {
|
||||
reconstructSleepSessions,
|
||||
pickMainNightAndNaps,
|
||||
} from "@/lib/analytics/sleep-night";
|
||||
import { resolveUserTimezone } from "@/lib/tz/resolver";
|
||||
import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql";
|
||||
import { NextRequest } from "next/server";
|
||||
import type {
|
||||
MeasurementType,
|
||||
MeasurementSource,
|
||||
GlucoseContext,
|
||||
} from "@/generated/prisma/client";
|
||||
import { Prisma } from "@/generated/prisma/client";
|
||||
|
||||
export const GET = apiHandler(async (request: NextRequest) => {
|
||||
const { user } = await requireRecordAuth("read", "measurements");
|
||||
@@ -393,36 +394,12 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
// The route's response is bounded by the `BUCKET_CAP` ceiling per
|
||||
// grain (monthly: 24, weekly: 105, daily: 365) so a multi-decade
|
||||
// account never paints an unbounded series.
|
||||
// v1.4.36 W1 — rollup-table read for daily aggregates.
|
||||
//
|
||||
// The Insights trends row fires three parallel
|
||||
// `GET /api/measurements?type=…&aggregate=daily&limit=5000` requests
|
||||
// (BP_SYS / BP_DIA / WEIGHT). The legacy `date_trunc('day', …)`
|
||||
// path scanned the measurements table each time — a full year window
|
||||
// on the maintainer's account hit 3 × ~3 s on the v1.4.35 HAR. Reading from
|
||||
// the persistent `measurement_rollups` DAY buckets drops each call
|
||||
// to a small indexed read against the ~5 k-row rollup table.
|
||||
//
|
||||
// Gating:
|
||||
// - `source=rollup` is an explicit client opt-in (`@/lib/validations/measurement`
|
||||
// constrains the enum to `["rollup"]`).
|
||||
// - We require `aggregate=daily` because the rollup populator only
|
||||
// keeps DAY buckets on the synchronous write hook; the WEEK /
|
||||
// MONTH paths still depend on the async pg-boss recomputes which
|
||||
// may lag behind a recent write.
|
||||
// - `type` is mandatory so the response shape mirrors the per-type
|
||||
// ask the chart code makes.
|
||||
// - When the rollup window returns zero rows we fall through to
|
||||
// the live `date_trunc` path so a brand-new account whose
|
||||
// populator hasn't caught up still sees a correct chart on its
|
||||
// first render.
|
||||
// `source=rollup` + `aggregate=daily` — the chart's series read. Windows up
|
||||
// to the DAY cap fold live into the user's local days (#1026); wider
|
||||
// windows step up to the WEEK / MONTH / YEAR rollup tier. `type` is
|
||||
// mandatory so the response mirrors the per-type ask the chart makes.
|
||||
if (source === "rollup" && aggregate === "daily" && type && from && to) {
|
||||
// v1.18.6 — the rollup-daily read (per-source collapse + coverage
|
||||
// probe + inline fold + live-SQL fallback) moved to the shared
|
||||
// `readDailySeries` reader so the batched series endpoint
|
||||
// (`/api/measurements/series-batch`) reads through the SAME path
|
||||
// instead of duplicating this SQL. Behaviour is unchanged: the rows
|
||||
// returned here are byte-identical with the pre-extraction branch.
|
||||
// Shared with the batched series endpoint, so both read the same rows.
|
||||
const cap = Math.min(limit, BUCKET_CAP.daily);
|
||||
const priorityJson = await loadUserSourcePriority(user.id);
|
||||
const measurements = await readDailySeries({
|
||||
@@ -444,94 +421,29 @@ export const GET = apiHandler(async (request: NextRequest) => {
|
||||
},
|
||||
});
|
||||
}
|
||||
// `readDailySeries` already falls back to the live `date_trunc`
|
||||
// aggregate on a rollup-coverage miss, so an empty result here means
|
||||
// the window genuinely holds no rows. Fall through to the shared
|
||||
// live aggregate branch below to keep the response shape identical
|
||||
// for the truly-empty case.
|
||||
// An empty result means the window genuinely holds no rows. Fall
|
||||
// through to the shared live aggregate branch below to keep the
|
||||
// response shape identical for the truly-empty case.
|
||||
}
|
||||
|
||||
if (aggregate && aggregate !== "raw" && from && to) {
|
||||
const grain: AggregateGrain = aggregate;
|
||||
const grain: Exclude<AggregateGrain, "raw"> = aggregate;
|
||||
const cap = Math.min(limit, BUCKET_CAP[grain]);
|
||||
// Postgres `date_trunc` requires the unit argument to be a SQL
|
||||
// literal — it cannot be supplied as a prepared-statement parameter.
|
||||
// The previous `${truncUnit}` (bound) interpolation 500'd in
|
||||
// production for every grain; the grain string is restricted to
|
||||
// the `AggregateGrain` enum upstream via the Zod schema, so
|
||||
// injecting the mapped unit via `Prisma.raw` is safe. The mapping
|
||||
// also fixes the "weekly"/"monthly" passthrough — Postgres expects
|
||||
// the singular forms `week`/`month`.
|
||||
const TRUNC_UNIT: Record<Exclude<AggregateGrain, "raw">, string> = {
|
||||
daily: "day",
|
||||
weekly: "week",
|
||||
monthly: "month",
|
||||
};
|
||||
const truncUnit = TRUNC_UNIT[grain];
|
||||
const truncUnitLiteral = Prisma.raw(`'${truncUnit}'`);
|
||||
// Cumulative HK types (steps, active energy, flights, distance,
|
||||
// daylight) are partial-day increments — averaging across the day
|
||||
// understates the daily total by the per-bucket sample count.
|
||||
// SUM for those; AVG for spot metrics (BP, weight, pulse, BG,
|
||||
// body fat, mood, sleep). See R-A finding 2.
|
||||
const useSum =
|
||||
type != null && CUMULATIVE_HK_TYPES.has(type as MeasurementType);
|
||||
const aggregator = useSum
|
||||
? Prisma.raw(`SUM(m."value")::double precision`)
|
||||
: Prisma.raw(`AVG(m."value")::double precision`);
|
||||
// v1.11.1 — collapse overlapping sources to the ladder-canonical reading
|
||||
// per (type, bucket) before aggregating, so this cold/uncovered fallback
|
||||
// agrees with the warm rollup path. `canon` picks the winning source for
|
||||
// each (type, bucket) via the per-user rank; the join keeps only its rows.
|
||||
// Buckets are cut at the user's own calendar boundaries (#1026), the
|
||||
// same day the raw list and the chart's short range use. Cumulative
|
||||
// types SUM per bucket, every other type averages; overlapping sources
|
||||
// collapse to the ladder-canonical one first.
|
||||
const priorityJson = await loadUserSourcePriority(user.id);
|
||||
const rankRaw = Prisma.raw(
|
||||
buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'),
|
||||
);
|
||||
const typeFilter = type
|
||||
? Prisma.sql`AND m."type" = ${type}::measurement_type`
|
||||
: Prisma.empty;
|
||||
const buckets = await prisma.$queryRaw<
|
||||
Array<{ type: string; bucket_start: Date; avg: number; cnt: number }>
|
||||
>`
|
||||
WITH canon AS (
|
||||
SELECT DISTINCT ON (m."type", date_trunc(${truncUnitLiteral}, m."measured_at"))
|
||||
m."type" AS t,
|
||||
date_trunc(${truncUnitLiteral}, m."measured_at") AS d,
|
||||
m."source" AS canon
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = ${user.id}
|
||||
AND m."measured_at" >= ${from}
|
||||
AND m."measured_at" <= ${to}
|
||||
AND m."deleted_at" IS NULL
|
||||
${typeFilter}
|
||||
ORDER BY m."type", date_trunc(${truncUnitLiteral}, m."measured_at"), (${rankRaw}), m."source"
|
||||
)
|
||||
SELECT
|
||||
m."type"::text AS type,
|
||||
date_trunc(${truncUnitLiteral}, m."measured_at") AS bucket_start,
|
||||
${aggregator} AS avg,
|
||||
COUNT(*)::int AS cnt
|
||||
FROM measurements m
|
||||
JOIN canon c
|
||||
ON c.t = m."type"
|
||||
AND c.d = date_trunc(${truncUnitLiteral}, m."measured_at")
|
||||
AND c.canon = m."source"
|
||||
WHERE m."user_id" = ${user.id}
|
||||
AND m."measured_at" >= ${from}
|
||||
AND m."measured_at" <= ${to}
|
||||
AND m."deleted_at" IS NULL
|
||||
${typeFilter}
|
||||
GROUP BY m."type", bucket_start
|
||||
ORDER BY bucket_start ASC
|
||||
LIMIT ${cap}
|
||||
`;
|
||||
|
||||
const measurements = buckets.map((b) => ({
|
||||
type: b.type,
|
||||
value: Number(b.avg),
|
||||
measuredAt: b.bucket_start.toISOString(),
|
||||
count: Number(b.cnt),
|
||||
}));
|
||||
const measurements = await readLiveBuckets({
|
||||
userId: user.id,
|
||||
type: (type as MeasurementType | undefined) ?? null,
|
||||
from,
|
||||
to,
|
||||
cap,
|
||||
priorityJson,
|
||||
grain,
|
||||
timeZone: await resolveUserTimezone(user.id),
|
||||
});
|
||||
annotate({
|
||||
action: { name: "measurement.list" },
|
||||
meta: { total: measurements.length, type, aggregate: grain },
|
||||
|
||||
@@ -27,10 +27,6 @@ vi.mock("@/lib/rollups/measurement-read-wmy", () => ({
|
||||
readTieredRollupSeries: mocks.readTieredRollupSeries,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/rollups/measurement-rollups", () => ({
|
||||
recomputeUserRollups: vi.fn(),
|
||||
}));
|
||||
|
||||
import { readDailySeries } from "../daily-series-read";
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
@@ -39,47 +35,25 @@ beforeEach(() => {
|
||||
mocks.findMany.mockReset();
|
||||
mocks.queryRaw.mockReset();
|
||||
mocks.readTieredRollupSeries.mockReset();
|
||||
// Default DAY-rollup read returns enough rows (≥ SUSPICIOUS_ROW_FLOOR)
|
||||
// so the short-window path resolves without the coverage probe / live
|
||||
// query firing.
|
||||
mocks.findMany.mockResolvedValue([
|
||||
{
|
||||
// The daily path folds live, one row per local day.
|
||||
mocks.queryRaw.mockResolvedValue(
|
||||
["2026-06-01", "2026-06-02", "2026-06-03"].map((d, i) => ({
|
||||
type: "WEIGHT",
|
||||
source: "MANUAL",
|
||||
bucketStart: new Date("2026-06-01T00:00:00.000Z"),
|
||||
mean: 81,
|
||||
count: 1,
|
||||
sumValue: null,
|
||||
minValue: 81,
|
||||
maxValue: 81,
|
||||
},
|
||||
{
|
||||
type: "WEIGHT",
|
||||
source: "MANUAL",
|
||||
bucketStart: new Date("2026-06-02T00:00:00.000Z"),
|
||||
mean: 82,
|
||||
count: 1,
|
||||
sumValue: null,
|
||||
minValue: 82,
|
||||
maxValue: 82,
|
||||
},
|
||||
{
|
||||
type: "WEIGHT",
|
||||
source: "MANUAL",
|
||||
bucketStart: new Date("2026-06-03T00:00:00.000Z"),
|
||||
mean: 80,
|
||||
count: 1,
|
||||
sumValue: null,
|
||||
minValue: 80,
|
||||
maxValue: 80,
|
||||
},
|
||||
]);
|
||||
bucket_start: new Date(`${d}T00:00:00.000Z`),
|
||||
avg: 80 + i,
|
||||
cnt: 1,
|
||||
min_value: 80 + i,
|
||||
max_value: 80 + i,
|
||||
})),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
const TZ = "Europe/Berlin";
|
||||
|
||||
describe("readDailySeries — long-range tier step-up", () => {
|
||||
it("routes a multi-year window through the tiered reader and returns its whole-history rows", async () => {
|
||||
const to = new Date("2026-06-21T00:00:00.000Z");
|
||||
@@ -109,6 +83,7 @@ describe("readDailySeries — long-range tier step-up", () => {
|
||||
from,
|
||||
to,
|
||||
priorityJson: null,
|
||||
timeZone: TZ,
|
||||
});
|
||||
|
||||
expect(mocks.readTieredRollupSeries).toHaveBeenCalledTimes(1);
|
||||
@@ -119,8 +94,8 @@ describe("readDailySeries — long-range tier step-up", () => {
|
||||
// Whole-history coverage: the earliest 2017 bucket survives.
|
||||
expect(result[0].measuredAt).toBe("2017-01-01T00:00:00.000Z");
|
||||
expect(result).toHaveLength(2);
|
||||
// The DAY-rollup path was NOT consulted for the long window.
|
||||
expect(mocks.findMany).not.toHaveBeenCalled();
|
||||
// The daily path was NOT consulted for the long window.
|
||||
expect(mocks.queryRaw).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("falls through to the daily path on a tiered coverage miss (no silent empty)", async () => {
|
||||
@@ -134,11 +109,11 @@ describe("readDailySeries — long-range tier step-up", () => {
|
||||
from,
|
||||
to,
|
||||
priorityJson: null,
|
||||
timeZone: TZ,
|
||||
});
|
||||
|
||||
expect(mocks.readTieredRollupSeries).toHaveBeenCalledTimes(1);
|
||||
// Daily DAY-rollup read still ran as the fallback.
|
||||
expect(mocks.findMany).toHaveBeenCalled();
|
||||
expect(mocks.queryRaw).toHaveBeenCalled();
|
||||
expect(result).toHaveLength(3);
|
||||
});
|
||||
|
||||
@@ -146,56 +121,25 @@ describe("readDailySeries — long-range tier step-up", () => {
|
||||
const to = new Date("2026-06-21T00:00:00.000Z");
|
||||
const from = new Date(to.getTime() - 90 * DAY_MS);
|
||||
|
||||
await readDailySeries({
|
||||
userId: "u",
|
||||
type: "WEIGHT",
|
||||
from,
|
||||
to,
|
||||
priorityJson: null,
|
||||
});
|
||||
|
||||
expect(mocks.readTieredRollupSeries).not.toHaveBeenCalled();
|
||||
expect(mocks.findMany).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("readDailySeries — rollup read throw falls back to live SQL (F-DB-2)", () => {
|
||||
it("a rollup-table throw on a normal window degrades to the live date_trunc aggregate, not a 500", async () => {
|
||||
const to = new Date("2026-06-21T00:00:00.000Z");
|
||||
const from = new Date(to.getTime() - 90 * DAY_MS);
|
||||
|
||||
// The primary rollup read throws (statement_timeout / deadlock / reset).
|
||||
mocks.findMany
|
||||
.mockReset()
|
||||
.mockRejectedValue(new Error("statement timeout"));
|
||||
// Live fallback serves the tile.
|
||||
mocks.queryRaw.mockResolvedValueOnce([
|
||||
{
|
||||
type: "WEIGHT",
|
||||
bucket_start: new Date("2026-06-10T00:00:00.000Z"),
|
||||
avg: 80.5,
|
||||
cnt: 2,
|
||||
min_value: 79.8,
|
||||
max_value: 81.2,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await readDailySeries({
|
||||
userId: "u",
|
||||
type: "WEIGHT",
|
||||
from,
|
||||
to,
|
||||
priorityJson: null,
|
||||
timeZone: TZ,
|
||||
});
|
||||
|
||||
// The read did NOT throw — it fell through to live SQL and served a row.
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0]).toMatchObject({ type: "WEIGHT", value: 80.5, count: 2 });
|
||||
// The live fallback must carry the min/max spread the rollup path emits —
|
||||
// otherwise a rollup-cold account silently loses the chart's spread band
|
||||
// for the same underlying data.
|
||||
expect(result[0]).toMatchObject({ minValue: 79.8, maxValue: 81.2 });
|
||||
expect(mocks.queryRaw).toHaveBeenCalled();
|
||||
expect(mocks.readTieredRollupSeries).not.toHaveBeenCalled();
|
||||
// The daily series never reads the UTC-day DAY rollup (#1026).
|
||||
expect(mocks.findMany).not.toHaveBeenCalled();
|
||||
expect(result[0]).toMatchObject({
|
||||
type: "WEIGHT",
|
||||
value: 80,
|
||||
count: 1,
|
||||
minValue: 80,
|
||||
maxValue: 80,
|
||||
});
|
||||
});
|
||||
|
||||
it("a tiered-reader throw on a long window falls through to the daily path", async () => {
|
||||
@@ -209,11 +153,10 @@ describe("readDailySeries — rollup read throw falls back to live SQL (F-DB-2)"
|
||||
from,
|
||||
to,
|
||||
priorityJson: null,
|
||||
timeZone: TZ,
|
||||
});
|
||||
|
||||
// The daily DAY-rollup read still ran as the fallback (the default findMany
|
||||
// returns 3 rows), so the throw did not 500 the read.
|
||||
expect(mocks.findMany).toHaveBeenCalled();
|
||||
expect(mocks.queryRaw).toHaveBeenCalled();
|
||||
expect(result).toHaveLength(3);
|
||||
});
|
||||
});
|
||||
@@ -1,29 +1,26 @@
|
||||
/**
|
||||
* v1.18.6 — shared rollup-backed daily-series reader.
|
||||
* Shared daily-series reader for the chart surfaces.
|
||||
*
|
||||
* Extracted from the `GET /api/measurements` rollup-daily branch so the
|
||||
* single-type route AND the batched series endpoint
|
||||
* (`GET /api/measurements/series-batch`) read through the SAME path
|
||||
* instead of duplicating the rollup SQL + the coverage-fallback fold.
|
||||
*
|
||||
* `readDailySeries` reproduces the route's `source=rollup` +
|
||||
* `aggregate=daily` contract byte-for-byte: collapse per-source rollup
|
||||
* rows to the ladder-canonical reading per day, probe for an
|
||||
* under-converged rollup window and fold inline when the live table
|
||||
* carries more distinct days, and fall through to the live `date_trunc`
|
||||
* aggregate when the rollup is empty for the window (brand-new accounts /
|
||||
* write-hook race). The returned rows match the wire shape the chart
|
||||
* client consumes (`{ type, value, measuredAt, count, minValue?,
|
||||
* maxValue? }`).
|
||||
* Serves the `GET /api/measurements?aggregate=daily&source=rollup` contract
|
||||
* and the batched series endpoint (`GET /api/measurements/series-batch`) from
|
||||
* one path, so both read the same numbers. Windows up to the DAY cap are
|
||||
* folded live, one row per local calendar day of the user (#1026); longer
|
||||
* windows step up to the WEEK / MONTH / YEAR rollup tier. The returned rows
|
||||
* match the wire shape the chart client consumes (`{ type, value,
|
||||
* measuredAt, count, minValue?, maxValue? }`), with `measuredAt` the instant
|
||||
* of the bucket's local midnight.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { BUCKET_CAP } from "@/lib/measurements/range-aggregation";
|
||||
import {
|
||||
BUCKET_CAP,
|
||||
type AggregateGrain,
|
||||
} from "@/lib/measurements/range-aggregation";
|
||||
import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping";
|
||||
import { collapseRollupRowsBySource } from "@/lib/rollups/measurement-read";
|
||||
import { readTieredRollupSeries } from "@/lib/rollups/measurement-read-wmy";
|
||||
import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
|
||||
import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { isValidTimezone } from "@/lib/tz/format";
|
||||
import { resolveUserTimezone } from "@/lib/tz/resolver";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
import { Prisma } from "@/generated/prisma/client";
|
||||
|
||||
@@ -37,12 +34,9 @@ export interface DailySeriesRow {
|
||||
maxValue?: number | null;
|
||||
}
|
||||
|
||||
const SUSPICIOUS_ROW_FLOOR = 3;
|
||||
const SUSPICIOUS_WINDOW_DAYS = 7;
|
||||
|
||||
/**
|
||||
* Read one type's daily series for `[from, to]` through the rollup tier,
|
||||
* falling back to the live `date_trunc` aggregate on a coverage miss.
|
||||
* Read one type's daily series for `[from, to]`, one row per local day of
|
||||
* the user (or per tier bucket when the window exceeds the DAY cap).
|
||||
*
|
||||
* `priorityJson` is the user's source-priority ladder (load once per
|
||||
* request via `loadUserSourcePriority` and thread it across types so the
|
||||
@@ -56,6 +50,8 @@ export async function readDailySeries(opts: {
|
||||
to: Date;
|
||||
limit?: number;
|
||||
priorityJson: unknown;
|
||||
/** IANA zone the days are cut in; defaults to the user's own zone. */
|
||||
timeZone?: string;
|
||||
}): Promise<DailySeriesRow[]> {
|
||||
const { userId, type, from, to, priorityJson } = opts;
|
||||
const cap = Math.min(opts.limit ?? BUCKET_CAP.daily, BUCKET_CAP.daily);
|
||||
@@ -106,10 +102,10 @@ export async function readDailySeries(opts: {
|
||||
return tiered.rows;
|
||||
}
|
||||
// Coverage miss at every tier — fall through to the daily path, which
|
||||
// probes the live table and folds an under-converged window. The
|
||||
// daily fallback still caps at `cap`; the annotate below records that
|
||||
// the long-range read could not step up so a truncated daily slice is
|
||||
// an explicit, logged outcome rather than a silent one.
|
||||
// folds the live table. The daily fallback still caps at `cap`; the
|
||||
// annotate below records that the long-range read could not step up so
|
||||
// a truncated daily slice is an explicit, logged outcome rather than a
|
||||
// silent one.
|
||||
annotate({
|
||||
meta: {
|
||||
tiered_series_coverage_miss: true,
|
||||
@@ -119,139 +115,94 @@ export async function readDailySeries(opts: {
|
||||
});
|
||||
}
|
||||
|
||||
const readRollup = async () =>
|
||||
collapseRollupRowsBySource(
|
||||
await prisma.measurementRollup.findMany({
|
||||
where: {
|
||||
userId,
|
||||
type,
|
||||
granularity: "DAY",
|
||||
bucketStart: { gte: from, lte: to },
|
||||
},
|
||||
orderBy: { bucketStart: "asc" },
|
||||
select: {
|
||||
type: true,
|
||||
source: true,
|
||||
bucketStart: true,
|
||||
mean: true,
|
||||
count: true,
|
||||
sumValue: true,
|
||||
minValue: true,
|
||||
maxValue: true,
|
||||
},
|
||||
}),
|
||||
type,
|
||||
priorityJson,
|
||||
).slice(0, cap);
|
||||
|
||||
// F-DB-2 — the primary rollup read is wrapped: a transient rollup-table error
|
||||
// (statement_timeout, deadlock, connection reset) must NOT 500 the tile when
|
||||
// the live table can still serve it. Treat a throw as a coverage miss and fall
|
||||
// straight through to the live `date_trunc` aggregate — the same posture the
|
||||
// batched series reader already takes, so the single-type route and the batch
|
||||
// route no longer diverge on a rollup hiccup.
|
||||
let rollupRows: Awaited<ReturnType<typeof readRollup>>;
|
||||
try {
|
||||
rollupRows = await readRollup();
|
||||
} catch (err) {
|
||||
annotate({
|
||||
meta: {
|
||||
rollup_read_threw: true,
|
||||
rollup_read_error: err instanceof Error ? err.message : String(err),
|
||||
type,
|
||||
},
|
||||
});
|
||||
return readLiveDaily({ userId, type, from, to, cap, priorityJson });
|
||||
}
|
||||
|
||||
// Coverage-mismatch probe + inline fold (identical to the route).
|
||||
const windowMs = to.getTime() - from.getTime();
|
||||
const windowDays = Math.ceil(windowMs / 86_400_000);
|
||||
let effectiveRows = rollupRows;
|
||||
let coverageFallbackFired = false;
|
||||
if (
|
||||
rollupRows.length > 0 &&
|
||||
rollupRows.length < SUSPICIOUS_ROW_FLOOR &&
|
||||
windowDays >= SUSPICIOUS_WINDOW_DAYS
|
||||
) {
|
||||
const liveDayCountRows = await prisma.$queryRaw<Array<{ days: bigint }>>`
|
||||
SELECT COUNT(DISTINCT date_trunc('day', m."measured_at"))::bigint AS days
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = ${userId}
|
||||
AND m."type" = ${type}::measurement_type
|
||||
AND m."measured_at" >= ${from}
|
||||
AND m."measured_at" <= ${to}
|
||||
AND m."deleted_at" IS NULL
|
||||
`;
|
||||
const liveDays = Number(liveDayCountRows[0]?.days ?? BigInt(0));
|
||||
if (liveDays > rollupRows.length) {
|
||||
coverageFallbackFired = true;
|
||||
try {
|
||||
await recomputeUserRollups(userId, {
|
||||
types: [type],
|
||||
granularities: ["DAY"],
|
||||
from,
|
||||
to,
|
||||
});
|
||||
effectiveRows = await readRollup();
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
annotate({
|
||||
meta: {
|
||||
rollup_coverage_fallback_failed: true,
|
||||
rollup_coverage_fallback_error: message,
|
||||
type,
|
||||
},
|
||||
});
|
||||
effectiveRows = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (effectiveRows.length > 0) {
|
||||
const useSum = CUMULATIVE_HK_TYPES.has(type);
|
||||
annotate({
|
||||
action: { name: "measurement.list" },
|
||||
meta: {
|
||||
total: effectiveRows.length,
|
||||
type,
|
||||
aggregate: "daily",
|
||||
source: "rollup",
|
||||
coverage_fallback: coverageFallbackFired,
|
||||
},
|
||||
});
|
||||
return effectiveRows.map((r) => ({
|
||||
type: r.type,
|
||||
value: useSum ? (r.sumValue ?? r.mean * r.count) : r.mean,
|
||||
measuredAt: r.bucketStart.toISOString(),
|
||||
count: r.count,
|
||||
minValue: useSum ? undefined : r.minValue,
|
||||
maxValue: useSum ? undefined : r.maxValue,
|
||||
}));
|
||||
}
|
||||
|
||||
// Live `date_trunc` fallback — rollup empty for the window.
|
||||
return readLiveDaily({ userId, type, from, to, cap, priorityJson });
|
||||
// The daily series is cut at the user's own midnight (#1026). The DAY
|
||||
// rollup tier cannot serve it: its buckets are UTC days, so east of UTC a
|
||||
// sample shortly after local midnight counted on the previous day, and
|
||||
// every bucket's UTC-midnight start read as the previous local day west of
|
||||
// UTC. The 7-day chart buckets raw rows by the local day, so switching
|
||||
// ranges moved daily totals between neighbouring days. The live fold is
|
||||
// one indexed pass per window (about 0.1 s for 90 days of per-minute
|
||||
// samples) and returns at most one row per day.
|
||||
const timeZone = opts.timeZone ?? (await resolveUserTimezone(userId));
|
||||
const rows = await readLiveBuckets({
|
||||
userId,
|
||||
type,
|
||||
from,
|
||||
to,
|
||||
cap,
|
||||
priorityJson,
|
||||
grain: "daily",
|
||||
timeZone,
|
||||
});
|
||||
annotate({
|
||||
action: { name: "measurement.list" },
|
||||
meta: { total: rows.length, type, aggregate: "daily", source: "live" },
|
||||
});
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** Live `date_trunc('day', …)` aggregate — the rollup-empty fallback. */
|
||||
async function readLiveDaily(opts: {
|
||||
/** SQL `date_trunc` unit per aggregate grain (a closed map, never input). */
|
||||
const TRUNC_UNIT: Record<Exclude<AggregateGrain, "raw">, string> = {
|
||||
daily: "day",
|
||||
weekly: "week",
|
||||
monthly: "month",
|
||||
};
|
||||
|
||||
/**
|
||||
* Live aggregate of the measurements table into buckets cut at the user's
|
||||
* own calendar boundaries.
|
||||
*
|
||||
* Each sample is converted to wall-clock time in `timeZone` before
|
||||
* truncation, so a bucket is the user's local day (or week / month), and the
|
||||
* returned `measuredAt` is the instant of that local midnight. The chart
|
||||
* labels a row by formatting `measuredAt` in the same zone, which lands on
|
||||
* the bucket's own day for every offset, including half-hour and negative
|
||||
* ones, and across a DST change.
|
||||
*
|
||||
* Per bucket the overlapping sources collapse to the ladder-canonical one
|
||||
* before the fold (the same pick the rollup reader makes), cumulative types
|
||||
* SUM and every other type averages. The result set is at most `cap` rows,
|
||||
* however dense the underlying stream: the fold runs in Postgres.
|
||||
*
|
||||
* `type` narrows to one type; `null` folds every type in the window (the
|
||||
* multi-type `GET /api/measurements?aggregate=` form).
|
||||
*/
|
||||
export async function readLiveBuckets(opts: {
|
||||
userId: string;
|
||||
type: MeasurementType;
|
||||
type: MeasurementType | null;
|
||||
from: Date;
|
||||
to: Date;
|
||||
cap: number;
|
||||
priorityJson: unknown;
|
||||
grain: Exclude<AggregateGrain, "raw">;
|
||||
timeZone: string;
|
||||
}): Promise<DailySeriesRow[]> {
|
||||
const { userId, type, from, to, cap, priorityJson } = opts;
|
||||
const truncUnitLiteral = Prisma.raw(`'day'`);
|
||||
const useSum = CUMULATIVE_HK_TYPES.has(type);
|
||||
const aggregator = useSum
|
||||
? Prisma.raw(`SUM(m."value")::double precision`)
|
||||
: Prisma.raw(`AVG(m."value")::double precision`);
|
||||
const { userId, type, from, to, cap, priorityJson, grain } = opts;
|
||||
// A zone Postgres does not know would 500 the read; the resolver already
|
||||
// validates stored zones, this guards a caller passing a raw string.
|
||||
const timeZone = isValidTimezone(opts.timeZone) ? opts.timeZone : "UTC";
|
||||
// `date_trunc` needs its unit as a literal; the unit comes from the closed
|
||||
// `TRUNC_UNIT` map keyed by the Zod-constrained grain, never from input.
|
||||
const unit = Prisma.raw(`'${TRUNC_UNIT[grain]}'`);
|
||||
// Cumulative types SUM per bucket, every other type averages. The type list
|
||||
// is the closed code constant, spliced as literals; `s."type"` is a GROUP BY
|
||||
// column, so choosing the aggregate per group is legal.
|
||||
const cumulativeList = [...CUMULATIVE_HK_TYPES]
|
||||
.map((t) => `'${t}'`)
|
||||
.join(",");
|
||||
const aggregator = Prisma.raw(
|
||||
`(CASE WHEN s."type"::text IN (${cumulativeList}) THEN SUM(s."value") ELSE AVG(s."value") END)::double precision`,
|
||||
);
|
||||
const rankRaw = Prisma.raw(
|
||||
buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'),
|
||||
);
|
||||
const typeFilter = type
|
||||
? Prisma.sql`AND m."type" = ${type}::measurement_type`
|
||||
: Prisma.empty;
|
||||
// measured_at is a UTC wall-clock timestamp: pin it to UTC, re-read it in
|
||||
// the user's zone and truncate there. The bucket is computed once in `src`
|
||||
// and referenced by name afterwards, so the zone parameter never has to
|
||||
// match itself across SELECT and GROUP BY.
|
||||
const buckets = await prisma.$queryRaw<
|
||||
Array<{
|
||||
type: string;
|
||||
@@ -262,58 +213,54 @@ async function readLiveDaily(opts: {
|
||||
max_value: number | null;
|
||||
}>
|
||||
>`
|
||||
WITH canon AS (
|
||||
SELECT DISTINCT ON (m."type", date_trunc(${truncUnitLiteral}, m."measured_at"))
|
||||
m."type" AS t,
|
||||
date_trunc(${truncUnitLiteral}, m."measured_at") AS d,
|
||||
m."source" AS canon
|
||||
WITH src AS (
|
||||
SELECT
|
||||
m."type",
|
||||
m."source",
|
||||
m."value",
|
||||
date_trunc(${unit}, (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}) AS d
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = ${userId}
|
||||
AND m."measured_at" >= ${from}
|
||||
AND m."measured_at" <= ${to}
|
||||
AND m."deleted_at" IS NULL
|
||||
AND m."type" = ${type}::measurement_type
|
||||
ORDER BY m."type", date_trunc(${truncUnitLiteral}, m."measured_at"), (${rankRaw}), m."source"
|
||||
${typeFilter}
|
||||
),
|
||||
canon AS (
|
||||
SELECT DISTINCT ON (m."type", m.d)
|
||||
m."type" AS t,
|
||||
m.d AS d,
|
||||
m."source" AS canon
|
||||
FROM src m
|
||||
ORDER BY m."type", m.d, (${rankRaw}), m."source"
|
||||
)
|
||||
SELECT
|
||||
m."type"::text AS type,
|
||||
date_trunc(${truncUnitLiteral}, m."measured_at") AS bucket_start,
|
||||
s."type"::text AS type,
|
||||
s.d AT TIME ZONE ${timeZone} AS bucket_start,
|
||||
${aggregator} AS avg,
|
||||
COUNT(*)::int AS cnt,
|
||||
MIN(m."value") AS min_value,
|
||||
MAX(m."value") AS max_value
|
||||
FROM measurements m
|
||||
MIN(s."value") AS min_value,
|
||||
MAX(s."value") AS max_value
|
||||
FROM src s
|
||||
JOIN canon c
|
||||
ON c.t = m."type"
|
||||
AND c.d = date_trunc(${truncUnitLiteral}, m."measured_at")
|
||||
AND c.canon = m."source"
|
||||
WHERE m."user_id" = ${userId}
|
||||
AND m."measured_at" >= ${from}
|
||||
AND m."measured_at" <= ${to}
|
||||
AND m."deleted_at" IS NULL
|
||||
AND m."type" = ${type}::measurement_type
|
||||
GROUP BY m."type", bucket_start
|
||||
ORDER BY bucket_start ASC
|
||||
ON c.t = s."type"
|
||||
AND c.d = s.d
|
||||
AND c.canon = s."source"
|
||||
GROUP BY s."type", s.d
|
||||
ORDER BY s.d ASC
|
||||
LIMIT ${cap}
|
||||
`;
|
||||
annotate({
|
||||
action: { name: "measurement.list" },
|
||||
meta: { total: buckets.length, type, aggregate: "daily" },
|
||||
return buckets.map((b) => {
|
||||
const isSum = CUMULATIVE_HK_TYPES.has(b.type as MeasurementType);
|
||||
return {
|
||||
type: b.type,
|
||||
value: Number(b.avg),
|
||||
measuredAt: b.bucket_start.toISOString(),
|
||||
count: Number(b.cnt),
|
||||
// A SUM has no meaningful intra-day spread; the chart draws the band
|
||||
// for averaged types only.
|
||||
minValue: isSum || b.min_value === null ? undefined : Number(b.min_value),
|
||||
maxValue: isSum || b.max_value === null ? undefined : Number(b.max_value),
|
||||
};
|
||||
});
|
||||
return buckets.map((b) => ({
|
||||
type: b.type,
|
||||
value: Number(b.avg),
|
||||
measuredAt: b.bucket_start.toISOString(),
|
||||
count: Number(b.cnt),
|
||||
// The rollup path emits these; the live fallback used to omit them, so a
|
||||
// rollup-cold account silently lost the min/max spread band on the chart
|
||||
// for the same underlying data — contradicting this file's own
|
||||
// byte-for-byte parity claim.
|
||||
minValue: b.min_value === null ? undefined : Number(b.min_value),
|
||||
maxValue: b.max_value === null ? undefined : Number(b.max_value),
|
||||
// The rollup path emits these; the live fallback used to omit them, so a
|
||||
// rollup-cold account silently lost the min/max spread band on the chart
|
||||
// for the same underlying data — contradicting this file's own
|
||||
// byte-for-byte parity claim.
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* Issue #1026 — a daily total must land on the same calendar day whichever
|
||||
* chart range reads it.
|
||||
*
|
||||
* The 7-day chart pulls raw rows and buckets them by the user's own calendar
|
||||
* day. The 30-day chart reads the daily series from the rollup tier, whose
|
||||
* DAY buckets are cut at UTC midnight, and the chart then labels each bucket's
|
||||
* UTC-midnight start in the user's zone. East of UTC a sample shortly after
|
||||
* local midnight was counted on the previous day; west of UTC every bucket
|
||||
* was labelled one day early. Steps, active energy, flights and distance are
|
||||
* summed per day, so the drift showed up as whole totals moving between
|
||||
* neighbouring days when the range changed.
|
||||
*
|
||||
* This file seeds cumulative samples right next to local midnight for a
|
||||
* positive, a half-hour and a negative offset, warms the rollup tier the way
|
||||
* the live instance has it, and asserts that the daily series (both the
|
||||
* rollup-backed read and the live aggregate) buckets by the user's day.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
import {
|
||||
readDailySeries,
|
||||
readLiveBuckets,
|
||||
} from "@/lib/measurements/daily-series-read";
|
||||
import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
|
||||
import { invalidateUserTimezone } from "@/lib/tz/resolver";
|
||||
import { userDayKey } from "@/lib/tz/format";
|
||||
import { localHmAsUtc } from "@/lib/tz/local-day";
|
||||
|
||||
vi.mock("@/lib/db-compat", () => ({
|
||||
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
beforeEach(async () => {
|
||||
await truncateAllTables(getPrismaClient());
|
||||
});
|
||||
|
||||
const DAY = new Date("2026-09-10T12:00:00Z");
|
||||
const NEXT_DAY = new Date("2026-09-11T12:00:00Z");
|
||||
|
||||
async function seed(tz: string): Promise<string> {
|
||||
const prisma = getPrismaClient();
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `tz-${tz.replace(/\W/g, "-").toLowerCase()}`,
|
||||
email: `tz-${tz.replace(/\W/g, "-").toLowerCase()}@example.test`,
|
||||
role: "USER",
|
||||
timezone: tz,
|
||||
},
|
||||
});
|
||||
invalidateUserTimezone(user.id);
|
||||
const rows = [
|
||||
// Just after local midnight: belongs to DAY.
|
||||
{ at: localHmAsUtc(DAY, tz, 0, 20), value: 100 },
|
||||
// Just before the next local midnight: still DAY.
|
||||
{ at: localHmAsUtc(DAY, tz, 23, 40), value: 1000 },
|
||||
// A daily total stamped at the start of the next local day.
|
||||
{ at: localHmAsUtc(NEXT_DAY, tz, 0, 0), value: 7 },
|
||||
{ at: localHmAsUtc(NEXT_DAY, tz, 12, 0), value: 3 },
|
||||
];
|
||||
await prisma.measurement.createMany({
|
||||
data: rows.map((r) => ({
|
||||
userId: user.id,
|
||||
type: "ACTIVITY_STEPS" as const,
|
||||
value: r.value,
|
||||
unit: "count",
|
||||
measuredAt: r.at,
|
||||
source: "APPLE_HEALTH" as const,
|
||||
})),
|
||||
});
|
||||
return user.id;
|
||||
}
|
||||
|
||||
function byLocalDay(
|
||||
rows: Array<{ measuredAt: string; value: number }>,
|
||||
tz: string,
|
||||
): Record<string, number> {
|
||||
const out: Record<string, number> = {};
|
||||
for (const r of rows) {
|
||||
const key = userDayKey(new Date(r.measuredAt), tz);
|
||||
out[key] = (out[key] ?? 0) + r.value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const EXPECTED = { "2026-09-10": 1100, "2026-09-11": 10 };
|
||||
|
||||
describe("daily series buckets by the user's calendar day (#1026)", () => {
|
||||
for (const tz of ["Europe/Berlin", "Asia/Kolkata", "America/New_York"]) {
|
||||
it(`chart daily read agrees with the local day (${tz})`, async () => {
|
||||
const userId = await seed(tz);
|
||||
const from = new Date("2026-09-05T00:00:00Z");
|
||||
const to = new Date("2026-09-15T00:00:00Z");
|
||||
// Warm the rollup tier exactly as the write hook / backfill does.
|
||||
await recomputeUserRollups(userId, {
|
||||
types: ["ACTIVITY_STEPS"],
|
||||
granularities: ["DAY"],
|
||||
from,
|
||||
to,
|
||||
});
|
||||
const rows = await readDailySeries({
|
||||
userId,
|
||||
type: "ACTIVITY_STEPS",
|
||||
from,
|
||||
to,
|
||||
priorityJson: null,
|
||||
});
|
||||
expect(byLocalDay(rows, tz)).toEqual(EXPECTED);
|
||||
});
|
||||
|
||||
it(`live daily aggregate agrees with the local day (${tz})`, async () => {
|
||||
const userId = await seed(tz);
|
||||
const rows = await readLiveBuckets({
|
||||
userId,
|
||||
type: "ACTIVITY_STEPS",
|
||||
from: new Date("2026-09-05T00:00:00Z"),
|
||||
to: new Date("2026-09-15T00:00:00Z"),
|
||||
cap: 366,
|
||||
priorityJson: null,
|
||||
grain: "daily",
|
||||
timeZone: tz,
|
||||
});
|
||||
expect(byLocalDay(rows, tz)).toEqual(EXPECTED);
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -14,6 +14,7 @@
|
||||
*/
|
||||
import { NextRequest } from "next/server";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { userDayKey } from "@/lib/tz/format";
|
||||
|
||||
import { cookieJar } from "./mock-next-headers";
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
@@ -128,11 +129,15 @@ describe("GET /api/measurements?aggregate=… (real Postgres)", () => {
|
||||
expect(json.data.meta.aggregate).toBe("daily");
|
||||
expect(json.data.measurements).toHaveLength(2);
|
||||
|
||||
const may1 = json.data.measurements.find((m) =>
|
||||
m.measuredAt.startsWith("2026-05-01"),
|
||||
// Buckets are the account's own calendar days (Europe/Berlin, the
|
||||
// column default), keyed by the instant of local midnight (#1026).
|
||||
const may1 = json.data.measurements.find(
|
||||
(m) =>
|
||||
userDayKey(new Date(m.measuredAt), "Europe/Berlin") === "2026-05-01",
|
||||
);
|
||||
const may2 = json.data.measurements.find((m) =>
|
||||
m.measuredAt.startsWith("2026-05-02"),
|
||||
const may2 = json.data.measurements.find(
|
||||
(m) =>
|
||||
userDayKey(new Date(m.measuredAt), "Europe/Berlin") === "2026-05-02",
|
||||
);
|
||||
expect(may1).toBeDefined();
|
||||
expect(may1!.count).toBe(2);
|
||||
|
||||
Reference in new issue
Block a user