fix(measurements): cut chart day buckets at the user's midnight

Switching a chart from 7 to 30 days moved steps, active energy,
flights and distance totals onto neighbouring days. The 7-day range
buckets raw rows by the user's calendar day; the 30-day range read the
DAY rollup tier, whose buckets are UTC days, and the chart labelled each
bucket's UTC-midnight start in the user's zone. East of UTC a sample
shortly after local midnight counted on the previous day; west of UTC
every bucket read one day early.

The daily series (the chart read and the batched dashboard read) and the
aggregate=daily|weekly|monthly list now fold live in the user's zone and
return the instant of local midnight, so every range and every summed
metric shares one day boundary. The fold runs in Postgres and returns
one row per day; about 0.1 s for 90 days of per-minute samples. Windows
longer than the DAY cap still step up to the coarser rollup tiers.

Refs #1026
This commit is contained in:
Marc-André Bombeck committed 2026-09-24 07:56:33 +02:00
1 parent 01cd755de6
commit 88cea09c7b
6 files changed
+365 -702

No files matched your search

@@ -27,10 +27,6 @@ vi.mock("@/lib/rollups/measurement-read-wmy", () => ({
readTieredRollupSeries: mocks.readTieredRollupSeries,
}));
vi.mock("@/lib/rollups/measurement-rollups", () => ({
recomputeUserRollups: vi.fn(),
}));
import { readDailySeries } from "../daily-series-read";
const DAY_MS = 86_400_000;
@@ -39,47 +35,25 @@ beforeEach(() => {
mocks.findMany.mockReset();
mocks.queryRaw.mockReset();
mocks.readTieredRollupSeries.mockReset();
// Default DAY-rollup read returns enough rows (≥ SUSPICIOUS_ROW_FLOOR)
// so the short-window path resolves without the coverage probe / live
// query firing.
mocks.findMany.mockResolvedValue([
{
// The daily path folds live, one row per local day.
mocks.queryRaw.mockResolvedValue(
["2026-06-01", "2026-06-02", "2026-06-03"].map((d, i) => ({
type: "WEIGHT",
source: "MANUAL",
bucketStart: new Date("2026-06-01T00:00:00.000Z"),
mean: 81,
count: 1,
sumValue: null,
minValue: 81,
maxValue: 81,
},
{
type: "WEIGHT",
source: "MANUAL",
bucketStart: new Date("2026-06-02T00:00:00.000Z"),
mean: 82,
count: 1,
sumValue: null,
minValue: 82,
maxValue: 82,
},
{
type: "WEIGHT",
source: "MANUAL",
bucketStart: new Date("2026-06-03T00:00:00.000Z"),
mean: 80,
count: 1,
sumValue: null,
minValue: 80,
maxValue: 80,
},
]);
bucket_start: new Date(`${d}T00:00:00.000Z`),
avg: 80 + i,
cnt: 1,
min_value: 80 + i,
max_value: 80 + i,
})),
);
});
afterEach(() => {
vi.restoreAllMocks();
});
const TZ = "Europe/Berlin";
describe("readDailySeries — long-range tier step-up", () => {
it("routes a multi-year window through the tiered reader and returns its whole-history rows", async () => {
const to = new Date("2026-06-21T00:00:00.000Z");
@@ -109,6 +83,7 @@ describe("readDailySeries — long-range tier step-up", () => {
from,
to,
priorityJson: null,
timeZone: TZ,
});
expect(mocks.readTieredRollupSeries).toHaveBeenCalledTimes(1);
@@ -119,8 +94,8 @@ describe("readDailySeries — long-range tier step-up", () => {
// Whole-history coverage: the earliest 2017 bucket survives.
expect(result[0].measuredAt).toBe("2017-01-01T00:00:00.000Z");
expect(result).toHaveLength(2);
// The DAY-rollup path was NOT consulted for the long window.
expect(mocks.findMany).not.toHaveBeenCalled();
// The daily path was NOT consulted for the long window.
expect(mocks.queryRaw).not.toHaveBeenCalled();
});
it("falls through to the daily path on a tiered coverage miss (no silent empty)", async () => {
@@ -134,11 +109,11 @@ describe("readDailySeries — long-range tier step-up", () => {
from,
to,
priorityJson: null,
timeZone: TZ,
});
expect(mocks.readTieredRollupSeries).toHaveBeenCalledTimes(1);
// Daily DAY-rollup read still ran as the fallback.
expect(mocks.findMany).toHaveBeenCalled();
expect(mocks.queryRaw).toHaveBeenCalled();
expect(result).toHaveLength(3);
});
@@ -146,56 +121,25 @@ describe("readDailySeries — long-range tier step-up", () => {
const to = new Date("2026-06-21T00:00:00.000Z");
const from = new Date(to.getTime() - 90 * DAY_MS);
await readDailySeries({
userId: "u",
type: "WEIGHT",
from,
to,
priorityJson: null,
});
expect(mocks.readTieredRollupSeries).not.toHaveBeenCalled();
expect(mocks.findMany).toHaveBeenCalled();
});
});
describe("readDailySeries — rollup read throw falls back to live SQL (F-DB-2)", () => {
it("a rollup-table throw on a normal window degrades to the live date_trunc aggregate, not a 500", async () => {
const to = new Date("2026-06-21T00:00:00.000Z");
const from = new Date(to.getTime() - 90 * DAY_MS);
// The primary rollup read throws (statement_timeout / deadlock / reset).
mocks.findMany
.mockReset()
.mockRejectedValue(new Error("statement timeout"));
// Live fallback serves the tile.
mocks.queryRaw.mockResolvedValueOnce([
{
type: "WEIGHT",
bucket_start: new Date("2026-06-10T00:00:00.000Z"),
avg: 80.5,
cnt: 2,
min_value: 79.8,
max_value: 81.2,
},
]);
const result = await readDailySeries({
userId: "u",
type: "WEIGHT",
from,
to,
priorityJson: null,
timeZone: TZ,
});
// The read did NOT throw — it fell through to live SQL and served a row.
expect(result).toHaveLength(1);
expect(result[0]).toMatchObject({ type: "WEIGHT", value: 80.5, count: 2 });
// The live fallback must carry the min/max spread the rollup path emits —
// otherwise a rollup-cold account silently loses the chart's spread band
// for the same underlying data.
expect(result[0]).toMatchObject({ minValue: 79.8, maxValue: 81.2 });
expect(mocks.queryRaw).toHaveBeenCalled();
expect(mocks.readTieredRollupSeries).not.toHaveBeenCalled();
// The daily series never reads the UTC-day DAY rollup (#1026).
expect(mocks.findMany).not.toHaveBeenCalled();
expect(result[0]).toMatchObject({
type: "WEIGHT",
value: 80,
count: 1,
minValue: 80,
maxValue: 80,
});
});
it("a tiered-reader throw on a long window falls through to the daily path", async () => {
@@ -209,11 +153,10 @@ describe("readDailySeries — rollup read throw falls back to live SQL (F-DB-2)"
from,
to,
priorityJson: null,
timeZone: TZ,
});
// The daily DAY-rollup read still ran as the fallback (the default findMany
// returns 3 rows), so the throw did not 500 the read.
expect(mocks.findMany).toHaveBeenCalled();
expect(mocks.queryRaw).toHaveBeenCalled();
expect(result).toHaveLength(3);
});
});
+137 -190
View File
@@ -1,29 +1,26 @@
/**
* v1.18.6 — shared rollup-backed daily-series reader.
* Shared daily-series reader for the chart surfaces.
*
* Extracted from the `GET /api/measurements` rollup-daily branch so the
* single-type route AND the batched series endpoint
* (`GET /api/measurements/series-batch`) read through the SAME path
* instead of duplicating the rollup SQL + the coverage-fallback fold.
*
* `readDailySeries` reproduces the route's `source=rollup` +
* `aggregate=daily` contract byte-for-byte: collapse per-source rollup
* rows to the ladder-canonical reading per day, probe for an
* under-converged rollup window and fold inline when the live table
* carries more distinct days, and fall through to the live `date_trunc`
* aggregate when the rollup is empty for the window (brand-new accounts /
* write-hook race). The returned rows match the wire shape the chart
* client consumes (`{ type, value, measuredAt, count, minValue?,
* maxValue? }`).
* Serves the `GET /api/measurements?aggregate=daily&source=rollup` contract
* and the batched series endpoint (`GET /api/measurements/series-batch`) from
* one path, so both read the same numbers. Windows up to the DAY cap are
* folded live, one row per local calendar day of the user (#1026); longer
* windows step up to the WEEK / MONTH / YEAR rollup tier. The returned rows
* match the wire shape the chart client consumes (`{ type, value,
* measuredAt, count, minValue?, maxValue? }`), with `measuredAt` the instant
* of the bucket's local midnight.
*/
import { prisma } from "@/lib/db";
import { BUCKET_CAP } from "@/lib/measurements/range-aggregation";
import {
BUCKET_CAP,
type AggregateGrain,
} from "@/lib/measurements/range-aggregation";
import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping";
import { collapseRollupRowsBySource } from "@/lib/rollups/measurement-read";
import { readTieredRollupSeries } from "@/lib/rollups/measurement-read-wmy";
import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql";
import { annotate } from "@/lib/logging/context";
import { isValidTimezone } from "@/lib/tz/format";
import { resolveUserTimezone } from "@/lib/tz/resolver";
import type { MeasurementType } from "@/generated/prisma/client";
import { Prisma } from "@/generated/prisma/client";
@@ -37,12 +34,9 @@ export interface DailySeriesRow {
maxValue?: number | null;
}
const SUSPICIOUS_ROW_FLOOR = 3;
const SUSPICIOUS_WINDOW_DAYS = 7;
/**
* Read one type's daily series for `[from, to]` through the rollup tier,
* falling back to the live `date_trunc` aggregate on a coverage miss.
* Read one type's daily series for `[from, to]`, one row per local day of
* the user (or per tier bucket when the window exceeds the DAY cap).
*
* `priorityJson` is the user's source-priority ladder (load once per
* request via `loadUserSourcePriority` and thread it across types so the
@@ -56,6 +50,8 @@ export async function readDailySeries(opts: {
to: Date;
limit?: number;
priorityJson: unknown;
/** IANA zone the days are cut in; defaults to the user's own zone. */
timeZone?: string;
}): Promise<DailySeriesRow[]> {
const { userId, type, from, to, priorityJson } = opts;
const cap = Math.min(opts.limit ?? BUCKET_CAP.daily, BUCKET_CAP.daily);
@@ -106,10 +102,10 @@ export async function readDailySeries(opts: {
return tiered.rows;
}
// Coverage miss at every tier — fall through to the daily path, which
// probes the live table and folds an under-converged window. The
// daily fallback still caps at `cap`; the annotate below records that
// the long-range read could not step up so a truncated daily slice is
// an explicit, logged outcome rather than a silent one.
// folds the live table. The daily fallback still caps at `cap`; the
// annotate below records that the long-range read could not step up so
// a truncated daily slice is an explicit, logged outcome rather than a
// silent one.
annotate({
meta: {
tiered_series_coverage_miss: true,
@@ -119,139 +115,94 @@ export async function readDailySeries(opts: {
});
}
const readRollup = async () =>
collapseRollupRowsBySource(
await prisma.measurementRollup.findMany({
where: {
userId,
type,
granularity: "DAY",
bucketStart: { gte: from, lte: to },
},
orderBy: { bucketStart: "asc" },
select: {
type: true,
source: true,
bucketStart: true,
mean: true,
count: true,
sumValue: true,
minValue: true,
maxValue: true,
},
}),
type,
priorityJson,
).slice(0, cap);
// F-DB-2 — the primary rollup read is wrapped: a transient rollup-table error
// (statement_timeout, deadlock, connection reset) must NOT 500 the tile when
// the live table can still serve it. Treat a throw as a coverage miss and fall
// straight through to the live `date_trunc` aggregate — the same posture the
// batched series reader already takes, so the single-type route and the batch
// route no longer diverge on a rollup hiccup.
let rollupRows: Awaited<ReturnType<typeof readRollup>>;
try {
rollupRows = await readRollup();
} catch (err) {
annotate({
meta: {
rollup_read_threw: true,
rollup_read_error: err instanceof Error ? err.message : String(err),
type,
},
});
return readLiveDaily({ userId, type, from, to, cap, priorityJson });
}
// Coverage-mismatch probe + inline fold (identical to the route).
const windowMs = to.getTime() - from.getTime();
const windowDays = Math.ceil(windowMs / 86_400_000);
let effectiveRows = rollupRows;
let coverageFallbackFired = false;
if (
rollupRows.length > 0 &&
rollupRows.length < SUSPICIOUS_ROW_FLOOR &&
windowDays >= SUSPICIOUS_WINDOW_DAYS
) {
const liveDayCountRows = await prisma.$queryRaw<Array<{ days: bigint }>>`
SELECT COUNT(DISTINCT date_trunc('day', m."measured_at"))::bigint AS days
FROM measurements m
WHERE m."user_id" = ${userId}
AND m."type" = ${type}::measurement_type
AND m."measured_at" >= ${from}
AND m."measured_at" <= ${to}
AND m."deleted_at" IS NULL
`;
const liveDays = Number(liveDayCountRows[0]?.days ?? BigInt(0));
if (liveDays > rollupRows.length) {
coverageFallbackFired = true;
try {
await recomputeUserRollups(userId, {
types: [type],
granularities: ["DAY"],
from,
to,
});
effectiveRows = await readRollup();
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
annotate({
meta: {
rollup_coverage_fallback_failed: true,
rollup_coverage_fallback_error: message,
type,
},
});
effectiveRows = [];
}
}
}
if (effectiveRows.length > 0) {
const useSum = CUMULATIVE_HK_TYPES.has(type);
annotate({
action: { name: "measurement.list" },
meta: {
total: effectiveRows.length,
type,
aggregate: "daily",
source: "rollup",
coverage_fallback: coverageFallbackFired,
},
});
return effectiveRows.map((r) => ({
type: r.type,
value: useSum ? (r.sumValue ?? r.mean * r.count) : r.mean,
measuredAt: r.bucketStart.toISOString(),
count: r.count,
minValue: useSum ? undefined : r.minValue,
maxValue: useSum ? undefined : r.maxValue,
}));
}
// Live `date_trunc` fallback — rollup empty for the window.
return readLiveDaily({ userId, type, from, to, cap, priorityJson });
// The daily series is cut at the user's own midnight (#1026). The DAY
// rollup tier cannot serve it: its buckets are UTC days, so east of UTC a
// sample shortly after local midnight counted on the previous day, and
// every bucket's UTC-midnight start read as the previous local day west of
// UTC. The 7-day chart buckets raw rows by the local day, so switching
// ranges moved daily totals between neighbouring days. The live fold is
// one indexed pass per window (about 0.1 s for 90 days of per-minute
// samples) and returns at most one row per day.
const timeZone = opts.timeZone ?? (await resolveUserTimezone(userId));
const rows = await readLiveBuckets({
userId,
type,
from,
to,
cap,
priorityJson,
grain: "daily",
timeZone,
});
annotate({
action: { name: "measurement.list" },
meta: { total: rows.length, type, aggregate: "daily", source: "live" },
});
return rows;
}
/** Live `date_trunc('day', …)` aggregate — the rollup-empty fallback. */
async function readLiveDaily(opts: {
/** SQL `date_trunc` unit per aggregate grain (a closed map, never input). */
const TRUNC_UNIT: Record<Exclude<AggregateGrain, "raw">, string> = {
daily: "day",
weekly: "week",
monthly: "month",
};
/**
* Live aggregate of the measurements table into buckets cut at the user's
* own calendar boundaries.
*
* Each sample is converted to wall-clock time in `timeZone` before
* truncation, so a bucket is the user's local day (or week / month), and the
* returned `measuredAt` is the instant of that local midnight. The chart
* labels a row by formatting `measuredAt` in the same zone, which lands on
* the bucket's own day for every offset, including half-hour and negative
* ones, and across a DST change.
*
* Per bucket the overlapping sources collapse to the ladder-canonical one
* before the fold (the same pick the rollup reader makes), cumulative types
* SUM and every other type averages. The result set is at most `cap` rows,
* however dense the underlying stream: the fold runs in Postgres.
*
* `type` narrows to one type; `null` folds every type in the window (the
* multi-type `GET /api/measurements?aggregate=` form).
*/
export async function readLiveBuckets(opts: {
userId: string;
type: MeasurementType;
type: MeasurementType | null;
from: Date;
to: Date;
cap: number;
priorityJson: unknown;
grain: Exclude<AggregateGrain, "raw">;
timeZone: string;
}): Promise<DailySeriesRow[]> {
const { userId, type, from, to, cap, priorityJson } = opts;
const truncUnitLiteral = Prisma.raw(`'day'`);
const useSum = CUMULATIVE_HK_TYPES.has(type);
const aggregator = useSum
? Prisma.raw(`SUM(m."value")::double precision`)
: Prisma.raw(`AVG(m."value")::double precision`);
const { userId, type, from, to, cap, priorityJson, grain } = opts;
// A zone Postgres does not know would 500 the read; the resolver already
// validates stored zones, this guards a caller passing a raw string.
const timeZone = isValidTimezone(opts.timeZone) ? opts.timeZone : "UTC";
// `date_trunc` needs its unit as a literal; the unit comes from the closed
// `TRUNC_UNIT` map keyed by the Zod-constrained grain, never from input.
const unit = Prisma.raw(`'${TRUNC_UNIT[grain]}'`);
// Cumulative types SUM per bucket, every other type averages. The type list
// is the closed code constant, spliced as literals; `s."type"` is a GROUP BY
// column, so choosing the aggregate per group is legal.
const cumulativeList = [...CUMULATIVE_HK_TYPES]
.map((t) => `'${t}'`)
.join(",");
const aggregator = Prisma.raw(
`(CASE WHEN s."type"::text IN (${cumulativeList}) THEN SUM(s."value") ELSE AVG(s."value") END)::double precision`,
);
const rankRaw = Prisma.raw(
buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'),
);
const typeFilter = type
? Prisma.sql`AND m."type" = ${type}::measurement_type`
: Prisma.empty;
// measured_at is a UTC wall-clock timestamp: pin it to UTC, re-read it in
// the user's zone and truncate there. The bucket is computed once in `src`
// and referenced by name afterwards, so the zone parameter never has to
// match itself across SELECT and GROUP BY.
const buckets = await prisma.$queryRaw<
Array<{
type: string;
@@ -262,58 +213,54 @@ async function readLiveDaily(opts: {
max_value: number | null;
}>
>`
WITH canon AS (
SELECT DISTINCT ON (m."type", date_trunc(${truncUnitLiteral}, m."measured_at"))
m."type" AS t,
date_trunc(${truncUnitLiteral}, m."measured_at") AS d,
m."source" AS canon
WITH src AS (
SELECT
m."type",
m."source",
m."value",
date_trunc(${unit}, (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}) AS d
FROM measurements m
WHERE m."user_id" = ${userId}
AND m."measured_at" >= ${from}
AND m."measured_at" <= ${to}
AND m."deleted_at" IS NULL
AND m."type" = ${type}::measurement_type
ORDER BY m."type", date_trunc(${truncUnitLiteral}, m."measured_at"), (${rankRaw}), m."source"
${typeFilter}
),
canon AS (
SELECT DISTINCT ON (m."type", m.d)
m."type" AS t,
m.d AS d,
m."source" AS canon
FROM src m
ORDER BY m."type", m.d, (${rankRaw}), m."source"
)
SELECT
m."type"::text AS type,
date_trunc(${truncUnitLiteral}, m."measured_at") AS bucket_start,
s."type"::text AS type,
s.d AT TIME ZONE ${timeZone} AS bucket_start,
${aggregator} AS avg,
COUNT(*)::int AS cnt,
MIN(m."value") AS min_value,
MAX(m."value") AS max_value
FROM measurements m
MIN(s."value") AS min_value,
MAX(s."value") AS max_value
FROM src s
JOIN canon c
ON c.t = m."type"
AND c.d = date_trunc(${truncUnitLiteral}, m."measured_at")
AND c.canon = m."source"
WHERE m."user_id" = ${userId}
AND m."measured_at" >= ${from}
AND m."measured_at" <= ${to}
AND m."deleted_at" IS NULL
AND m."type" = ${type}::measurement_type
GROUP BY m."type", bucket_start
ORDER BY bucket_start ASC
ON c.t = s."type"
AND c.d = s.d
AND c.canon = s."source"
GROUP BY s."type", s.d
ORDER BY s.d ASC
LIMIT ${cap}
`;
annotate({
action: { name: "measurement.list" },
meta: { total: buckets.length, type, aggregate: "daily" },
return buckets.map((b) => {
const isSum = CUMULATIVE_HK_TYPES.has(b.type as MeasurementType);
return {
type: b.type,
value: Number(b.avg),
measuredAt: b.bucket_start.toISOString(),
count: Number(b.cnt),
// A SUM has no meaningful intra-day spread; the chart draws the band
// for averaged types only.
minValue: isSum || b.min_value === null ? undefined : Number(b.min_value),
maxValue: isSum || b.max_value === null ? undefined : Number(b.max_value),
};
});
return buckets.map((b) => ({
type: b.type,
value: Number(b.avg),
measuredAt: b.bucket_start.toISOString(),
count: Number(b.cnt),
// The rollup path emits these; the live fallback used to omit them, so a
// rollup-cold account silently lost the min/max spread band on the chart
// for the same underlying data — contradicting this file's own
// byte-for-byte parity claim.
minValue: b.min_value === null ? undefined : Number(b.min_value),
maxValue: b.max_value === null ? undefined : Number(b.max_value),
// The rollup path emits these; the live fallback used to omit them, so a
// rollup-cold account silently lost the min/max spread band on the chart
// for the same underlying data — contradicting this file's own
// byte-for-byte parity claim.
}));
}