fix(charts): serve the All range's coarse buckets as their average day

The All range is read from the WEEK or MONTH rollup tier once a history is
long enough, from DAY rows for a short one, and from the live table on a
coverage miss. The chart folds every path into the average day of each
week or month, but a tier row carried a cumulative metric's whole total
and a level's reading-weighted mean. A month of steps therefore read about
thirty times higher for a long history than for a short one, under a
caption that said average.

A tier row now carries the average day of its bucket on every tier: a
cumulative metric's total over the days that hold a reading, a level's
mean of its daily means. The rollup fold counts the bucket's days and adds
up their means for it. Only the web chart's series reads (the chart
endpoint and the dashboard batch) use these rows.
This commit is contained in:
Marc-André Bombeck committed 2026-09-27 14:08:49 +02:00
1 parent c7e53176c1
commit b45a668cf4
6 files changed
+365 -25

No files matched your search

+3 -1
View File
@@ -40744,7 +40744,9 @@ components:
$ref: "#/components/schemas/MeasurementType"
value:
type: number
description: Daily mean (spot metrics) or SUM (cumulative).
description: "The day's mean (spot metrics) or total (cumulative). A row of a coarser tier (a window longer than the
daily cap) carries the average day of its bucket, the same quantity: a spot metric's mean of its
daily means, a cumulative metric's total over the days of the bucket that hold a reading."
measuredAt:
type: string
format: date-time
+5 -1
View File
@@ -760,7 +760,11 @@ const seriesResponse = z
// `GET /api/measurements?aggregate=daily&source=rollup` emits.
const seriesBatchRowSchema = z.object({
type: measurementTypeEnum,
value: z.number().describe("Daily mean (spot metrics) or SUM (cumulative)."),
value: z
.number()
.describe(
"The day's mean (spot metrics) or total (cumulative). A row of a coarser tier (a window longer than the daily cap) carries the average day of its bucket, the same quantity: a spot metric's mean of its daily means, a cumulative metric's total over the days of the bucket that hold a reading.",
),
measuredAt: z.iso
.datetime({ offset: true })
.describe(
@@ -122,9 +122,18 @@ function folded(
sumY += value;
}
const values = readings.map(([, v]) => v);
const byDay = new Map<string, number[]>();
for (const [day, value] of readings) {
byDay.set(day, [...(byDay.get(day) ?? []), value]);
}
const dayMeans = [...byDay.values()].map(
(v) => v.reduce((s, x) => s + x, 0) / v.length,
);
return {
bucket_start: new Date(bucketStart),
count: readings.length,
days: byDay.size,
sum_day_mean: dayMeans.reduce((s, x) => s + x, 0),
sum_y: sumY,
min_value: Math.min(...values),
max_value: Math.max(...values),
@@ -475,27 +484,115 @@ describe("readTieredRollupSeries", () => {
});
});
it("serves the summed total for step-like metrics and drops the spread", async () => {
aggregate.mockResolvedValueOnce(
spanOf("2024-01-01T00:00:00.000Z", "2026-01-01T00:00:00.000Z"),
);
queryRaw.mockResolvedValueOnce([
folded(
"2024-01-01T00:00:00.000Z",
Array.from({ length: 30 }, (_, i): [string, number] => [
`2024-01-${String(i + 1).padStart(2, "0")}`,
8000,
]),
),
// The chart folds its shorter ranges (and the live fallback) from day
// points into the average day of each week or month. A tier row must be the
// same quantity, whichever tier the history's length selects.
describe("the average day at every tier", () => {
// January 2026: steps on 20 of its days, from 4 000 to 13 500.
const JAN = Array.from({ length: 20 }, (_, i): [string, number] => [
`2026-01-${String(i + 1).padStart(2, "0")}`,
4_000 + i * 500,
]);
const result = await readTieredRollupSeries({
userId: "u",
type: "ACTIVITY_STEPS",
...win(3650),
const averageDay = JAN.reduce((s, [, v]) => s + v, 0) / JAN.length;
/** How the chart folds a tier's rows into the month on screen. */
function chartMonth(rows: Array<{ measuredAt: string; value: number }>) {
return bucketTimeSeries(
rows.map((row) => ({
timestamp: new Date(row.measuredAt),
values: { v: row.value },
})),
{ bucket: "month", timeZone: "Europe/Berlin" },
).points.map((p) => p.values.v);
}
it("serves a MONTH row of a long history as its average day", async () => {
aggregate.mockResolvedValueOnce(
spanOf("2023-01-01T00:00:00.000Z", "2026-01-20T00:00:00.000Z"),
);
queryRaw.mockResolvedValueOnce([folded("2026-01-01T00:00:00.000Z", JAN)]);
const result = await readTieredRollupSeries({
userId: "u",
type: "ACTIVITY_STEPS",
...win(3650),
});
expect(result?.granularity).toBe("MONTH");
expect(result?.rows[0].value).toBe(averageDay);
expect(result?.rows[0].count).toBe(JAN.length);
expect(result?.rows[0].minValue).toBeUndefined();
expect(result?.rows[0].maxValue).toBeUndefined();
});
it("gives the same month for a short history served by days", async () => {
aggregate.mockResolvedValueOnce(
spanOf("2026-01-01T00:00:00.000Z", "2026-01-20T00:00:00.000Z"),
);
findMany.mockResolvedValueOnce(
JAN.map(([day, value]) => dayRow(day, "APPLE_HEALTH", 1, value)),
);
const result = await readTieredRollupSeries({
userId: "u",
type: "ACTIVITY_STEPS",
...win(3650),
});
expect(result?.granularity).toBe("DAY");
expect(chartMonth(result!.rows)).toEqual([averageDay]);
});
it("gives each WEEK row of a middle-length history its own average day", async () => {
aggregate.mockResolvedValueOnce(
spanOf("2025-01-01T00:00:00.000Z", "2026-01-20T00:00:00.000Z"),
);
const week = JAN.slice(4, 11); // Monday 5 to Sunday 11 January
queryRaw.mockResolvedValueOnce([
folded("2026-01-05T00:00:00.000Z", week),
]);
const result = await readTieredRollupSeries({
userId: "u",
type: "ACTIVITY_STEPS",
...win(3650),
});
expect(result?.granularity).toBe("WEEK");
expect(result?.rows[0].value).toBe(
week.reduce((s, [, v]) => s + v, 0) / week.length,
);
});
it("weighs each day once for a level, as the chart's own fold does", async () => {
// Two readings on the 3rd (120, 130), one on the 4th (100): the
// average day is 112.5, while every reading weighing one gives 116.7.
const readings: Array<[string, number]> = [
["2026-01-03", 120],
["2026-01-03", 130],
["2026-01-04", 100],
];
aggregate.mockResolvedValueOnce(
spanOf("2023-01-01T00:00:00.000Z", "2026-01-04T00:00:00.000Z"),
);
queryRaw.mockResolvedValueOnce([
folded("2026-01-01T00:00:00.000Z", readings),
]);
const monthly = await readTieredRollupSeries({
userId: "u",
type: "BLOOD_PRESSURE_SYS",
...win(3650),
});
expect(monthly?.rows[0].value).toBe(112.5);
aggregate.mockResolvedValueOnce(
spanOf("2026-01-03T00:00:00.000Z", "2026-01-04T00:00:00.000Z"),
);
findMany.mockResolvedValueOnce([
dayRow("2026-01-03", "MANUAL", 2, 125),
dayRow("2026-01-04", "MANUAL", 1, 100),
]);
const daily = await readTieredRollupSeries({
userId: "u",
type: "BLOOD_PRESSURE_SYS",
...win(3650),
});
expect(chartMonth(daily!.rows)).toEqual([112.5]);
});
expect(result?.rows[0].value).toBe(240_000);
expect(result?.rows[0].minValue).toBeUndefined();
expect(result?.rows[0].maxValue).toBeUndefined();
});
// v1.37.29 — the tier keys off the ACTUAL data span, not the requested
+39 -3
View File
@@ -296,6 +296,37 @@ export function pickRollupGranularityForWindow(
*/
const TIER_ROW_ANCHOR_MS = 12 * 3_600_000;
/**
* A cumulative type's tier row as the average day of its bucket: the bucket's
* total over the days that hold a reading. A day row is its own total, which
* is the same quantity. The chart folds its shorter ranges from day totals
* into the mean day of each week or month, and the live fallback serves day
* totals too; a WEEK or MONTH row carrying its whole total put a monthly sum
* where every other path puts a daily figure, so the "All" range read ~30
* times higher once a history was long enough to be served monthly, under a
* caption that said average.
*/
function dailyAverage(r: {
sumValue: number | null;
mean: number;
count: number;
days: number;
}): number {
const total = r.sumValue ?? r.mean * r.count;
return r.days > 1 ? total / r.days : total;
}
/**
* A level type's tier row: the mean of its days' means. The count-weighted
* `mean` let a day with many readings outweigh the rest, while the chart's
* shorter ranges and the live fallback weigh each day once.
*/
function chartLevel(r: { mean: number; dayMean?: number }): number {
return typeof r.dayMean === "number" && Number.isFinite(r.dayMean)
? r.dayMean
: r.mean;
}
/** DAY → WEEK → MONTH → YEAR, finest first. */
const TIER_ORDER: RollupGranularity[] = ["DAY", "WEEK", "MONTH", "YEAR"];
@@ -317,8 +348,10 @@ const TIER_ORDER: RollupGranularity[] = ["DAY", "WEEK", "MONTH", "YEAR"];
* the compositional contract above), so a 5-year window rendered as ~60
* MONTH points carries the same trend as the ~1 800 DAY points would have
* — minus the truncation. `minValue` / `maxValue` ride through for the
* range band on spot metrics; cumulative metrics (steps, energy,
* distance) surface the bucket's summed total and drop the spread.
* range band on spot metrics. The value is the average day of the bucket
* at every tier, as the chart's own fold of day points gives it: a level's
* days each weigh one, and a cumulative metric (steps, energy, distance)
* carries its total over the days that hold a reading, without a spread.
*
* Coverage handling: the tier is chosen from the span of the window's DAY
* buckets and read folded from them. Returns `null` when the window holds no
@@ -409,7 +442,10 @@ export async function readTieredRollupSeries(opts: {
granularity,
rows: rows.map((r) => ({
type,
value: useSum ? (r.sumValue ?? r.mean * r.count) : r.mean,
// The chart's quantity at every tier: the average day of the bucket,
// each day weighing one, as its shorter ranges and the live fallback
// fold it.
value: useSum ? dailyAverage(r) : chartLevel(r),
measuredAt: new Date(
r.bucketStart.getTime() + TIER_ROW_ANCHOR_MS,
).toISOString(),
+16 -1
View File
@@ -462,6 +462,13 @@ export function utcBucketEnd(at: Date, granularity: RollupGranularity): Date {
export interface CanonicalRollupBucket {
bucketStart: Date;
count: number;
/** Days of the bucket that hold a reading (1 for a DAY bucket). */
days: number;
/**
* The mean of the bucket's daily means, each day weighing one: what the
* chart's own fold of day points gives. `mean` weighs every reading.
*/
dayMean: number;
mean: number;
minValue: number;
maxValue: number;
@@ -483,6 +490,10 @@ export interface CanonicalRollupBucket {
interface FoldedBucketRow {
bucket_start: Date;
count: number;
/** Canonical DAY rows folded into the bucket: its days with a reading. */
days: number;
/** The days' own means added up (each day weighs one). */
sum_day_mean: number;
sum_y: number;
min_value: number;
max_value: number;
@@ -528,6 +539,8 @@ function composeFoldedBucket(row: FoldedBucketRow): CanonicalRollupBucket {
return {
bucketStart: new Date(row.bucket_start),
count,
days: Number(row.days),
dayMean: Number(row.sum_day_mean) / Number(row.days),
mean,
minValue: Number(row.min_value),
maxValue: Number(row.max_value),
@@ -616,7 +629,7 @@ export async function readCanonicalRollupBuckets(opts: {
? opts.userPriorityJson
: await loadUserSourcePriority(userId);
return collapseRollupRowsBySource(dayRows, type, priority).map(
({ source: _source, ...day }) => day,
({ source: _source, ...day }) => ({ ...day, days: 1, dayMean: day.mean }),
);
}
@@ -653,6 +666,8 @@ export async function readCanonicalRollupBuckets(opts: {
date_trunc(${unit}, c."bucket_start" AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
AS bucket_start,
SUM(c."count")::int AS count,
COUNT(*)::int AS days,
SUM(c."mean")::double precision AS sum_day_mean,
SUM(COALESCE(c."sum_value", c."count" * c."mean"))::double precision
AS sum_y,
MIN(c."min_value")::double precision AS min_value,
@@ -0,0 +1,186 @@
/**
* v1.39.4 — the chart's "All" range means one thing at every history length.
*
* A long history is served from the WEEK or MONTH rollup tier, a short one
* from DAY rows, and a coverage miss from the live table. The chart folds
* whatever arrives into the average day of each week (a range up to two
* years) or month (longer). Seeded with real rows and folded through the real
* rollup writer, a fully covered week or month must read the same average day
* on every path: for steps the average daily total, for a level the average
* of its days. Before, a MONTH-tier month of steps read
* as the month's total, about thirty times the same month served by days.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import { readDailySeries } from "@/lib/measurements/daily-series-read";
import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
import { bucketTimeSeries } from "@/lib/charts/bucket-time-series";
import type { MeasurementType } from "@/generated/prisma/client";
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/jobs/boss-instance", () => ({
getGlobalBoss: vi.fn(() => null),
}));
const TZ = "America/Los_Angeles";
const DAY_MS = 86_400_000;
const NOW = new Date("2026-09-20T17:00:00.000Z");
/** The week and the month every history below covers fully. */
const WEEK = { from: "2026-06-08", to: "2026-06-14" };
const MONTH = { from: "2026-06-01", to: "2026-06-30" };
let seq = 0;
async function seedUser() {
seq += 1;
return getPrismaClient().user.create({
data: {
username: `chart-all-${seq}`,
email: `chart-all-${seq}@example.test`,
role: "USER",
timezone: TZ,
},
});
}
/**
* `days` days ending yesterday, at 18:00 UTC (11:00 in Los Angeles, the same
* calendar day in both). Steps come as two rows a day; the level as one or
* two readings, so a reading-weighted mean would differ from a day-weighted
* one.
*/
function history(days: number) {
const rows: Array<{ type: MeasurementType; value: number; at: Date }> = [];
const end = Date.parse("2026-09-19T18:00:00.000Z");
for (let i = 0; i < days; i += 1) {
const at = new Date(end - i * DAY_MS);
const steps = 6_000 + ((i * 37) % 5_000);
rows.push({ type: "ACTIVITY_STEPS", value: steps - 1_000, at });
rows.push({
type: "ACTIVITY_STEPS",
value: 1_000,
at: new Date(at.getTime() + 60_000),
});
rows.push({ type: "PULSE", value: 60 + (i % 9), at });
if (i % 3 === 0) {
rows.push({
type: "PULSE",
value: 90,
at: new Date(at.getTime() + 120_000),
});
}
}
return rows;
}
/** The average day of a period straight from the seeded rows. */
function expectedAverageDay(
rows: ReturnType<typeof history>,
type: MeasurementType,
period: { from: string; to: string },
): number {
const byDay = new Map<string, number[]>();
for (const r of rows) {
if (r.type !== type) continue;
const day = r.at.toISOString().slice(0, 10);
if (day < period.from || day > period.to) continue;
byDay.set(day, [...(byDay.get(day) ?? []), r.value]);
}
const perDay = [...byDay.values()].map((values) =>
type === "ACTIVITY_STEPS"
? values.reduce((s, v) => s + v, 0)
: values.reduce((s, v) => s + v, 0) / values.length,
);
return perDay.reduce((s, v) => s + v, 0) / perDay.length;
}
/** The chart's point for `period`, folded at the bucket the range shows. */
async function chartPoint(
userId: string,
type: MeasurementType,
bucket: "week" | "month",
period: { from: string },
) {
const rows = await readDailySeries({
userId,
type,
from: new Date(NOW.getTime() - 3_650 * DAY_MS),
to: NOW,
priorityJson: null,
timeZone: TZ,
});
const points = bucketTimeSeries(
rows.map((row) => ({
timestamp: new Date(row.measuredAt),
values: { v: row.value },
})),
{ bucket, timeZone: TZ },
).points;
const point = points.find(
(p) => new Date(p.timestamp).toISOString().slice(0, 10) === period.from,
);
return point?.values.v;
}
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
});
afterEach(() => {
vi.restoreAllMocks();
});
describe("chart All range — one quantity at every history length", () => {
it.each([
["days (a short history)", 200],
["weeks (one to two years)", 500],
["months (a long history)", 900],
])(
"reads the same average day for a period served by %s, and by the live table",
async (_label, days) => {
const user = await seedUser();
const rows = history(days);
await getPrismaClient().measurement.createMany({
data: rows.map((r) => ({
userId: user.id,
type: r.type,
value: r.value,
unit: r.type === "PULSE" ? "bpm" : "count",
source: "APPLE_HEALTH" as const,
measuredAt: r.at,
})),
});
await recomputeUserRollups(user.id, {
from: new Date(NOW.getTime() - (days + 40) * DAY_MS),
to: NOW,
});
// The chart folds a span over two years into months, else weeks.
const bucket = days > 730 ? "month" : "week";
const period = bucket === "month" ? MONTH : WEEK;
for (const type of ["ACTIVITY_STEPS", "PULSE"] as const) {
const expected = expectedAverageDay(rows, type, period);
const tier = await chartPoint(user.id, type, bucket, period);
expect(tier, `${type} from the tier`).toBeCloseTo(expected, 6);
}
// A coverage miss serves the live table's days instead. Its daily
// path keeps the oldest 365 days, so only the short history still
// reaches June through it.
if (days <= 365) {
await getPrismaClient().measurementRollup.deleteMany({
where: { userId: user.id },
});
for (const type of ["ACTIVITY_STEPS", "PULSE"] as const) {
const live = await chartPoint(user.id, type, bucket, period);
expect(live, `${type} from the live table`).toBeCloseTo(
expectedAverageDay(rows, type, period),
6,
);
}
}
},
);
});