From b2a5f2c410a7dd0250fd58e7a2b9cf2978ad4564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Mon, 18 May 2026 12:17:21 +0200 Subject: [PATCH] feat(rollups): populate sum_value alongside mean/count in every fold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v1.4.39 schema scaffolding added `measurement_rollups.sum_value` for the cumulative-metric daily totals (steps, flights, distance, daylight, active-energy). Extends the rollup aggregator to also `SUM` each bucket and persist the column for every (user, type, granularity, bucket) row regardless of metric kind — spot metrics carry the column for free because the SUM rides on the same $queryRaw aggregate as AVG / MIN / MAX / STDDEV_POP / REGR_*. Boot-time backfill discovery now unions in users whose existing DAY rollup rows carry `sum_value IS NULL`. Re-folding converges those rows because `persistRollupRows` always writes the new column on upsert, and the union runs as one indexed pass alongside the per- type missing-coverage discovery. Adds `src/lib/measurements/rollup-read-cumulative.ts` with the new `readCumulativeDaySums` / `readCumulativeDaySumsBatch` helpers consumers use to swap off the per-type chunked findMany loop. Helpers fall back to `mean * count` when the legacy NULL surfaces so the chart never paints a hole during the boot-backfill convergence window. Audit anchor: .planning/round-v1438-perf-analysis.md §3 (cumulative daily sums) + §5 P3. --- .../measurements/rollup-read-cumulative.ts | 167 ++++++++++++++++++ src/lib/measurements/rollups.ts | 45 +++-- 2 files changed, 202 insertions(+), 10 deletions(-) create mode 100644 src/lib/measurements/rollup-read-cumulative.ts diff --git a/src/lib/measurements/rollup-read-cumulative.ts b/src/lib/measurements/rollup-read-cumulative.ts new file mode 100644 index 000000000..d88f4fe6f --- /dev/null +++ b/src/lib/measurements/rollup-read-cumulative.ts @@ -0,0 +1,167 @@ +/** + * v1.4.39 W-SUM — read helpers for cumulative-metric daily sums. + * + * The v1.4.36 read path consumes the rollup table for spot-metric means + * via `rollup-read.ts`. Cumulative HK types (steps, flights, distance, + * daylight, active-energy) need the per-day SUM rather than the MEAN — + * every row is a partial-day slice from HealthKit, so MEAN understates + * the daily total by the per-bucket sample count. + * + * The v1.4.39 schema scaffolding (migration 0072) added `sum_value` to + * `measurement_rollups`. The writer in `rollups.ts` populates it on + * every fold (including for non-cumulative types — the cost is one + * extra `SUM` in the existing aggregator). The legacy-NULL backfill + * runs through `enqueueBootTimeRollupBackfill` so existing rows + * converge on the next worker boot. + * + * Callers fall back to `mean * count` when `sumValue` is null. The + * shortcut is mathematically exact for a single source per day: + * + * sum = mean × count = (Σvalue / count) × count = Σvalue + * + * Multiple sources for the same `(user, type, day)` mix into the same + * bucket today — the source-priority resolution that + * `pickCanonicalSourceRows` runs in JS happens AFTER the rollup + * aggregate is built. The W8c per-source rollup deferred to v1.5 fixes + * that; until then, the cumulative read paths trust the bucket sum as + * the canonical daily total and the iOS drain (which collapses per + * source via `dailyStatsExternalId`) keeps Marc's tenant on the + * single-source-per-day path. + */ +import { prisma } from "@/lib/db"; +import type { MeasurementType } from "@/generated/prisma/client"; +import { CUMULATIVE_HK_TYPES } from "./apple-health-mapping"; + +/** + * Cumulative measurement types the rollup `sum_value` column serves. + * Re-exported from the canonical Apple Health mapping so the + * consumer-side gate stays in one place. + * + * Audit anchor: `.planning/round-v1438-perf-analysis.md` §3 + * "Cumulative daily sums" + §5 P3. + */ +export type CumulativeType = + | "ACTIVITY_STEPS" + | "ACTIVE_ENERGY_BURNED" + | "FLIGHTS_CLIMBED" + | "WALKING_RUNNING_DISTANCE" + | "TIME_IN_DAYLIGHT"; + +/** + * `true` when `type` is one of the cumulative HK types whose daily + * total the rollup `sum_value` column carries. + */ +export function isCumulativeType(type: MeasurementType): type is CumulativeType { + return CUMULATIVE_HK_TYPES.has(type); +} + +/** One DAY-bucket row read from `measurement_rollups`. */ +export interface CumulativeDaySumRow { + bucketStart: Date; + /** + * Per-bucket SUM. `null` only when the bucket pre-dates the v1.4.39 + * writer change AND the boot-time backfill has not yet converged on + * this user. Callers fall back to `mean * count` for the null case. + */ + sumValue: number | null; + count: number; + mean: number; +} + +/** + * Read every DAY bucket for `(userId, type)` since `since`. Returns + * rows ascending by `bucketStart`. Pre-v1.4.39 rows surface here with + * `sumValue = null`; the caller decides whether to fall back to + * `mean * count` or skip them. + * + * Bounded to the cumulative-type set so a misuse (passing a spot + * metric) is a type error rather than a silent SUM on the wrong + * column. The query is a plain `findMany` against the + * `(userId, type, granularity, bucketStart)` composite primary key — + * single round-trip, fully indexed. + */ +export async function readCumulativeDaySums( + userId: string, + type: CumulativeType, + since: Date, +): Promise { + const rows = await prisma.measurementRollup.findMany({ + where: { + userId, + type: type as MeasurementType, + granularity: "DAY", + bucketStart: { gte: since }, + }, + orderBy: { bucketStart: "asc" }, + select: { + bucketStart: true, + sumValue: true, + count: true, + mean: true, + }, + }); + return rows.map((r) => ({ + bucketStart: r.bucketStart, + sumValue: r.sumValue, + count: r.count, + mean: r.mean, + })); +} + +/** + * Batch read DAY buckets for several cumulative types in one round + * trip. Eliminates the per-type chunked-findMany loop in + * `/api/analytics` A2 for the five highest-row-count metrics on + * Marc's tenant. + * + * Returns a Map keyed on `MeasurementType` so the caller can route + * each type's rows back into the per-metric branch without scanning + * the flat list. + */ +export async function readCumulativeDaySumsBatch( + userId: string, + types: readonly CumulativeType[], + since: Date, +): Promise> { + if (types.length === 0) return new Map(); + const rows = await prisma.measurementRollup.findMany({ + where: { + userId, + type: { in: types as unknown as MeasurementType[] }, + granularity: "DAY", + bucketStart: { gte: since }, + }, + orderBy: [{ type: "asc" }, { bucketStart: "asc" }], + select: { + type: true, + bucketStart: true, + sumValue: true, + count: true, + mean: true, + }, + }); + const out = new Map(); + for (const t of types) out.set(t, []); + for (const row of rows) { + const bucket = out.get(row.type as CumulativeType); + if (!bucket) continue; + bucket.push({ + bucketStart: row.bucketStart, + sumValue: row.sumValue, + count: row.count, + mean: row.mean, + }); + } + return out; +} + +/** + * Resolve a per-bucket cumulative total. Reads `sumValue` when the + * v1.4.39 writer has populated it; falls back to `mean * count` for + * legacy NULL rows so the chart never paints a hole during the boot- + * backfill convergence window. + */ +export function resolveBucketSum(row: CumulativeDaySumRow): number { + if (row.sumValue !== null) return row.sumValue; + return row.mean * row.count; +} diff --git a/src/lib/measurements/rollups.ts b/src/lib/measurements/rollups.ts index 897ce2f21..9cb3fe394 100644 --- a/src/lib/measurements/rollups.ts +++ b/src/lib/measurements/rollups.ts @@ -107,6 +107,7 @@ interface RollupRow { mean: number | null; min_value: number | null; max_value: number | null; + sum_value: number | null; sd: number | null; slope: number | null; r2: number | null; @@ -387,6 +388,7 @@ async function runRollupAggregate(input: { AVG(m."value")::double precision AS mean, MIN(m."value")::double precision AS min_value, MAX(m."value")::double precision AS max_value, + SUM(m."value")::double precision AS sum_value, STDDEV_POP(m."value")::double precision AS sd, REGR_SLOPE( m."value", @@ -419,6 +421,7 @@ async function runRollupAggregate(input: { AVG(m."value")::double precision AS mean, MIN(m."value")::double precision AS min_value, MAX(m."value")::double precision AS max_value, + SUM(m."value")::double precision AS sum_value, STDDEV_POP(m."value")::double precision AS sd, REGR_SLOPE( m."value", @@ -483,6 +486,12 @@ async function persistRollupRows( mean: row.mean ?? 0, minValue: row.min_value ?? 0, maxValue: row.max_value ?? 0, + // v1.4.39 W-SUM — populate for every type. Cumulative read + // paths (steps, flights, distance, daylight, active-energy) + // consume this column directly; spot metrics carry it for + // free because the underlying SUM aggregator runs in the + // same query as AVG / MIN / MAX. + sumValue: row.sum_value ?? null, sd: row.sd, slope: row.slope, r2: row.r2, @@ -493,6 +502,7 @@ async function persistRollupRows( mean: row.mean ?? 0, minValue: row.min_value ?? 0, maxValue: row.max_value ?? 0, + sumValue: row.sum_value ?? null, sd: row.sd, slope: row.slope, r2: row.r2, @@ -741,17 +751,32 @@ export async function enqueueBootTimeRollupBackfill(): Promise<{ // the LEFT JOIN) marks the unmatched partitions. A user surfaces // here when ANY type is missing — including the brand-new-type // case that v1.4.35.1's zero-rollup-only shape silently stranded. + // + // v1.4.39 W-SUM — the discovery also surfaces users whose existing + // DAY rollup rows carry `sum_value IS NULL`. Those rows pre-date + // the v1.4.39 writer change; re-folding the user converges them + // because `persistRollupRows` always writes the new column on + // upsert. The union runs in one query so the existing per-type + // coverage gap and the legacy-NULL backfill share a single planner + // pass. const users = await prisma.$queryRaw>` - SELECT DISTINCT mt."user_id" AS id - FROM ( - SELECT DISTINCT m."user_id", m."type" - FROM measurements m - ) mt - LEFT JOIN measurement_rollups r - ON r."user_id" = mt."user_id" - AND r."type" = mt."type" - AND r."granularity" = 'DAY' - WHERE r."bucket_start" IS NULL + SELECT DISTINCT id FROM ( + SELECT DISTINCT mt."user_id" AS id + FROM ( + SELECT DISTINCT m."user_id", m."type" + FROM measurements m + ) mt + LEFT JOIN measurement_rollups r + ON r."user_id" = mt."user_id" + AND r."type" = mt."type" + AND r."granularity" = 'DAY' + WHERE r."bucket_start" IS NULL + UNION + SELECT DISTINCT r2."user_id" AS id + FROM measurement_rollups r2 + WHERE r2."granularity" = 'DAY' + AND r2."sum_value" IS NULL + ) discovery `; if (users.length === 0) {