tom/HealthLog
1
0 Fork 0
mirror of https://github.com/MBombeck/HealthLog.git synced 2026-11-10 13:22:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(rollups): populate sum_value alongside mean/count in every fold

Browse Source 
The v1.4.39 schema scaffolding added `measurement_rollups.sum_value`
for the cumulative-metric daily totals (steps, flights, distance,
daylight, active-energy). Extends the rollup aggregator to also `SUM`
each bucket and persist the column for every (user, type,
granularity, bucket) row regardless of metric kind — spot metrics
carry the column for free because the SUM rides on the same
$queryRaw aggregate as AVG / MIN / MAX / STDDEV_POP / REGR_*.

Boot-time backfill discovery now unions in users whose existing DAY
rollup rows carry `sum_value IS NULL`. Re-folding converges those
rows because `persistRollupRows` always writes the new column on
upsert, and the union runs as one indexed pass alongside the per-
type missing-coverage discovery.

Adds `src/lib/measurements/rollup-read-cumulative.ts` with the new
`readCumulativeDaySums` / `readCumulativeDaySumsBatch` helpers
consumers use to swap off the per-type chunked findMany loop.
Helpers fall back to `mean * count` when the legacy NULL surfaces so
the chart never paints a hole during the boot-backfill convergence
window.

Audit anchor: .planning/round-v1438-perf-analysis.md §3 (cumulative
daily sums) + §5 P3.
This commit is contained in:
Marc-André Bombeck committed 2026-05-18 12:17:21 +02:00
1 parent 6742b4f90b
commit b2a5f2c410
2 files changed
+202 -10
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/lib/measurements/rollup-read-cumulative.ts
+167
View File
@@ -0,0 +1,167 @@
/**
* v1.4.39 W-SUM — read helpers for cumulative-metric daily sums.
*
* The v1.4.36 read path consumes the rollup table for spot-metric means
* via `rollup-read.ts`. Cumulative HK types (steps, flights, distance,
* daylight, active-energy) need the per-day SUM rather than the MEAN —
* every row is a partial-day slice from HealthKit, so MEAN understates
* the daily total by the per-bucket sample count.
*
* The v1.4.39 schema scaffolding (migration 0072) added `sum_value` to
* `measurement_rollups`. The writer in `rollups.ts` populates it on
* every fold (including for non-cumulative types — the cost is one
* extra `SUM` in the existing aggregator). The legacy-NULL backfill
* runs through `enqueueBootTimeRollupBackfill` so existing rows
* converge on the next worker boot.
*
* Callers fall back to `mean * count` when `sumValue` is null. The
* shortcut is mathematically exact for a single source per day:
*
* sum = mean × count = (Σvalue / count) × count = Σvalue
*
* Multiple sources for the same `(user, type, day)` mix into the same
* bucket today — the source-priority resolution that
* `pickCanonicalSourceRows` runs in JS happens AFTER the rollup
* aggregate is built. The W8c per-source rollup deferred to v1.5 fixes
* that; until then, the cumulative read paths trust the bucket sum as
* the canonical daily total and the iOS drain (which collapses per
* source via `dailyStatsExternalId`) keeps Marc's tenant on the
* single-source-per-day path.
*/
import { prisma } from "@/lib/db";
import type { MeasurementType } from "@/generated/prisma/client";
import { CUMULATIVE_HK_TYPES } from "./apple-health-mapping";
/**
* Cumulative measurement types the rollup `sum_value` column serves.
* Re-exported from the canonical Apple Health mapping so the
* consumer-side gate stays in one place.
*
* Audit anchor: `.planning/round-v1438-perf-analysis.md` §3
* "Cumulative daily sums" + §5 P3.
*/
export type CumulativeType =
| "ACTIVITY_STEPS"
| "ACTIVE_ENERGY_BURNED"
| "FLIGHTS_CLIMBED"
| "WALKING_RUNNING_DISTANCE"
| "TIME_IN_DAYLIGHT";
/**
* `true` when `type` is one of the cumulative HK types whose daily
* total the rollup `sum_value` column carries.
*/
export function isCumulativeType(type: MeasurementType): type is CumulativeType {
return CUMULATIVE_HK_TYPES.has(type);
}
/** One DAY-bucket row read from `measurement_rollups`. */
export interface CumulativeDaySumRow {
bucketStart: Date;
/**
* Per-bucket SUM. `null` only when the bucket pre-dates the v1.4.39
* writer change AND the boot-time backfill has not yet converged on
* this user. Callers fall back to `mean * count` for the null case.
*/
sumValue: number | null;
count: number;
mean: number;
}
/**
* Read every DAY bucket for `(userId, type)` since `since`. Returns
* rows ascending by `bucketStart`. Pre-v1.4.39 rows surface here with
* `sumValue = null`; the caller decides whether to fall back to
* `mean * count` or skip them.
*
* Bounded to the cumulative-type set so a misuse (passing a spot
* metric) is a type error rather than a silent SUM on the wrong
* column. The query is a plain `findMany` against the
* `(userId, type, granularity, bucketStart)` composite primary key —
* single round-trip, fully indexed.
*/
export async function readCumulativeDaySums(
userId: string,
type: CumulativeType,
since: Date,
): Promise<CumulativeDaySumRow[]> {
const rows = await prisma.measurementRollup.findMany({
where: {
userId,
type: type as MeasurementType,
granularity: "DAY",
bucketStart: { gte: since },
},
orderBy: { bucketStart: "asc" },
select: {
bucketStart: true,
sumValue: true,
count: true,
mean: true,
},
});
return rows.map((r) => ({
bucketStart: r.bucketStart,
sumValue: r.sumValue,
count: r.count,
mean: r.mean,
}));
}
/**
* Batch read DAY buckets for several cumulative types in one round
* trip. Eliminates the per-type chunked-findMany loop in
* `/api/analytics` A2 for the five highest-row-count metrics on
* Marc's tenant.
*
* Returns a Map keyed on `MeasurementType` so the caller can route
* each type's rows back into the per-metric branch without scanning
* the flat list.
*/
export async function readCumulativeDaySumsBatch(
userId: string,
types: readonly CumulativeType[],
since: Date,
): Promise<Map<CumulativeType, CumulativeDaySumRow[]>> {
if (types.length === 0) return new Map();
const rows = await prisma.measurementRollup.findMany({
where: {
userId,
type: { in: types as unknown as MeasurementType[] },
granularity: "DAY",
bucketStart: { gte: since },
},
orderBy: [{ type: "asc" }, { bucketStart: "asc" }],
select: {
type: true,
bucketStart: true,
sumValue: true,
count: true,
mean: true,
},
});
const out = new Map<CumulativeType, CumulativeDaySumRow[]>();
for (const t of types) out.set(t, []);
for (const row of rows) {
const bucket = out.get(row.type as CumulativeType);
if (!bucket) continue;
bucket.push({
bucketStart: row.bucketStart,
sumValue: row.sumValue,
count: row.count,
mean: row.mean,
});
}
return out;
}
/**
* Resolve a per-bucket cumulative total. Reads `sumValue` when the
* v1.4.39 writer has populated it; falls back to `mean * count` for
* legacy NULL rows so the chart never paints a hole during the boot-
* backfill convergence window.
*/
export function resolveBucketSum(row: CumulativeDaySumRow): number {
if (row.sumValue !== null) return row.sumValue;
return row.mean * row.count;
}
src/lib/measurements/rollups.ts
+35 -10
View File
@@ -107,6 +107,7 @@ interface RollupRow {
mean: number | null;
min_value: number | null;
max_value: number | null;
sum_value: number | null;
sd: number | null;
slope: number | null;
r2: number | null;
@@ -387,6 +388,7 @@ async function runRollupAggregate(input: {
AVG(m."value")::double precision AS mean,
MIN(m."value")::double precision AS min_value,
MAX(m."value")::double precision AS max_value,
SUM(m."value")::double precision AS sum_value,
STDDEV_POP(m."value")::double precision AS sd,
REGR_SLOPE(
m."value",
@@ -419,6 +421,7 @@ async function runRollupAggregate(input: {
AVG(m."value")::double precision AS mean,
MIN(m."value")::double precision AS min_value,
MAX(m."value")::double precision AS max_value,
SUM(m."value")::double precision AS sum_value,
STDDEV_POP(m."value")::double precision AS sd,
REGR_SLOPE(
m."value",
@@ -483,6 +486,12 @@ async function persistRollupRows(
mean: row.mean ?? 0,
minValue: row.min_value ?? 0,
maxValue: row.max_value ?? 0,
// v1.4.39 W-SUM — populate for every type. Cumulative read
// paths (steps, flights, distance, daylight, active-energy)
// consume this column directly; spot metrics carry it for
// free because the underlying SUM aggregator runs in the
// same query as AVG / MIN / MAX.
sumValue: row.sum_value ?? null,
sd: row.sd,
slope: row.slope,
r2: row.r2,
@@ -493,6 +502,7 @@ async function persistRollupRows(
mean: row.mean ?? 0,
minValue: row.min_value ?? 0,
maxValue: row.max_value ?? 0,
sumValue: row.sum_value ?? null,
sd: row.sd,
slope: row.slope,
r2: row.r2,
@@ -741,17 +751,32 @@ export async function enqueueBootTimeRollupBackfill(): Promise<{
// the LEFT JOIN) marks the unmatched partitions. A user surfaces
// here when ANY type is missing — including the brand-new-type
// case that v1.4.35.1's zero-rollup-only shape silently stranded.
//
// v1.4.39 W-SUM — the discovery also surfaces users whose existing
// DAY rollup rows carry `sum_value IS NULL`. Those rows pre-date
// the v1.4.39 writer change; re-folding the user converges them
// because `persistRollupRows` always writes the new column on
// upsert. The union runs in one query so the existing per-type
// coverage gap and the legacy-NULL backfill share a single planner
// pass.
const users = await prisma.$queryRaw<Array<{ id: string }>>`
SELECT DISTINCT mt."user_id" AS id
FROM (
SELECT DISTINCT m."user_id", m."type"
FROM measurements m
) mt
LEFT JOIN measurement_rollups r
ON r."user_id" = mt."user_id"
AND r."type" = mt."type"
AND r."granularity" = 'DAY'
WHERE r."bucket_start" IS NULL
SELECT DISTINCT id FROM (
SELECT DISTINCT mt."user_id" AS id
FROM (
SELECT DISTINCT m."user_id", m."type"
FROM measurements m
) mt
LEFT JOIN measurement_rollups r
ON r."user_id" = mt."user_id"
AND r."type" = mt."type"
AND r."granularity" = 'DAY'
WHERE r."bucket_start" IS NULL
UNION
SELECT DISTINCT r2."user_id" AS id
FROM measurement_rollups r2
WHERE r2."granularity" = 'DAY'
AND r2."sum_value" IS NULL
) discovery
`;
if (users.length === 0) {