feat(rollups): MeasurementRollup foundation — populator + write hooks + warm-on-read

WIP toward v1.4.35. Lands the persistent rollup foundation from the
v1.4.34 cache-aggregation research §9. Reads still served by the
v1.4.34.1 live-SQL aggregator — the parity-safe read-swap
(count/min/max/mean + dailyByType from buckets, slope/r2/sd stay on
live aggregation) ships in a follow-up commit so v1.4.35 is the
single release that flips both foundation + read-swap.

Schema:
  - new MeasurementRollup model with composite PK
    (userId, type, granularity, bucketStart) and a descending
    bucketStart index. RollupGranularity enum (DAY/WEEK/MONTH/YEAR).
  - Prisma migration 0067_v1434_measurement_rollups is additive:
    CREATE TYPE / CREATE TABLE IF NOT EXISTS / CREATE INDEX IF NOT
    EXISTS / ADD CONSTRAINT FK. No ALTER on existing tables.
  - db-compat schema-bootstrap path mirrored.

Populator (src/lib/measurements/rollups.ts):
  - recomputeBucketsForMeasurement: sync DAY; WEEK/MONTH/YEAR via
    pg-boss (singleton-keyed).
  - recomputeUserRollups: bounded full-range, used by Apple Health
    import + backfill.
  - ensureUserRollupsFresh: cheap stale-detection + bounded DAY-90d
    recompute. Error-swallowing.
  - SQL via $queryRawUnsafe + strict enum whitelist for date_trunc
    unit (Postgres 42803 forbids binding the unit literal).

Write hooks on every measurement-mutating endpoint; each wrapped
in try/catch so a populator hiccup never fails the write.

Apple Health import worker recomputes the full range once at
completion (per-row hooks would multiply cost).

Backfill (scripts/backfill-rollups.ts): idempotent, single-user
serial, single-digit minutes for a 100k-row account.

Tests: +14 unit, +3 integration. setup.ts.truncateAllTables
extends to the new table.

Read-swap follow-up:
  - comprehensive-aggregator + analytics/route + summaries-slice
    each call ensureUserRollupsFresh as a read side-effect (warms
    the table). The response stays on the live-SQL aggregator so
    byte-shape parity with v1.4.34.5 is preserved.
  - Next commit lands the partial read-swap: count/min/max/mean
    and dailyByType from buckets (math-identical compose), while
    slope/r2/sd stay on indexed live aggregation. Together they
    ship as v1.4.35.

Pre-shipping state — do NOT tag v1.4.35 from this commit alone.
This commit is contained in:
Marc-André Bombeck committed 2026-05-17 09:32:30 +02:00
1 parent 2982ed1f0b
commit 9872081d0e
19 files changed
+1821 -4

No files matched your search

+10
View File
@@ -29,6 +29,7 @@ import type {
} from "@/generated/prisma/client";
import { measurementTypeEnum } from "@/lib/validations/measurement";
import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority";
import { ensureUserRollupsFresh } from "@/lib/measurements/rollups";
export const dynamic = "force-dynamic";
@@ -117,6 +118,15 @@ type AuthedUser = Awaited<ReturnType<typeof requireAuth>>["user"];
* `buildCoachSnapshotImpl` shape.
*/
async function buildAnalyticsResponse(user: AuthedUser) {
// v1.5.0 — warm the persistent rollup table as a side effect of
// the analytics fan-out. No-op when rollups are already ahead of
// the newest measurement; on first cold-mount after a process
// restart it folds the trailing 90-day window so downstream
// consumers (Coach drawer, weekly report, admin analytics) see a
// warm rollup table on their next read. The route's response
// shape is untouched.
await ensureUserRollupsFresh(user.id);
// v1.4.25 W7b — every day-bucket call inside this route now honours
// the user's display timezone. The legacy `berlinDayKey()` import
// remains for sleep-stage and correlation paths that share their
+39
View File
@@ -10,6 +10,7 @@ import {
} from "@/lib/api-response";
import { updateMeasurementSchema } from "@/lib/validations/measurement";
import { invalidateUserMeasurements } from "@/lib/cache/invalidate";
import { recomputeBucketsForMeasurement } from "@/lib/measurements/rollups";
import { Prisma } from "@/generated/prisma/client";
import { NextRequest } from "next/server";
@@ -112,6 +113,30 @@ export const PUT = apiHandler(
// caches so subsequent reads reflect the edited row.
invalidateUserMeasurements(user.id);
// v1.5.0 — refresh the rollup row for the affected day. When the
// measuredAt moved across day boundaries (or the row was re-typed)
// both the old and the new bucket need a recompute. Best-effort
// — a populator hiccup never fails the user's edit.
try {
await recomputeBucketsForMeasurement(
user.id,
measurement.type,
measurement.measuredAt,
);
if (
existing.measuredAt.getTime() !== measurement.measuredAt.getTime() ||
existing.type !== measurement.type
) {
await recomputeBucketsForMeasurement(
user.id,
existing.type,
existing.measuredAt,
);
}
} catch (err) {
console.warn("[measurements] rollup recompute failed", err);
}
return apiSuccess(measurement);
},
);
@@ -150,6 +175,20 @@ export const DELETE = apiHandler(
// caches so subsequent reads reflect the deletion.
invalidateUserMeasurements(user.id);
// v1.5.0 — refresh the rollup row for the affected day (the
// recompute drops the row when the day's measurement count goes
// to zero). Best-effort — a populator hiccup never fails the
// user's delete.
try {
await recomputeBucketsForMeasurement(
user.id,
existing.type,
existing.measuredAt,
);
} catch (err) {
console.warn("[measurements] rollup recompute failed", err);
}
return apiSuccess({ deleted: true });
},
);
+30 -1
View File
@@ -38,7 +38,11 @@ import { deviceTypeEnum } from "@/lib/validations/source-priority";
import { checkRateLimit } from "@/lib/rate-limit";
import { enqueuePrDetection } from "@/lib/jobs/pr-detection";
import { invalidateUserMeasurements } from "@/lib/cache/invalidate";
import { Prisma } from "@/generated/prisma/client";
import {
recomputeBucketsForMeasurement,
collapseToTypeDayKeys,
} from "@/lib/measurements/rollups";
import { Prisma, type MeasurementType } from "@/generated/prisma/client";
// v1.4.25 W16c — historical-backfill threshold for PR push
// suppression. A batch larger than this fires the detection job with
@@ -373,6 +377,31 @@ async function postBatch(request: NextRequest): Promise<Response> {
// gate the invalidation on `insertedCount > 0`.
if (insertedCount > 0) {
invalidateUserMeasurements(user.id);
// v1.5.0 — refresh the persistent rollup table for every distinct
// (type, day) the batch touched. We use the `prepared` rows
// because `createMany` does not return the inserted rows; the
// (type, measuredAt) tuples on `prepared` are exactly the row
// shape we need. Collapsed by day so a 500-row Apple Health
// batch fires N recomputes (typically <30, one per type-day
// pair) instead of one per row. Best-effort — a populator hiccup
// never fails the user's ingest.
try {
const insertedKeys = collapseToTypeDayKeys(
prepared.map((p) => ({
type: p.row.type as MeasurementType,
measuredAt:
p.row.measuredAt instanceof Date
? p.row.measuredAt
: new Date(p.row.measuredAt as string),
})),
);
for (const k of insertedKeys) {
await recomputeBucketsForMeasurement(user.id, k.type, k.measuredAt);
}
} catch (err) {
console.warn("[measurements] rollup recompute failed", err);
}
}
return apiSuccess({
@@ -32,6 +32,10 @@ import {
safeJson,
} from "@/lib/api-response";
import { invalidateUserMeasurements } from "@/lib/cache/invalidate";
import {
recomputeBucketsForMeasurement,
collapseToTypeDayKeys,
} from "@/lib/measurements/rollups";
const MAX_BATCH_ENTRIES = 500;
@@ -83,9 +87,18 @@ async function deleteByExternalIds(request: NextRequest): Promise<Response> {
// Prisma `deleteMany` with the userId predicate is the cross-user
// 404 guard: rows belonging to other users are simply not matched.
// No need for a pre-flight findMany — the unique index on
// `(userId, type, source, externalId)` keeps the matching set small
// even when the user has overlapping HealthKit history.
// We pre-fetch the (type, measuredAt) tuples so the rollup recompute
// step below knows which buckets to refresh after the deletion.
// The unique index on `(userId, type, source, externalId)` keeps
// the matched set small even with overlapping HealthKit history.
const affectedRows = await prisma.measurement.findMany({
where: {
userId: user.id,
externalId: { in: externalIds },
},
select: { type: true, measuredAt: true },
});
const result = await prisma.measurement.deleteMany({
where: {
userId: user.id,
@@ -109,6 +122,19 @@ async function deleteByExternalIds(request: NextRequest): Promise<Response> {
// reconciliation. A 0-delete batch is a no-op and skips the eviction.
if (result.count > 0) {
invalidateUserMeasurements(user.id);
// v1.5.0 — refresh the rollup row for every distinct (type, day)
// tuple the deletion touched. Collapsed by day so the same
// morning's deletes fold into one recompute per type. Best-
// effort — a populator hiccup never fails the user's reconciliation.
try {
const keys = collapseToTypeDayKeys(affectedRows);
for (const k of keys) {
await recomputeBucketsForMeasurement(user.id, k.type, k.measuredAt);
}
} catch (err) {
console.warn("[measurements] rollup recompute failed", err);
}
}
return apiSuccess({ deletedCount: result.count });
+37
View File
@@ -21,6 +21,10 @@ import {
import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping";
import { withIdempotency } from "@/lib/idempotency";
import { invalidateUserMeasurements } from "@/lib/cache/invalidate";
import {
recomputeBucketsForMeasurement,
collapseToTypeDayKeys,
} from "@/lib/measurements/rollups";
import { NextRequest } from "next/server";
import type {
MeasurementType,
@@ -219,6 +223,23 @@ async function postMeasurement(request: NextRequest) {
// measurement set so the next read paints the new rows.
invalidateUserMeasurements(user.id);
// v1.5.0 — refresh the persistent rollup table for every distinct
// (type, day) the batch touched so the next analytics / coach read
// hits the cache rather than falling through to live aggregation.
// Collapsed by day so a multi-entry batch on the same morning fires
// one DAY recompute per type instead of one per row. Best-effort
// — a populator hiccup never fails the user's write.
try {
const keys = collapseToTypeDayKeys(
results.map((r) => ({ type: r.type, measuredAt: r.measuredAt })),
);
for (const k of keys) {
await recomputeBucketsForMeasurement(user.id, k.type, k.measuredAt);
}
} catch (err) {
console.warn("[measurements] rollup recompute failed", err);
}
return apiSuccess(results, 201);
}
@@ -279,5 +300,21 @@ async function postMeasurement(request: NextRequest) {
// measurement set so the next read paints the new row.
invalidateUserMeasurements(user.id);
// v1.5.0 — refresh the persistent rollup row for the affected
// (type, day) tuple. Runs inline so the next read of the
// comprehensive / analytics surface is correct without waiting on
// pg-boss; WEEK / MONTH / YEAR recomputes are enqueued under the
// hood. Best-effort — a populator hiccup never fails the user's
// write.
try {
await recomputeBucketsForMeasurement(
user.id,
measurement.type,
measurement.measuredAt,
);
} catch (err) {
console.warn("[measurements] rollup recompute failed", err);
}
return apiSuccess(measurement, 201);
}