/** * v1.4.36 — SQL-driven aggregator for `/api/insights/comprehensive`, * rollup-first read with live-SQL fallback on cold mounts. * * Background * ---------- * The route previously pulled every measurement of every type from the * trailing 90 days into JS, then ran `summarize()` per type plus a * pile of in-JS correlation pairings. Post Apple-Health import the * unbounded `findMany` on line 49 of the old route routinely returned * 100 000+ rows and dominated request time (29 s in the v1.4.34 prod * HAR capture, with sibling endpoints cascading into 503s on pool * starvation). * * The shape mirrors the `summaries-slice` aggregator * (`src/lib/analytics/summaries-slice.ts`) the v1.4.33 C1 dashboard * tile strip already proved out: a single `$queryRaw` per surface, * grouped by `type`, with Postgres-side `AVG` / `MIN` / `MAX` / * `REGR_SLOPE` / `REGR_R2` against the index path * `measurements (user_id, type, measured_at)`. Difference vs. the * slim slice: the comprehensive endpoint also needs `anomalyCount`, * `avg30LastMonth`, `avg30LastYear`, and the 90-day window cap. * * Read-swap (v1.4.35 → v1.4.36) * ----------------------------- * v1.4.35 landed a partial read-swap that ran both the live aggregate * AND the rollup read in parallel, then preferred the rollup-derived * values when the count matched. That left the regression intact: the * heavy `$queryRaw` over `measurements` still ran on every request, * still scanning the 311k-row table on a power-user account. * * v1.4.36 promotes the rollup table to the canonical source on the * happy path: * * - On entry we call `ensureUserRollupsFresh` (warm-on-read populator * for the trailing 90-day DAY window) and then probe the rollup * table with a single COUNT. * * - When the DAY rollup table has rows for this user the **heavy** * `$queryRaw` over `measurements` is skipped. `count / min / max / * mean` per type compose from the persisted DAY buckets via * `aggregateBuckets`; `dailyByType` reads the same buckets keyed * on `(type, bucketStart)`. * * - A **narrow** `$queryRaw` still fires for the non-composable * fields: `stddev`, `anomalyCount`, `avg7 / avg30 / avg30LastMonth`, * and the `slope7 / slope30` tuples. These don't compose * linearly across DAY buckets so live SQL stays canonical, but the * query no longer carries the redundant `COUNT(*)` / `MIN` / `MAX` * / `AVG` columns that duplicated the bucket-derived values. * * - `latest / latestMeasuredAt` keep their dedicated `DISTINCT ON * (type)` pass; the bucket stores the day's mean rather than the * last reading. * * - Fallback path — when the COUNT probe returns zero rows * (post-restart cold mount or a brand-new account whose boot-time * backfill is mid-flight) we run the legacy heavy aggregate so the * response shape is correct on the very first request. * * Scope notes * ----------- * - The 90-day window is preserved (the legacy `findMany` clamped on * `measuredAt >= ninetyDaysAgo`). `min` / `max` / `mean` therefore * describe the 90-day window, NOT the user's all-time range — * identical to the old behaviour. * * - For correlations: * - **weight × BP**: directive accepts a daily-key swap from the * old `pairByTimestamp` (24h default tolerance). Postgres * joins on `date_trunc('day', measured_at)`. * - **mood × {BP, weight, pulse}**: same daily-key shape the * legacy code already used (the `buildMoodMetricPairs` helper * bucketed by `toISOString().slice(0,10)`). * * - BP target adherence (`bpPctInTarget`) keeps the 5-minute * tolerance contract — that pairing is sys/dia within the *same* * reading session. The aggregator pulls sys+dia raw rows (90d, * bounded — typically <2k rows even for power users) so * `pairByTimestamp` semantics survive byte-identical. * * - `bpMedicationCorrelation` is computed in the route itself; the * aggregator surfaces daily BP_SYS means via the same daily-join * query so the medication block can pair compliance × systolic * without a second findMany. * * - Mood data still loads via `prisma.moodEntry.findMany` — that * table is small (a row per user-recorded entry, bounded) and the * `moodSummary` field calls `summarize()` against raw scores. The * legacy route's `moodSummary` was built off raw rows so we keep * the raw fetch. * * Output shape * ------------ * The aggregator returns the bundle of pre-computed values the route * stitches into the final response. Helper functions inside the * aggregator collapse `$queryRaw` rows into the same `DataSummary` * shape `summarize()` would return. */ import { prisma } from "@/lib/db"; import { runDetached } from "@/lib/logging/detached"; import { trackBackgroundTask } from "@/lib/logging/background-tasks"; import type { DataSummary } from "@/lib/analytics/trends"; import { liveTrend, liveTrendSql, type LiveTrendColumns, } from "@/lib/analytics/trend-evidence-sql"; import { boundInstantSql, windowMeanColumnsSql, } from "@/lib/analytics/request-clock-sql"; import { annotate } from "@/lib/logging/context"; import type { MeasurementSource, MeasurementType, } from "@/generated/prisma/client"; import { ensureUserRollupsFresh } from "@/lib/rollups/measurement-rollups"; import { aggregateBuckets, collapseRollupRowsBySource, composeWindowedTrend, hasPendingAccumulatorBackfill, loadUserSourcePriority, type AccumulatorBucketRow, } from "@/lib/rollups/measurement-read"; import { windowWeighting } from "@/lib/measurements/day-statistic"; import { readLatestCanonical } from "@/lib/measurements/latest-canonical"; import { SESSION_DAY_FRAME, dayWeightedRows, windowMeanSql, } from "@/lib/measurements/day-mean"; import { startOfUtcDay } from "@/lib/tz/start-of-utc-day"; import { buildSourceRankCase, canonicalMeasurementsCte, canonicalMeasurementsFrom, } from "@/lib/analytics/source-rank-sql"; import { isFullyCovered, probeRollupCoverage, } from "@/lib/rollups/measurement-coverage"; import { dateOnlyKey } from "@/lib/tz/date-only"; /** The request's `now`, bound at `$2` of every live aggregate here. */ const NOW_SQL = boundInstantSql(2); const NINETY_DAYS = { from: NOW_SQL, interval: "90 days" } as const; /** * Heavy aggregate row — count/min/max/mean alongside the non-composable * windowed columns. Used on the cold-mount fallback path where the * rollup table is empty (the populator hasn't caught up yet) so the * response shape is correct on the very first request. */ interface HeavyAggregateRow extends LiveTrendColumns { type: string; count: bigint; min_value: number | null; max_value: number | null; mean_value: number | null; stddev_value: number | null; anomaly_count: bigint; avg7: number | null; avg30: number | null; avg30_last_month: number | null; } /** * Narrow aggregate row — only the columns the DAY rollup buckets cannot * compose linearly. `count` rides along so the anomaly z-score loop * knows the divisor for the window-wide mean / stddev; the per-type * count exposed on `DataSummary` still comes from the bucket * composition. */ interface NarrowAggregateRow { type: string; count: bigint; stddev_value: number | null; anomaly_count: bigint; avg7: number | null; avg30: number | null; avg30_last_month: number | null; // v1.20.0 — the slope7/30/90 + r2_7/30/90 columns have moved off this // live query onto the rollup accumulators (composeWindowedRegression); // they no longer scan the raw measurements partition on the warm path. } /** * Result bundle. Everything the route handler needs to assemble the * comprehensive response without holding the 100k-row raw measurement * array in memory. */ export interface ComprehensiveAggregate { /** Per-type DataSummary, 90-day window. Only types with rows are * populated — matches the legacy route's behaviour (only types * with rows landed on `summaries[t]`). */ summaries: Record; /** Sys/dia raw rows for the BP target adherence path. Bounded: * typically <2k rows per user over 90 days even with Apple Health. */ bpRawRows: { sys: Array<{ measuredAt: Date; value: number }>; dia: Array<{ measuredAt: Date; value: number }>; }; /** Raw weight rows over the same 90-day window, same canonical-source * dedup as `bpRawRows` (they ride one query). Weight is sparse * (~1-2 rows/day). v1.32.17 derives the mood × weight correlation from * these local-day-keyed means instead of the UTC `dailyByType.WEIGHT`, * so the mood join partner shares the mood series' local-day key space. */ weightRawRows: Array<{ measuredAt: Date; value: number }>; /** Daily means per (type, day) for the correlation pairings. The * route filters down to the types it needs (WEIGHT, BLOOD_PRESSURE_SYS, * PULSE) when joining against mood / weight / bp. */ dailyByType: Record>; /** Earliest measurement timestamp across the 90-day window — feeds * the route's `dataSpanDays` field. NULL when the user has no rows. */ firstMeasurementAt: Date | null; /** Total row count across all types in the 90-day window — feeds the * wide-event meta + the response's `totalMeasurements` field. */ totalMeasurements: number; } function round2(value: number | null): number | null { if (value === null || !Number.isFinite(value)) return null; return Math.round(value * 100) / 100; } /** * Build the per-type `DataSummary` bundle for the comprehensive route. * * Two read shapes: * * - **Rollup-fresh path** (the happy path on every warm request): * `count / min / max / mean` + `dailyByType` compose from the * persisted DAY buckets; a narrow `$queryRaw` carries only the * non-composable windowed columns. The heavy COUNT/MIN/MAX/AVG * aggregate is **not** executed. * * - **Live fallback path** (cold mount before the populator catches * up, or a brand-new account whose backfill is mid-flight): the * legacy heavy aggregate runs so the response shape stays correct * on the very first request. Subsequent requests fall back to the * rollup-fresh path once `ensureUserRollupsFresh` has written * buckets. */ export async function buildComprehensiveAggregate( userId: string, ): Promise { // One clock for every window, JS and SQL alike (see `request-clock-sql.ts`). const now = new Date(); const ninetyDaysAgo = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000); // v1.4.38 — per-sub-query wall-clock timing. Emitted under // `meta.insights.sub_*_ms` so the next perf-verify can attribute // a regression to a specific aggregate (narrow, latests, day-buckets, // bp-raw, first-at, coverage-probe) without re-instrumenting. const timings: Record = {}; // v1.4.37.1 — fire-and-forget (see `src/app/api/analytics/route.ts`): // awaiting it can stall the event loop for 30–60 s on accounts whose iOS // steps keep the 90-day window stale, and the coverage probe falls back to // live SQL meanwhile. Registered so the suite's settle hook and the // statement counter can await it. trackBackgroundTask(runDetached(() => ensureUserRollupsFresh(userId))); // v1.4.36 — per-type coverage probe. The legacy global COUNT // returned true as soon as any one type had a DAY bucket, which made // the bucket-derived path swallow a brand-new type's all-time history // (the per-write hook upserts a single bucket for the new type, the // probe flips to true, the comprehensive response then iterates only // bucket-bearing types and the trailing-90d narrow aggregate becomes // the new "all-time" total for that type). We now decide per type // and only take the rollup path when EVERY type the user has logged // is covered. Partial coverage falls back to the live aggregate so // a fresh-type measurement doesn't break its summary card. const probeStart = Date.now(); const coverage = await probeRollupCoverage(userId); timings["insights.sub_coverage_ms"] = Date.now() - probeStart; const result = isFullyCovered(coverage) ? await buildFromRollups(userId, now, ninetyDaysAgo, timings) : await buildFromLiveAggregate(userId, now, ninetyDaysAgo, timings); annotate({ meta: timings }); return result; } /** * v1.4.38 — tiny timing helper. Wraps a promise-returning sub-query * builder so each Promise.all slot lands a wall-clock observation under * `meta.insights.sub_