From 497da7ca8b78862c18743e3b6ebddd84d1b5e4d2 Mon Sep 17 00:00:00 2001 From: BernhardBuckel <33808384+BernhardBuckel@users.noreply.github.com> Date: Sun, 4 Oct 2026 10:40:58 +0200 Subject: [PATCH 01/10] fix(pulse): make a day of pulse the mean of its hours' means A watch samples pulse about every 5 to 10 minutes at rest and about every 5 seconds in a workout. The daily value was the plain mean over a day's readings, so a workout hour of several hundred readings outweighed the other 23 hours: a day with one hard hour and 23 resting hours (60 bpm) read about 128 bpm, not 64. A day of pulse is now the mean of its per-hour means, in the rollup DAY bucket, the live per-day read the chart serves, and the series route's day grain. Windows over several days weigh each day once instead of by its reading count, so the bias does not return one level up. Reading counts, min/max, and the regression sums still run over every reading. The chart tooltip and the data table name the statistic, in all seven locales. The statistic is limited to pulse; HRV, SpO2 and glucose keep the plain mean (see src/lib/measurements/day-statistic.ts). Stored DAY rows are rewritten by the next rollup recompute (admin recompute route), not on read. Fixes #1107 --- messages/de.json | 4 +- messages/en.json | 4 +- messages/es.json | 4 +- messages/fr.json | 4 +- messages/it.json | 4 +- messages/ko.json | 4 +- messages/pl.json | 4 +- src/app/api/measurements/series/route.ts | 40 ++- .../__tests__/chart-data-table.test.tsx | 61 ++++ src/components/charts/chart-data-table.tsx | 16 + src/components/charts/health-chart.tsx | 10 +- src/lib/analytics/range-delta.ts | 10 +- src/lib/analytics/summaries-slice.ts | 2 + src/lib/insights/comprehensive-aggregator.ts | 3 +- src/lib/mcp/rich-reads.ts | 11 +- .../__tests__/day-statistic.test.ts | 36 +++ src/lib/measurements/daily-series-read.ts | 43 ++- src/lib/measurements/day-statistic.ts | 32 ++ .../__tests__/measurement-read-wmy.test.ts | 41 +++ .../__tests__/measurement-read.test.ts | 44 +++ src/lib/rollups/measurement-read-wmy.ts | 22 +- src/lib/rollups/measurement-read.ts | 58 +++- src/lib/rollups/measurement-rollups.ts | 96 +++++- tests/integration/measurement-rollups.test.ts | 31 +- .../integration/pulse-hourly-mean-day.test.ts | 282 ++++++++++++++++++ 25 files changed, 816 insertions(+), 50 deletions(-) create mode 100644 src/lib/measurements/__tests__/day-statistic.test.ts create mode 100644 src/lib/measurements/day-statistic.ts create mode 100644 tests/integration/pulse-hourly-mean-day.test.ts diff --git a/messages/de.json b/messages/de.json index 0c7173452..206fc0f7a 100644 --- a/messages/de.json +++ b/messages/de.json @@ -2437,13 +2437,15 @@ "loadErrorTitle": "Diagramm konnte nicht geladen werden", "loadErrorReload": "Neu laden", "sparseDataCaption": "Mehr Messtage füllen den Trend.", + "tooltipHourlyMean": "{metric} (Durchschnitt der Stunden)", "dataTable": { "toggle": "Datenpunkte ({count})", "columnDate": "Datum", "noValue": "Kein Wert", "captionDay": "Eine Zeile pro Tag für {metric} — genau die Punkte, die das Diagramm zeichnet.", "captionWeek": "Eine Zeile pro Woche für {metric}, der Mittelwert der Tage dieser Woche — genau die Punkte, die das Diagramm zeichnet.", - "captionMonth": "Eine Zeile pro Monat für {metric}, der Mittelwert der Tage dieses Monats — genau die Punkte, die das Diagramm zeichnet." + "captionMonth": "Eine Zeile pro Monat für {metric}, der Mittelwert der Tage dieses Monats — genau die Punkte, die das Diagramm zeichnet.", + "hourlyMeanNote": "{metric} ist der Tagesdurchschnitt, bei dem jede Stunde einmal zählt, damit ein Training nicht den Rest des Tages überwiegt." }, "personalBaseline": "Dein Mittel", "deltaVsBaseline": "{delta} vs. dein Mittel", diff --git a/messages/en.json b/messages/en.json index 3cd9aa9a4..cbe6ac31c 100644 --- a/messages/en.json +++ b/messages/en.json @@ -2437,13 +2437,15 @@ "loadErrorTitle": "Chart could not be loaded", "loadErrorReload": "Reload", "sparseDataCaption": "More days will fill out the trend.", + "tooltipHourlyMean": "{metric} (average of the hours)", "dataTable": { "toggle": "Data points ({count})", "columnDate": "Date", "noValue": "No value", "captionDay": "One row per day of {metric}, the same points the chart draws.", "captionWeek": "One row per week of {metric}, the average of that week's days — the same points the chart draws.", - "captionMonth": "One row per month of {metric}, the average of that month's days — the same points the chart draws." + "captionMonth": "One row per month of {metric}, the average of that month's days — the same points the chart draws.", + "hourlyMeanNote": "{metric} is each day's average, with every hour counting once, so a workout does not outweigh the rest of the day." }, "personalBaseline": "Your normal", "deltaVsBaseline": "{delta} vs. your normal", diff --git a/messages/es.json b/messages/es.json index d40131208..754446f0d 100644 --- a/messages/es.json +++ b/messages/es.json @@ -2437,13 +2437,15 @@ "loadErrorTitle": "No se pudo cargar el gráfico", "loadErrorReload": "Recargar", "sparseDataCaption": "Más días completarán la tendencia.", + "tooltipHourlyMean": "{metric} (promedio de las horas)", "dataTable": { "toggle": "Puntos de datos ({count})", "columnDate": "Fecha", "noValue": "Sin valor", "captionDay": "Una fila por día de {metric}: los mismos puntos que dibuja el gráfico.", "captionWeek": "Una fila por semana de {metric}, la media de los días de esa semana: los mismos puntos que dibuja el gráfico.", - "captionMonth": "Una fila por mes de {metric}, la media de los días de ese mes: los mismos puntos que dibuja el gráfico." + "captionMonth": "Una fila por mes de {metric}, la media de los días de ese mes: los mismos puntos que dibuja el gráfico.", + "hourlyMeanNote": "{metric} es el promedio de cada día, con cada hora contando una sola vez, para que un entrenamiento no pese más que el resto del día." }, "personalBaseline": "Tu media", "deltaVsBaseline": "{delta} vs. tu media", diff --git a/messages/fr.json b/messages/fr.json index a5d0afa08..60d00f7e7 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -2437,13 +2437,15 @@ "loadErrorTitle": "Impossible de charger le graphique", "loadErrorReload": "Recharger", "sparseDataCaption": "Davantage de jours étofferont la tendance.", + "tooltipHourlyMean": "{metric} (moyenne des heures)", "dataTable": { "toggle": "Points de données ({count})", "columnDate": "Date", "noValue": "Aucune valeur", "captionDay": "Une ligne par jour pour {metric} : les mêmes points que ceux tracés par le graphique.", "captionWeek": "Une ligne par semaine pour {metric}, la moyenne des jours de cette semaine : les mêmes points que ceux tracés par le graphique.", - "captionMonth": "Une ligne par mois pour {metric}, la moyenne des jours de ce mois : les mêmes points que ceux tracés par le graphique." + "captionMonth": "Une ligne par mois pour {metric}, la moyenne des jours de ce mois : les mêmes points que ceux tracés par le graphique.", + "hourlyMeanNote": "{metric} est la moyenne de chaque jour, chaque heure comptant une seule fois, afin qu'un entraînement ne pèse pas plus que le reste de la journée." }, "personalBaseline": "Ta moyenne", "deltaVsBaseline": "{delta} vs. ta moyenne", diff --git a/messages/it.json b/messages/it.json index e0e489863..5a57cc3fb 100644 --- a/messages/it.json +++ b/messages/it.json @@ -2437,13 +2437,15 @@ "loadErrorTitle": "Impossibile caricare il grafico", "loadErrorReload": "Ricarica", "sparseDataCaption": "Più giorni completeranno il trend.", + "tooltipHourlyMean": "{metric} (media delle ore)", "dataTable": { "toggle": "Punti dati ({count})", "columnDate": "Data", "noValue": "Nessun valore", "captionDay": "Una riga per giorno di {metric}: gli stessi punti disegnati dal grafico.", "captionWeek": "Una riga per settimana di {metric}, la media dei giorni di quella settimana: gli stessi punti disegnati dal grafico.", - "captionMonth": "Una riga per mese di {metric}, la media dei giorni di quel mese: gli stessi punti disegnati dal grafico." + "captionMonth": "Una riga per mese di {metric}, la media dei giorni di quel mese: gli stessi punti disegnati dal grafico.", + "hourlyMeanNote": "{metric} è la media di ogni giorno, con ogni ora conteggiata una sola volta, così un allenamento non pesa più del resto della giornata." }, "personalBaseline": "La tua media", "deltaVsBaseline": "{delta} vs. la tua media", diff --git a/messages/ko.json b/messages/ko.json index e51af7017..e6943d644 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -2429,13 +2429,15 @@ "loadErrorTitle": "차트를 불러오지 못했어요", "loadErrorReload": "다시 불러오기", "sparseDataCaption": "날이 쌓이면 추이가 채워져요.", + "tooltipHourlyMean": "{metric} (시간별 평균)", "dataTable": { "toggle": "데이터 포인트 ({count})", "columnDate": "날짜", "noValue": "값 없음", "captionDay": "{metric} 하루당 한 행, 차트가 그리는 값과 같아요.", "captionWeek": "{metric} 한 주당 한 행, 그 주 일별 평균 — 차트가 그리는 값과 같아요.", - "captionMonth": "{metric} 한 달당 한 행, 그달 일별 평균 — 차트가 그리는 값과 같아요." + "captionMonth": "{metric} 한 달당 한 행, 그달 일별 평균 — 차트가 그리는 값과 같아요.", + "hourlyMeanNote": "{metric}은(는) 하루의 평균이며 각 시간을 한 번씩만 반영하므로, 운동이 하루의 나머지보다 더 크게 반영되지 않습니다." }, "personalBaseline": "내 평소값", "deltaVsBaseline": "평소값 대비 {delta}", diff --git a/messages/pl.json b/messages/pl.json index caa7f9245..ecf3924de 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -2437,13 +2437,15 @@ "loadErrorTitle": "Nie udało się załadować wykresu", "loadErrorReload": "Załaduj ponownie", "sparseDataCaption": "Więcej dni dopełni trend.", + "tooltipHourlyMean": "{metric} (średnia z godzin)", "dataTable": { "toggle": "Punkty danych ({count})", "columnDate": "Data", "noValue": "Brak wartości", "captionDay": "Jeden wiersz na dzień dla {metric} — dokładnie te punkty, które rysuje wykres.", "captionWeek": "Jeden wiersz na tydzień dla {metric}, średnia z dni tego tygodnia — dokładnie te punkty, które rysuje wykres.", - "captionMonth": "Jeden wiersz na miesiąc dla {metric}, średnia z dni tego miesiąca — dokładnie te punkty, które rysuje wykres." + "captionMonth": "Jeden wiersz na miesiąc dla {metric}, średnia z dni tego miesiąca — dokładnie te punkty, które rysuje wykres.", + "hourlyMeanNote": "{metric} to średnia z każdego dnia, w której każda godzina liczy się raz, dzięki czemu trening nie przeważa nad resztą dnia." }, "personalBaseline": "Twoja średnia", "deltaVsBaseline": "{delta} vs. twoja średnia", diff --git a/src/app/api/measurements/series/route.ts b/src/app/api/measurements/series/route.ts index 35b22894e..794660abe 100644 --- a/src/app/api/measurements/series/route.ts +++ b/src/app/api/measurements/series/route.ts @@ -26,6 +26,7 @@ import type { MeasurementType, SleepStage } from "@/generated/prisma/client"; import { reconstructSleepNights } from "@/lib/analytics/sleep-night"; import { VALUE_RANGES } from "@/lib/validations/measurement"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { seriesRowsFrom } from "@/lib/measurements/series-canonical"; import { resolveUserTimezone, userDayKey } from "@/lib/tz/resolver"; import { convertGlucose, resolveGlucoseUnit } from "@/lib/glucose"; @@ -410,6 +411,11 @@ export const GET = apiHandler(async (request: NextRequest) => { ]); const type = KIND_TO_TYPE[kind]; const seriesRows = seriesRowsFrom(priorityJson, type, days); + // A day of a type with an activity-dependent sampling rate (pulse) is the + // mean of its local hours' means, each hour once, so a workout hour with + // hundreds of times the readings does not outweigh the rest of the day (see + // `day-statistic.ts`). The hour grain is already one mean per hour. + const hourWeighted = grain === "day" && usesHourlyMeanDay(type); const bucketRows = await prisma.$queryRawUnsafe< Array<{ bucket_start: Date; @@ -418,7 +424,39 @@ export const GET = apiHandler(async (request: NextRequest) => { max_value: number; }> >( - ` + hourWeighted + ? ` + WITH localized AS ( + SELECT + m."value", + m."value_min", + m."value_max", + date_trunc( + 'hour', + (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE $3 + ) AS local_hour + FROM ${seriesRows} + WHERE m."measured_at" >= $4 + ), + hourly AS ( + SELECT + date_trunc($2, local_hour) AS local_day, + AVG("value") AS hour_mean, + MIN(COALESCE("value_min", "value")) AS min_value, + MAX(COALESCE("value_max", "value")) AS max_value + FROM localized + GROUP BY local_hour + ) + SELECT + local_day AT TIME ZONE $3 AS bucket_start, + AVG(hour_mean)::double precision AS mean, + MIN(min_value)::double precision AS min_value, + MAX(max_value)::double precision AS max_value + FROM hourly + GROUP BY local_day + ORDER BY bucket_start ASC + ` + : ` WITH localized AS ( SELECT m."value", diff --git a/src/components/charts/__tests__/chart-data-table.test.tsx b/src/components/charts/__tests__/chart-data-table.test.tsx index 32afaa285..507010898 100644 --- a/src/components/charts/__tests__/chart-data-table.test.tsx +++ b/src/components/charts/__tests__/chart-data-table.test.tsx @@ -135,4 +135,65 @@ describe("", () => { /data-slot="table-container"[^>]*class="[^"]*overflow-x-auto/, ); }); + + describe("naming the statistic", () => { + const pulsePoints = points(3).map((p) => ({ ...p, PULSE: 64 })); + const pulseProps = (over: Partial = {}) => + props({ + points: pulsePoints, + columns: [{ key: "PULSE", label: "Pulse" }], + unit: "bpm", + metricLabel: "Pulse", + ...over, + }); + + it("says a day of pulse is the average of its hours, so a workout does not outweigh the day", () => { + const html = render(); + expect(html).toContain('data-slot="chart-data-table-statistic"'); + expect(html).toContain( + "Pulse is each day's average, with every hour counting once", + ); + }); + + it("says it in German too", () => { + const html = renderToStaticMarkup( + + + , + ); + expect(html).toContain("jede Stunde einmal zählt"); + }); + + it("says it for the week and month grains as well", () => { + for (const bucket of ["week", "month"] as const) { + expect( + render(), + ).toContain('data-slot="chart-data-table-statistic"'); + } + }); + + it("adds nothing for a metric that is the plain mean of its readings", () => { + const html = render(); + expect(html).not.toContain('data-slot="chart-data-table-statistic"'); + }); + + it("names only the pulse column when it shares a table with others", () => { + const html = render( + , + ); + expect( + html.match(/data-slot="chart-data-table-statistic"/g), + ).toHaveLength(1); + expect(html).toContain("Pulse is each day"); + expect(html).not.toContain("Systolic is each day"); + }); + }); }); diff --git a/src/components/charts/chart-data-table.tsx b/src/components/charts/chart-data-table.tsx index 5c28eee29..dbb29c192 100644 --- a/src/components/charts/chart-data-table.tsx +++ b/src/components/charts/chart-data-table.tsx @@ -15,6 +15,7 @@ import { import { useTranslations } from "@/lib/i18n/context"; import { cn } from "@/lib/utils"; import type { ChartBucketType } from "@/lib/charts/bucket-time-series"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; /** * The data points behind the chart, as a table. @@ -132,6 +133,21 @@ export function ChartDataTable({ {t(BUCKET_CAPTION_KEY[bucket], { metric: metricLabel })} + {/* Name the statistic where it is not the plain mean of the + readings: a day of pulse is the mean of its hours' means. */} + {columns + .filter((column) => usesHourlyMeanDay(column.key)) + .map((column) => ( + + {t("charts.dataTable.hourlyMeanNote", { + metric: column.label, + })} + + ))} diff --git a/src/components/charts/health-chart.tsx b/src/components/charts/health-chart.tsx index ee00071dd..086ff7470 100644 --- a/src/components/charts/health-chart.tsx +++ b/src/components/charts/health-chart.tsx @@ -34,6 +34,7 @@ import { type ComponentType, } from "react"; import { Skeleton } from "@/components/ui/skeleton"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { RichChartTooltip, type RichTooltipRow } from "./chart-tooltip"; import { ChartEmptyState } from "./chart-empty-state"; import { ChartErrorState } from "./chart-error-state"; @@ -2240,7 +2241,14 @@ export function HealthChart({ } } rows.push({ - name: item.name ?? dataKey, + // Name the statistic where it is not the plain mean + // of the readings: a day of pulse is the mean of its + // hours' means. + name: usesHourlyMeanDay(dataKey) + ? t("charts.tooltipHourlyMean", { + metric: item.name ?? dataKey, + }) + : (item.name ?? dataKey), value: `${formatTooltipValue(item.value)}${ unit ? ` ${unit}` : "" }`, diff --git a/src/lib/analytics/range-delta.ts b/src/lib/analytics/range-delta.ts index 667cf37c9..4bee215a5 100644 --- a/src/lib/analytics/range-delta.ts +++ b/src/lib/analytics/range-delta.ts @@ -40,6 +40,10 @@ import type { MeasurementType } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; +import { + windowWeighting, + type WindowWeighting, +} from "@/lib/measurements/day-statistic"; import { reconstructSleepNights, type SleepStageRow, @@ -88,6 +92,7 @@ export function sliceWindowDelta( rows: RollupBucketRow[], windowDays: number, now: number, + weighting: WindowWeighting = "count", ): { current: WindowAggregate; previous: WindowAggregate } { const currentStart = now - windowDays * DAY_MS; const previousStart = now - 2 * windowDays * DAY_MS; @@ -102,8 +107,8 @@ export function sliceWindowDelta( } } return { - current: aggregateWmyBuckets(currentRows), - previous: aggregateWmyBuckets(previousRows), + current: aggregateWmyBuckets(currentRows, weighting), + previous: aggregateWmyBuckets(previousRows, weighting), }; } @@ -334,6 +339,7 @@ export async function computeRangeDelta( resolved.rows, windowDays, now, + windowWeighting(type), ); const { delta, deltaPct } = composeDelta(current, previous); return { diff --git a/src/lib/analytics/summaries-slice.ts b/src/lib/analytics/summaries-slice.ts index 11c74e262..76dc34e6d 100644 --- a/src/lib/analytics/summaries-slice.ts +++ b/src/lib/analytics/summaries-slice.ts @@ -60,6 +60,7 @@ import type { MeasurementType } from "@/generated/prisma/client"; import { Prisma } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; +import { windowWeighting } from "@/lib/measurements/day-statistic"; import type { DataSummary } from "@/lib/analytics/trends"; import { measurementTypeEnum } from "@/lib/validations/measurement"; import { annotate } from "@/lib/logging/context"; @@ -1109,6 +1110,7 @@ async function computeAvg30LastYearForType( minValue: r.minValue, maxValue: r.maxValue, })), + windowWeighting(type), ); return composed.count > 0 ? composed.mean : null; } diff --git a/src/lib/insights/comprehensive-aggregator.ts b/src/lib/insights/comprehensive-aggregator.ts index 548f89268..e828eb189 100644 --- a/src/lib/insights/comprehensive-aggregator.ts +++ b/src/lib/insights/comprehensive-aggregator.ts @@ -113,6 +113,7 @@ import { loadUserSourcePriority, type AccumulatorBucketRow, } from "@/lib/rollups/measurement-read"; +import { windowWeighting } from "@/lib/measurements/day-statistic"; import { startOfUtcDay } from "@/lib/tz/start-of-utc-day"; import { buildSourceRankCase, @@ -544,7 +545,7 @@ async function buildFromRollups( // windowed columns (avg + anomaly) alongside; slope / r² / sd compose // from the per-bucket regression accumulators (v1.20.0 F6). for (const [type, buckets] of bucketsByType.entries()) { - const composed = aggregateBuckets(buckets); + const composed = aggregateBuckets(buckets, windowWeighting(type)); if (composed.count === 0) continue; totalMeasurements += composed.count; const latest = latestByType.get(type); diff --git a/src/lib/mcp/rich-reads.ts b/src/lib/mcp/rich-reads.ts index 3893df9d8..05cb1823e 100644 --- a/src/lib/mcp/rich-reads.ts +++ b/src/lib/mcp/rich-reads.ts @@ -31,6 +31,7 @@ import type { MeasurementType } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; +import { windowWeighting } from "@/lib/measurements/day-statistic"; import { annotate } from "@/lib/logging/context"; import { isModuleEnabled } from "@/lib/modules/gate"; import { moduleForMeasurementType } from "@/lib/modules/measurement-scope"; @@ -854,7 +855,10 @@ async function snapshotMetricWindow( windowDays, ); if (!read || read.rows.length === 0) return null; - const agg = aggregateWmyBuckets(read.rows); + const agg = aggregateWmyBuckets( + read.rows, + windowWeighting(metric.measurementType), + ); if (agg.count === 0) return null; return { label: metric.label, @@ -895,7 +899,10 @@ async function snapshotMetricRange( return t >= fromMs && t <= toMs; }); if (rows.length === 0) return null; - const agg = aggregateWmyBuckets(rows); + const agg = aggregateWmyBuckets( + rows, + windowWeighting(metric.measurementType), + ); if (agg.count === 0) return null; return { label: metric.label, diff --git a/src/lib/measurements/__tests__/day-statistic.test.ts b/src/lib/measurements/__tests__/day-statistic.test.ts new file mode 100644 index 000000000..9941ea553 --- /dev/null +++ b/src/lib/measurements/__tests__/day-statistic.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; + +import { + HOURLY_MEAN_DAY_TYPES, + usesHourlyMeanDay, + windowWeighting, +} from "../day-statistic"; + +describe("which types a day is the mean of the hours' means for", () => { + it("is pulse, and only pulse", () => { + expect([...HOURLY_MEAN_DAY_TYPES]).toEqual(["PULSE"]); + }); + + it.each(["PULSE"])( + "%s uses the hourly mean and weighs each day once", + (type) => { + expect(usesHourlyMeanDay(type)).toBe(true); + expect(windowWeighting(type)).toBe("day"); + }, + ); + + // A CGM samples at a fixed rate, so glucose has no activity bias. HRV and + // SpO2 may follow once the effect is shown on real data. Cumulative types sum. + it.each([ + "BLOOD_GLUCOSE", + "HEART_RATE_VARIABILITY", + "OXYGEN_SATURATION", + "RESTING_HEART_RATE", + "WEIGHT", + "ACTIVITY_STEPS", + "HRV_RMSSD", + ])("%s keeps the plain mean and weighs each day by its readings", (type) => { + expect(usesHourlyMeanDay(type)).toBe(false); + expect(windowWeighting(type)).toBe("count"); + }); +}); diff --git a/src/lib/measurements/daily-series-read.ts b/src/lib/measurements/daily-series-read.ts index 945345dd4..ef2b0659e 100644 --- a/src/lib/measurements/daily-series-read.ts +++ b/src/lib/measurements/daily-series-read.ts @@ -16,6 +16,7 @@ import { type AggregateGrain, } from "@/lib/measurements/range-aggregation"; import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; +import { HOURLY_MEAN_DAY_TYPES } from "@/lib/measurements/day-statistic"; import { readTieredRollupSeries } from "@/lib/rollups/measurement-read-wmy"; import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; import { annotate } from "@/lib/logging/context"; @@ -192,6 +193,11 @@ export async function readLiveBuckets(opts: { .map((t) => `'${t}'`) .join(","); const isCumulative = Prisma.raw(`c."type"::text IN (${cumulativeList})`); + // Types whose day is the mean of its per-local-hour means (see + // `day-statistic.ts`). Closed code constants, spliced as enum literals. + const hourlyMeanList = Prisma.raw( + [...HOURLY_MEAN_DAY_TYPES].map((t) => `'${t}'::measurement_type`).join(","), + ); const rankRaw = Prisma.raw( buildSourceRankCase(priorityJson, 'p."type"', 'p."source"'), ); @@ -236,17 +242,50 @@ export async function readLiveBuckets(opts: { ${typeFilter} GROUP BY m."type", m."source", 3 ), + -- A day of a type with an activity-dependent sampling rate is the mean of + -- its local hours' means, each hour once, not the mean of its readings: a + -- workout hour holds hundreds of times as many readings as a resting one. + -- Empty for every other type. + hour_mean AS ( + SELECT h."type", h."source", h.day, + AVG(h.hour_mean)::double precision AS day_mean + FROM ( + SELECT + m."type", + m."source", + date_trunc('day', (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}) AS day, + date_trunc('hour', (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}) AS hr, + AVG(m."value") AS hour_mean + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."measured_at" >= ${from} + AND m."measured_at" <= ${to} + AND m."deleted_at" IS NULL + AND m."type" IN (${hourlyMeanList}) + ${typeFilter} + GROUP BY m."type", m."source", 3, 4 + ) h + GROUP BY h."type", h."source", h.day + ), canon AS ( SELECT DISTINCT ON (p."type", p.day) - p."type", p.day, p.total, p.cnt, p.min_value, p.max_value + p."type", p.day, p.total, p.cnt, p.min_value, p.max_value, hm.day_mean FROM per_source p + LEFT JOIN hour_mean hm + ON hm."type" = p."type" AND hm."source" = p."source" AND hm.day = p.day ORDER BY p."type", p.day, (${rankRaw}), p."source"::text ), folded AS ( SELECT c."type", date_trunc(${unit}, c.day) AS d, - (CASE WHEN ${isCumulative} THEN SUM(c.total) ELSE SUM(c.total) / SUM(c.cnt) END)::double precision AS avg, + (CASE + WHEN ${isCumulative} THEN SUM(c.total) + -- Hourly-mean types: every day once, so a window of days does not + -- hand a workout day its thousands of readings as weight. + WHEN COUNT(c.day_mean) = COUNT(*) THEN AVG(c.day_mean) + ELSE SUM(c.total) / SUM(c.cnt) + END)::double precision AS avg, SUM(c.cnt)::int AS cnt, MIN(c.min_value) AS min_value, MAX(c.max_value) AS max_value diff --git a/src/lib/measurements/day-statistic.ts b/src/lib/measurements/day-statistic.ts new file mode 100644 index 000000000..2b42cf4f0 --- /dev/null +++ b/src/lib/measurements/day-statistic.ts @@ -0,0 +1,32 @@ +/** + * Which statistic a type's DAY value is, and how days combine into a window. + * + * A watch samples heart rate about every 5 to 10 minutes at rest and about + * every 5 seconds in a workout. A plain mean over a day's rows therefore counts + * the workout hour hundreds of times and the other twenty-three hours a few + * dozen times each, so a workout day reads far above the rest of the day. + * + * For such a type the DAY value is the mean of its per-hour means, each hour + * weighing one. That is deterministic, needs no extra column, and matches what + * an hourly bucket already is. A window of days then weighs each day once too + * (the mean of the daily means), because weighting days by their sample count + * would bring the same bias back one level up. + * + * Pulse only. A CGM samples glucose at a fixed rate, so glucose has no + * activity bias; HRV and SpO2 can follow when the effect is shown on real data. + */ +import type { MeasurementType } from "@/generated/prisma/client"; + +export const HOURLY_MEAN_DAY_TYPES: ReadonlySet = + new Set(["PULSE"]); + +export function usesHourlyMeanDay(type: string): boolean { + return HOURLY_MEAN_DAY_TYPES.has(type as MeasurementType); +} + +/** How the days of a window combine: by sample count, or one weight per day. */ +export type WindowWeighting = "count" | "day"; + +export function windowWeighting(type: string): WindowWeighting { + return usesHourlyMeanDay(type) ? "day" : "count"; +} diff --git a/src/lib/rollups/__tests__/measurement-read-wmy.test.ts b/src/lib/rollups/__tests__/measurement-read-wmy.test.ts index 8ec37ec0a..5dd53703b 100644 --- a/src/lib/rollups/__tests__/measurement-read-wmy.test.ts +++ b/src/lib/rollups/__tests__/measurement-read-wmy.test.ts @@ -724,3 +724,44 @@ describe("server tier vs client caption ladder", () => { } }); }); + +describe("aggregateWmyBuckets, one weight per day", () => { + // A week of seven days and a week of two, both read at their own mean. With + // one weight per day the longer week counts for its seven days, not for its + // readings. + const week = ( + days: number, + count: number, + mean: number, + ): Parameters[0][number] => ({ + bucketStart: new Date("2026-05-04T00:00:00.000Z"), + count, + days, + mean, + sd: null, + slope: null, + r2: null, + sumValue: null, + minValue: mean - 5, + maxValue: mean + 5, + }); + + it("weighs a bucket by its readings by default", () => { + const result = aggregateWmyBuckets([week(7, 3000, 60), week(2, 1000, 90)]); + expect(result.mean).toBeCloseTo((3000 * 60 + 1000 * 90) / 4000, 9); + }); + + it("weighs a bucket by the days it holds when asked", () => { + const result = aggregateWmyBuckets( + [week(7, 3000, 60), week(2, 1000, 90)], + "day", + ); + expect(result.mean).toBeCloseTo((7 * 60 + 2 * 90) / 9, 9); + expect(result.count).toBe(4000); + }); + + it("treats a row with no day count as one day", () => { + const row = { ...week(1, 10, 70), days: undefined }; + expect(aggregateWmyBuckets([row], "day").mean).toBe(70); + }); +}); diff --git a/src/lib/rollups/__tests__/measurement-read.test.ts b/src/lib/rollups/__tests__/measurement-read.test.ts index 6cbbac500..a0f192f24 100644 --- a/src/lib/rollups/__tests__/measurement-read.test.ts +++ b/src/lib/rollups/__tests__/measurement-read.test.ts @@ -65,3 +65,47 @@ describe("aggregateBuckets", () => { expect(result.mean).toBe(82); }); }); + +describe("aggregateBuckets, one weight per day", () => { + // A workout day (many readings) and a rest day (few). By readings the window + // mean is dominated by the workout day; with one weight per day it is the + // mean of the two days' means. + const workout: DailyMeanRow = { + day: new Date("2026-05-08T00:00:00.000Z"), + count: 900, + mean: 64, + minValue: 60, + maxValue: 150, + }; + const rest: DailyMeanRow = { + day: new Date("2026-05-09T00:00:00.000Z"), + count: 100, + mean: 56, + minValue: 50, + maxValue: 70, + }; + + it("weighs a day by its readings by default", () => { + const result = aggregateBuckets([workout, rest]); + expect(result.mean).toBeCloseTo((900 * 64 + 100 * 56) / 1000, 9); + }); + + it("weighs every day once when asked, and still counts every reading", () => { + const result = aggregateBuckets([workout, rest], "day"); + expect(result.mean).toBeCloseTo((64 + 56) / 2, 9); + expect(result.count).toBe(1000); + expect(result.min).toBe(50); + expect(result.max).toBe(150); + }); + + it("gives a day with no readings no weight under either rule", () => { + const empty: DailyMeanRow = { ...rest, count: 0, mean: 999 }; + expect(aggregateBuckets([workout, empty], "day").mean).toBe(64); + expect(aggregateBuckets([empty], "day")).toEqual({ + count: 0, + min: null, + max: null, + mean: null, + }); + }); +}); diff --git a/src/lib/rollups/measurement-read-wmy.ts b/src/lib/rollups/measurement-read-wmy.ts index a67d91c3e..865b08e67 100644 --- a/src/lib/rollups/measurement-read-wmy.ts +++ b/src/lib/rollups/measurement-read-wmy.ts @@ -37,6 +37,7 @@ import type { } from "@/generated/prisma/client"; import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; +import type { WindowWeighting } from "@/lib/measurements/day-statistic"; import { annotate } from "@/lib/logging/context"; import { prisma } from "@/lib/db"; import { @@ -54,6 +55,8 @@ import { export interface RollupBucketRow { bucketStart: Date; count: number; + /** Days of the bucket that hold a reading (1 for a DAY row). */ + days?: number; mean: number; sd: number | null; slope: number | null; @@ -162,7 +165,10 @@ export async function readBestGranularityRollups( * stats do not compose across coarser buckets and the consumers that * need them stay on live SQL. */ -export function aggregateWmyBuckets(rows: RollupBucketRow[]): { +export function aggregateWmyBuckets( + rows: RollupBucketRow[], + weighting: WindowWeighting = "count", +): { count: number; min: number | null; max: number | null; @@ -174,13 +180,20 @@ export function aggregateWmyBuckets(rows: RollupBucketRow[]): { } let totalCount = 0; let sumWeighted = 0; + let totalWeight = 0; let sumCumulative = 0; let sawSum = false; let min = Infinity; let max = -Infinity; for (const row of rows) { totalCount += row.count; - sumWeighted += row.count * row.mean; + // Each reading weighs one, or, for a type whose DAY value is the mean of + // its hourly means (pulse), each DAY weighs one: a bucket weighs the days + // it holds. See `aggregateBuckets` in measurement-read.ts. + const weight = + weighting === "day" ? (row.count > 0 ? (row.days ?? 1) : 0) : row.count; + sumWeighted += weight * row.mean; + totalWeight += weight; if (row.sumValue !== null && Number.isFinite(row.sumValue)) { sumCumulative += row.sumValue; sawSum = true; @@ -188,14 +201,14 @@ export function aggregateWmyBuckets(rows: RollupBucketRow[]): { if (row.minValue < min) min = row.minValue; if (row.maxValue > max) max = row.maxValue; } - if (totalCount === 0) { + if (totalCount === 0 || totalWeight === 0) { return { count: 0, min: null, max: null, mean: null, sum: null }; } return { count: totalCount, min: Number.isFinite(min) ? min : null, max: Number.isFinite(max) ? max : null, - mean: sumWeighted / totalCount, + mean: sumWeighted / totalWeight, sum: sawSum ? sumCumulative : null, }; } @@ -234,6 +247,7 @@ async function readGranularity( return rows.map((r) => ({ bucketStart: r.bucketStart, count: r.count, + days: r.days, mean: r.mean, sd: r.sd, slope: r.slope, diff --git a/src/lib/rollups/measurement-read.ts b/src/lib/rollups/measurement-read.ts index c183dc110..21fa56fdd 100644 --- a/src/lib/rollups/measurement-read.ts +++ b/src/lib/rollups/measurement-read.ts @@ -28,6 +28,10 @@ import type { import { Prisma } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; +import { + windowWeighting, + type WindowWeighting, +} from "@/lib/measurements/day-statistic"; import { metricKeyForType } from "@/lib/measurements/cumulative-day-sum"; import { startOfUtcDay } from "@/lib/tz/start-of-utc-day"; import { @@ -386,8 +390,19 @@ export function hasPendingAccumulatorBackfill( * Combine DAY buckets into the linearly-composable window stats — * `count`, `min`, `max`, `mean`. SD / slope / R² are intentionally * NOT computed here because they don't compose across DAY rollups. + * + * `weighting` decides how the days make up the window mean. By default each + * day weighs its sample count, which is the plain mean over every reading. For + * a type whose DAY value is the mean of its hourly means (pulse, see + * `day-statistic.ts`) pass `"day"`: every day weighs one. Weighting days by how + * many samples they hold would hand a workout day, with its thousands of + * samples, the same outsized share the hourly mean exists to take away. + * `count` stays the number of readings either way. */ -export function aggregateBuckets(rows: DailyMeanRow[]): { +export function aggregateBuckets( + rows: DailyMeanRow[], + weighting: WindowWeighting = "count", +): { count: number; min: number | null; max: number | null; @@ -398,22 +413,26 @@ export function aggregateBuckets(rows: DailyMeanRow[]): { } let totalCount = 0; let sumWeighted = 0; + let totalWeight = 0; let min = Infinity; let max = -Infinity; for (const r of rows) { totalCount += r.count; - sumWeighted += r.count * r.mean; + // A day with no readings carries no weight under either rule. + const weight = weighting === "day" ? (r.count > 0 ? 1 : 0) : r.count; + sumWeighted += weight * r.mean; + totalWeight += weight; if (r.minValue < min) min = r.minValue; if (r.maxValue > max) max = r.maxValue; } - if (totalCount === 0) { + if (totalCount === 0 || totalWeight === 0) { return { count: 0, min: null, max: null, mean: null }; } return { count: totalCount, min: Number.isFinite(min) ? min : null, max: Number.isFinite(max) ? max : null, - mean: sumWeighted / totalCount, + mean: sumWeighted / totalWeight, }; } @@ -518,10 +537,18 @@ const COARSE_TRUNC_UNIT: Record, string> = { * `sd`, `slope` and `r2` come from the summed regression accumulators * (`composeRegression`) and are `null` when a day lacks them. */ -function composeFoldedBucket(row: FoldedBucketRow): CanonicalRollupBucket { +function composeFoldedBucket( + row: FoldedBucketRow, + weighting: WindowWeighting = "count", +): CanonicalRollupBucket { const count = Number(row.count); const sumY = Number(row.sum_y); - const mean = sumY / count; + // The mean of every reading: what the regression below composes from. + const sampleMean = sumY / count; + const dayMean = Number(row.sum_day_mean) / Number(row.days); + // The bucket's own `mean`: every reading, or, for a type whose DAY value is + // the mean of its hourly means (pulse), every day once. + const mean = weighting === "day" ? dayMean : sampleMean; const accumulatorsComplete = row.sum_x !== null && row.sum_xy !== null && @@ -533,14 +560,14 @@ function composeFoldedBucket(row: FoldedBucketRow): CanonicalRollupBucket { const sumYy = accumulatorsComplete ? Number(row.sum_yy) : null; const regression = accumulatorsComplete ? composeRegression([ - { count, mean, sumValue: sumY, sumX, sumXy, sumXx, sumYy }, + { count, mean: sampleMean, sumValue: sumY, sumX, sumXy, sumXx, sumYy }, ]) : { slope: null, r2: null, sdPop: null }; return { bucketStart: new Date(row.bucket_start), count, days: Number(row.days), - dayMean: Number(row.sum_day_mean) / Number(row.days), + dayMean, mean, minValue: Number(row.min_value), maxValue: Number(row.max_value), @@ -691,10 +718,13 @@ export async function readCanonicalRollupBuckets(opts: { `; const fromMs = from.getTime(); const toMs = to?.getTime() ?? null; - return rows.map(composeFoldedBucket).filter((b) => { - const start = b.bucketStart.getTime(); - if (start < fromMs) return false; - if (toMs === null) return true; - return opts.toInclusive ? start <= toMs : start < toMs; - }); + const weighting = windowWeighting(type); + return rows + .map((r) => composeFoldedBucket(r, weighting)) + .filter((b) => { + const start = b.bucketStart.getTime(); + if (start < fromMs) return false; + if (toMs === null) return true; + return opts.toInclusive ? start <= toMs : start < toMs; + }); } diff --git a/src/lib/rollups/measurement-rollups.ts b/src/lib/rollups/measurement-rollups.ts index 867863642..f5982bd4b 100644 --- a/src/lib/rollups/measurement-rollups.ts +++ b/src/lib/rollups/measurement-rollups.ts @@ -43,6 +43,7 @@ import { utcBucketEnd, utcBucketStart, } from "@/lib/rollups/measurement-read"; +import { HOURLY_MEAN_DAY_TYPES } from "@/lib/measurements/day-statistic"; import { startOfUtcDay } from "@/lib/tz/start-of-utc-day"; import type { MeasurementType, @@ -526,11 +527,15 @@ async function runRollupAggregate( AND m."deleted_at" IS NULL GROUP BY m."type", m."source", ${dateTrunc} `; - return db.$queryRawUnsafe( - sql, - input.userId, - input.from, - input.to, + return applyHourlyMeanDay( + db, + input, + await db.$queryRawUnsafe( + sql, + input.userId, + input.from, + input.to, + ), ); } @@ -569,12 +574,89 @@ async function runRollupAggregate( AND m."deleted_at" IS NULL GROUP BY m."type", m."source", ${dateTrunc} `; - return db.$queryRawUnsafe( - sql, + return applyHourlyMeanDay( + db, + input, + await db.$queryRawUnsafe( + sql, + input.userId, + input.from, + input.to, + ), + ); +} + +/** + * For a type whose DAY value is the mean of its per-hour means (see + * `day-statistic.ts`), replace the DAY bucket's `mean`. Only `mean` changes: + * `count`, `min`, `max` and the sums the regression composes from stay over + * every reading, so the band and the windowed slope are untouched. + * + * Hours are UTC hours, matching the UTC day this tier buckets by. + */ +async function applyHourlyMeanDay( + db: Pick, + input: { + userId: string; + granularity: RollupGranularity; + from: Date; + to: Date; + }, + rows: RollupRow[], +): Promise { + if (input.granularity !== "DAY") return rows; + const types = [ + ...new Set( + rows + .map((r) => r.type) + .filter((t) => HOURLY_MEAN_DAY_TYPES.has(t as MeasurementType)), + ), + ]; + if (types.length === 0) return rows; + for (const t of types) { + if (!/^[A-Z0-9_]+$/.test(t)) { + throw new Error(`invalid measurement type: ${t}`); + } + } + const typeList = types.map((t) => `'${t}'::"measurement_type"`).join(","); + const means = await db.$queryRawUnsafe< + Array<{ type: string; source: string; bucket_start: Date; mean: number }> + >( + ` + SELECT h."type"::text AS type, + h."source"::text AS source, + date_trunc('day', h.hr) AS bucket_start, + AVG(h.hour_mean)::double precision AS mean + FROM ( + SELECT m."type", m."source", + date_trunc('hour', m."measured_at") AS hr, + AVG(m."value") AS hour_mean + FROM measurements m + WHERE m."user_id" = $1 + AND m."type" IN (${typeList}) + AND m."measured_at" >= $2 + AND m."measured_at" < $3 + AND m."deleted_at" IS NULL + GROUP BY m."type", m."source", date_trunc('hour', m."measured_at") + ) h + GROUP BY 1, 2, 3 + `, input.userId, input.from, input.to, ); + const byKey = new Map( + means.map((m) => [ + `${m.type}|${m.source}|${new Date(m.bucket_start).getTime()}`, + m.mean, + ]), + ); + return rows.map((r) => { + const mean = byKey.get( + `${r.type}|${r.source}|${new Date(r.bucket_start).getTime()}`, + ); + return mean === undefined ? r : { ...r, mean }; + }); } /** diff --git a/tests/integration/measurement-rollups.test.ts b/tests/integration/measurement-rollups.test.ts index 5e09f9b3b..6e6613f1a 100644 --- a/tests/integration/measurement-rollups.test.ts +++ b/tests/integration/measurement-rollups.test.ts @@ -439,15 +439,28 @@ describe("measurement rollups — integration", () => { Array<{ type: string; day: string; mean_value: number }> >` SELECT - m."type"::text AS type, - TO_CHAR(date_trunc('day', m."measured_at"), 'YYYY-MM-DD') AS day, - (ROUND((AVG(m."value"))::numeric, 2))::double precision AS mean_value - FROM measurements m - WHERE m."user_id" = ${user.id} - AND m."measured_at" >= ${ninetyDaysAgo} - AND m."type" IN ('WEIGHT', 'BLOOD_PRESSURE_SYS', 'PULSE') - GROUP BY m."type", date_trunc('day', m."measured_at") - ORDER BY m."type", day ASC + h."type"::text AS type, + TO_CHAR(h.day, 'YYYY-MM-DD') AS day, + -- A day of pulse is the mean of its hours' means (day-statistic.ts); + -- every other type is the plain mean of its readings. + (ROUND((CASE WHEN h."type" = 'PULSE' + THEN AVG(h.hour_mean) ELSE AVG(h.plain_mean) END)::numeric, 2))::double precision AS mean_value + FROM ( + SELECT + m."type", + date_trunc('day', m."measured_at") AS day, + AVG(m."value") AS hour_mean, + AVG(m."value") AS plain_mean + FROM measurements m + WHERE m."user_id" = ${user.id} + AND m."measured_at" >= ${ninetyDaysAgo} + AND m."type" IN ('WEIGHT', 'BLOOD_PRESSURE_SYS', 'PULSE') + GROUP BY m."type", date_trunc('day', m."measured_at"), + CASE WHEN m."type" = 'PULSE' + THEN date_trunc('hour', m."measured_at") END + ) h + GROUP BY h."type", h.day + ORDER BY h."type", day ASC `; const liveByType = new Map>(); diff --git a/tests/integration/pulse-hourly-mean-day.test.ts b/tests/integration/pulse-hourly-mean-day.test.ts new file mode 100644 index 000000000..5d87bbc8a --- /dev/null +++ b/tests/integration/pulse-hourly-mean-day.test.ts @@ -0,0 +1,282 @@ +/** + * A day of pulse is the mean of its hours' means, and a window of days weighs + * each day once. + * + * A watch samples about every 5 to 10 minutes at rest and about every 5 seconds + * in a workout, so a plain mean over a day's readings counts the workout hour + * hundreds of times and the other twenty-three hours a few dozen times each. + * The worked example here is the one from the issue: a workout hour of 700 + * readings at 150 bpm and 23 resting hours of 10 readings at 60 bpm. + * + * mean of the readings (700 x 150 + 230 x 60) / 930 = 127.7 + * mean of the hours' means (150 + 23 x 60) / 24 = 63.75 + * + * Asserted against a real Postgres, in every place a daily pulse value or a + * window of days is made: the live per-day read the chart serves, the + * rollup's DAY bucket and the weeks folded from it, and the series route's + * day bucket. HRV is seeded with the very same readings as a control: a type + * without an activity-dependent sampling rate keeps the plain mean. + */ +import { NextRequest } from "next/server"; +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { cookieJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { Prisma } from "@/generated/prisma/client"; +import { readLiveBuckets } from "@/lib/measurements/daily-series-read"; +import { readCanonicalRollupBuckets } from "@/lib/rollups/measurement-read"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const USER = "pulse-hourly-mean"; +const DAY = 86_400_000; + +/** Monday of the ISO week two weeks back, 00:00 UTC. */ +const now = new Date(); +const dow = (now.getUTCDay() + 6) % 7; +const MONDAY = Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() - dow - 14, +); +/** Tuesday: the workout day. Wednesday: an ordinary day. Same ISO week. */ +const WORKOUT_DAY = MONDAY + DAY; +const REST_DAY = MONDAY + 2 * DAY; + +const WORKOUT_HOUR = 10; +const WORKOUT_READINGS = 700; +const RESTING_READINGS_PER_HOUR = 10; + +function dayRows( + type: "PULSE" | "HEART_RATE_VARIABILITY", + unit: string, + dayStart: number, + workout: boolean, +): Prisma.MeasurementCreateManyInput[] { + const rows: Prisma.MeasurementCreateManyInput[] = []; + for (let h = 0; h < 24; h += 1) { + const isWorkoutHour = workout && h === WORKOUT_HOUR; + const n = isWorkoutHour ? WORKOUT_READINGS : RESTING_READINGS_PER_HOUR; + for (let i = 0; i < n; i += 1) { + // Inside the hour, at least 1 s apart; the workout hour is dense. + const offsetMs = isWorkoutHour ? i * 5_000 : (5 + i * 4) * 60_000; + rows.push({ + id: `${type}-${dayStart}-${h}-${i}`, + userId: USER, + type, + unit, + source: "APPLE_HEALTH", + value: isWorkoutHour ? 150 : 60, + measuredAt: new Date(dayStart + h * 3_600_000 + offsetMs), + }); + } + } + return rows; +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, timezone: "UTC" }, + }); + invalidateUserTimezone(USER); + for (const [type, unit] of [ + ["PULSE", "bpm"], + ["HEART_RATE_VARIABILITY", "ms"], + ] as const) { + await prisma.measurement.createMany({ + data: [ + ...dayRows(type, unit, WORKOUT_DAY, true), + ...dayRows(type, unit, REST_DAY, false), + ], + }); + } + await recomputeUserRollups(USER, { + types: ["PULSE", "HEART_RATE_VARIABILITY"], + from: new Date(MONDAY), + to: new Date(MONDAY + 7 * DAY), + }); +}, 120_000); + +const SAMPLE_MEAN_WORKOUT_DAY = + (WORKOUT_READINGS * 150 + 23 * RESTING_READINGS_PER_HOUR * 60) / + (WORKOUT_READINGS + 23 * RESTING_READINGS_PER_HOUR); +const HOURLY_MEAN_WORKOUT_DAY = (150 + 23 * 60) / 24; + +describe("the worked example", () => { + it("really is two different numbers", () => { + expect(SAMPLE_MEAN_WORKOUT_DAY).toBeCloseTo(127.74, 2); + expect(HOURLY_MEAN_WORKOUT_DAY).toBeCloseTo(63.75, 5); + }); +}); + +describe("the live per-day read the chart serves", () => { + const read = ( + type: "PULSE" | "HEART_RATE_VARIABILITY", + grain: "daily" | "weekly", + ) => + readLiveBuckets({ + userId: USER, + type, + from: new Date(MONDAY), + to: new Date(MONDAY + 7 * DAY), + cap: 400, + priorityJson: null, + grain, + timeZone: "UTC", + }); + + it("gives a workout day the mean of its hours, not of its readings", async () => { + const rows = await read("PULSE", "daily"); + const workout = rows.find( + (r) => new Date(r.measuredAt).getTime() === WORKOUT_DAY, + ); + expect(workout?.value).toBeCloseTo(HOURLY_MEAN_WORKOUT_DAY, 6); + // The reading count and the band stay over every reading. + expect(workout?.count).toBe( + WORKOUT_READINGS + 23 * RESTING_READINGS_PER_HOUR, + ); + expect(workout?.minValue).toBe(60); + expect(workout?.maxValue).toBe(150); + const rest = rows.find( + (r) => new Date(r.measuredAt).getTime() === REST_DAY, + ); + expect(rest?.value).toBeCloseTo(60, 6); + }); + + it("weighs each day once when it folds days into a week", async () => { + const [week] = await read("PULSE", "weekly"); + // Mean of the two days' means, not of their readings (which would be ~114). + expect(week.value).toBeCloseTo((HOURLY_MEAN_WORKOUT_DAY + 60) / 2, 6); + expect(week.count).toBe( + WORKOUT_READINGS + + 23 * RESTING_READINGS_PER_HOUR + + 24 * RESTING_READINGS_PER_HOUR, + ); + }); + + it("keeps the plain mean for a type with no activity-dependent sampling (HRV)", async () => { + const rows = await read("HEART_RATE_VARIABILITY", "daily"); + const workout = rows.find( + (r) => new Date(r.measuredAt).getTime() === WORKOUT_DAY, + ); + expect(workout?.value).toBeCloseTo(SAMPLE_MEAN_WORKOUT_DAY, 6); + const [week] = await read("HEART_RATE_VARIABILITY", "weekly"); + const total = + WORKOUT_READINGS * 150 + + 23 * RESTING_READINGS_PER_HOUR * 60 + + 24 * RESTING_READINGS_PER_HOUR * 60; + const readings = WORKOUT_READINGS + 47 * RESTING_READINGS_PER_HOUR; + expect(week.value).toBeCloseTo(total / readings, 6); + }); +}); + +describe("the rollup tier", () => { + const rollup = ( + type: "PULSE" | "HEART_RATE_VARIABILITY", + granularity: "DAY" | "WEEK", + ) => + readCanonicalRollupBuckets({ + userId: USER, + type, + granularity, + from: new Date(MONDAY), + to: new Date(MONDAY + 7 * DAY), + toInclusive: true, + }); + + it("stores the mean of the hours' means as the DAY mean, and nothing else changes", async () => { + const days = await rollup("PULSE", "DAY"); + const workout = days.find((d) => d.bucketStart.getTime() === WORKOUT_DAY)!; + expect(workout.mean).toBeCloseTo(HOURLY_MEAN_WORKOUT_DAY, 6); + expect(workout.count).toBe( + WORKOUT_READINGS + 23 * RESTING_READINGS_PER_HOUR, + ); + expect(workout.minValue).toBe(60); + expect(workout.maxValue).toBe(150); + // The sums the regression composes from stay over every reading. + expect(workout.sumValue).toBe( + WORKOUT_READINGS * 150 + 23 * RESTING_READINGS_PER_HOUR * 60, + ); + }); + + it("gives a week the mean of its days' means", async () => { + const [week] = await rollup("PULSE", "WEEK"); + expect(week.days).toBe(2); + expect(week.mean).toBeCloseTo((HOURLY_MEAN_WORKOUT_DAY + 60) / 2, 6); + expect(week.dayMean).toBeCloseTo(week.mean, 9); + }); + + it("keeps the plain mean for HRV, DAY and WEEK alike", async () => { + const days = await rollup("HEART_RATE_VARIABILITY", "DAY"); + const workout = days.find((d) => d.bucketStart.getTime() === WORKOUT_DAY)!; + expect(workout.mean).toBeCloseTo(SAMPLE_MEAN_WORKOUT_DAY, 6); + const [week] = await rollup("HEART_RATE_VARIABILITY", "WEEK"); + const total = WORKOUT_READINGS * 150 + 47 * RESTING_READINGS_PER_HOUR * 60; + expect(week.mean).toBeCloseTo( + total / (WORKOUT_READINGS + 47 * RESTING_READINGS_PER_HOUR), + 6, + ); + }); +}); + +describe("the series route's day bucket", () => { + async function series(kind: string) { + const s = await prisma.session.create({ + data: { userId: USER, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.clear(); + cookieJar.set("healthlog_session", s.id); + const { GET } = await import("@/app/api/measurements/series/route"); + const res = await GET( + new NextRequest( + `http://localhost/api/measurements/series?kind=${kind}&days=365`, + ), + ); + expect(res.status).toBe(200); + return ( + (await res.json()) as { + data: { points: Array<{ id: string; at: string; value: number }> }; + } + ).data.points; + } + + it("is the mean of the hours' means for pulse", async () => { + const points = await series("pulse"); + expect(points.every((p) => p.id.startsWith("day:"))).toBe(true); + const workout = points.find( + (p) => new Date(p.at).getTime() === WORKOUT_DAY, + ); + expect(workout?.value).toBeCloseTo(HOURLY_MEAN_WORKOUT_DAY, 2); + }); + + it("is the plain mean for HRV", async () => { + const points = await series("heartRateVariability"); + const workout = points.find( + (p) => new Date(p.at).getTime() === WORKOUT_DAY, + ); + expect(workout?.value).toBeCloseTo(SAMPLE_MEAN_WORKOUT_DAY, 2); + }); +}); From c755a42d937dc70232aa82d52427726a6ceaaeda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 11:05:35 +0200 Subject: [PATCH 02/10] Add one day-mean helper for the hourly-mean types A day of pulse is the mean of its hours' means and a window the mean of its days, but only the rollup tier and the chart knew it. The helper gives every other reader the same statistic in both shapes it needs: in TypeScript over readings it holds, and in SQL as a per-reading day weight whose sums give the mean of the day values over any window. Every other type keeps its plain mean. --- .../measurements/__tests__/day-mean.test.ts | 145 +++++++ src/lib/measurements/day-mean.ts | 358 ++++++++++++++++++ 2 files changed, 503 insertions(+) create mode 100644 src/lib/measurements/__tests__/day-mean.test.ts create mode 100644 src/lib/measurements/day-mean.ts diff --git a/src/lib/measurements/__tests__/day-mean.test.ts b/src/lib/measurements/__tests__/day-mean.test.ts new file mode 100644 index 000000000..f2da9e6fc --- /dev/null +++ b/src/lib/measurements/__tests__/day-mean.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from "vitest"; + +import { + dayValue, + dayValueRows, + dayWeightedRows, + foldMeanSql, + hourMeanRows, + isHourlyMeanTypeSql, + readingsMean, + windowMeanSql, + zoneDayFrame, + SESSION_DAY_FRAME, +} from "../day-mean"; + +const at = (iso: string) => new Date(iso); + +/** One hard hour of 12 readings at 150 bpm, then three resting hours of one reading at 60. */ +function workoutDay(day: string) { + const rows = Array.from({ length: 12 }, (_, i) => ({ + value: 150, + measuredAt: at(`${day}T10:${String(i * 5).padStart(2, "0")}:00Z`), + })); + for (const h of ["12", "14", "16"]) { + rows.push({ value: 60, measuredAt: at(`${day}T${h}:00:00Z`) }); + } + return rows; +} + +describe("dayValue", () => { + it("is the mean of the hours' means for pulse", () => { + expect(dayValue("PULSE", workoutDay("2026-03-02"), "UTC")).toBeCloseTo( + (150 + 60 * 3) / 4, + 10, + ); + }); + + it("is the plain mean for every other type", () => { + const rows = workoutDay("2026-03-02"); + expect(dayValue("HEART_RATE_VARIABILITY", rows, "UTC")).toBe( + rows.reduce((s, r) => s + r.value, 0) / rows.length, + ); + }); + + it("is null without readings", () => { + expect(dayValue("PULSE", [], "UTC")).toBeNull(); + }); + + it("buckets hours on the zone's own half-hour boundaries", () => { + // Asia/Kolkata is UTC+05:30: 10:20Z and 10:40Z are 15:50 and 16:10 local, + // two different local hours, though one UTC hour. + const rows = [ + { value: 100, measuredAt: at("2026-03-02T10:20:00Z") }, + { value: 100, measuredAt: at("2026-03-02T10:25:00Z") }, + { value: 40, measuredAt: at("2026-03-02T10:40:00Z") }, + ]; + expect(dayValue("PULSE", rows, "Asia/Kolkata")).toBe(70); + expect(dayValue("PULSE", rows, "UTC")).toBe(80); + }); +}); + +describe("readingsMean", () => { + it("weighs each local day once for pulse", () => { + const rows = [ + ...workoutDay("2026-03-02"), + { value: 60, measuredAt: at("2026-03-03T08:00:00Z") }, + ]; + expect(readingsMean("PULSE", rows, "UTC")).toBeCloseTo((82.5 + 60) / 2, 10); + }); + + it("splits days at the zone's midnight", () => { + // 22:30Z is already the next day in Berlin (UTC+1 in March). + const rows = [ + { value: 100, measuredAt: at("2026-03-02T10:00:00Z") }, + { value: 100, measuredAt: at("2026-03-02T11:00:00Z") }, + { value: 40, measuredAt: at("2026-03-02T23:30:00Z") }, + ]; + expect(readingsMean("PULSE", rows, "UTC")).toBe(80); + expect(readingsMean("PULSE", rows, "Europe/Berlin")).toBe(70); + }); + + it("keeps the plain left-to-right mean for every other type", () => { + const rows = [0.1, 0.2, 0.3].map((value, i) => ({ + value, + measuredAt: at(`2026-03-0${i + 1}T08:00:00Z`), + })); + expect(readingsMean("WEIGHT", rows, "UTC")).toBe((0 + 0.1 + 0.2 + 0.3) / 3); + }); + + it("is null without readings", () => { + expect(readingsMean("PULSE", [], "UTC")).toBeNull(); + expect(readingsMean("WEIGHT", [], "UTC")).toBeNull(); + }); +}); + +describe("hourMeanRows / dayValueRows", () => { + it("collapse pulse to one row per hour / per day", () => { + const rows = workoutDay("2026-03-02"); + const hours = hourMeanRows("PULSE", rows, "UTC"); + expect(hours.map((h) => h.value)).toEqual([150, 60, 60, 60]); + expect(hours[0].measuredAt.toISOString()).toBe("2026-03-02T10:00:00.000Z"); + const days = dayValueRows("PULSE", rows, "UTC"); + expect(days).toHaveLength(1); + expect(days[0].value).toBeCloseTo(82.5, 10); + }); + + it("hand every other type back untouched", () => { + const rows = workoutDay("2026-03-02"); + expect(hourMeanRows("BLOOD_GLUCOSE", rows, "UTC")).toBe(rows); + expect(dayValueRows("BLOOD_GLUCOSE", rows, "UTC")).toBe(rows); + }); +}); + +describe("SQL builders", () => { + it("splice only the closed type list", () => { + expect(isHourlyMeanTypeSql('m."type"')).toBe( + `(m."type")::text IN ('PULSE')`, + ); + }); + + it("refuse a non-identifier source and an inline zone", () => { + expect(() => dayWeightedRows("cm; DROP", SESSION_DAY_FRAME)).toThrow(); + expect(() => zoneDayFrame("'Europe/Berlin'")).toThrow(); + expect(zoneDayFrame("$3")).toEqual({ kind: "zone", tzSql: "$3" }); + }); + + it("keep AVG for every other type and weigh pulse", () => { + const sql = windowMeanSql({ + typeColumn: 'm."type"', + value: 'm."value"', + weight: "m.day_weight", + filter: "m.x", + }); + expect(sql).toContain('ELSE AVG(m."value") FILTER (WHERE m.x) END'); + expect(sql).toContain('SUM(m."value" * m.day_weight) FILTER (WHERE m.x)'); + const fold = foldMeanSql({ + typeColumn: 'c."type"', + total: "c.total", + count: "c.cnt", + weightedSum: "c.wsum", + weightSum: "c.wdays", + }); + expect(fold).toContain("ELSE SUM(c.total) / SUM(c.cnt) END"); + }); +}); diff --git a/src/lib/measurements/day-mean.ts b/src/lib/measurements/day-mean.ts new file mode 100644 index 000000000..38b2e2970 --- /dev/null +++ b/src/lib/measurements/day-mean.ts @@ -0,0 +1,358 @@ +/** + * The one place that turns readings into a day value and a window mean for the + * types in `HOURLY_MEAN_DAY_TYPES` (see `day-statistic.ts`). + * + * For such a type a day is the mean of its hours' means, each hour weighing + * one, and a window of days is the mean of its day values, each day weighing + * one. Count, min and max stay over every reading. Every other type keeps the + * plain mean over its readings, computed exactly as before. + * + * "Day" and "hour" are the reader's own frame: a reader that buckets by the + * user's local day passes that zone, a reader that buckets by the session's + * (UTC) day passes `SESSION_DAY_FRAME`. The hours are always the hours of + * that same frame, so a day never splits an hour. + * + * Two shapes are offered, one per kind of reader: + * + * - TypeScript, for readers that hold the readings in memory: + * `readingsMean`, `dayValue` and `dayValueRows`. + * - SQL, for readers that aggregate in Postgres: `dayWeightedRows` attaches + * a `day_weight` to every reading (1 / (readings in its hour × hours in its + * day) for an hourly-mean type, 1 otherwise), so a day's weights add up to + * one and `SUM(value * day_weight) / SUM(day_weight)` is the mean of the + * day values over any window. `windowMeanSql` and `foldMeanSql` build the + * mean expressions, falling back to the unchanged plain mean for every + * other type. + * + * Type lists are closed code constants checked against an enum-shaped regex + * before they are spliced; every value (user id, zone, bounds) stays a bound + * parameter of the caller's query. + */ +import { Prisma } from "@/generated/prisma/client"; +import { tzOffsetMinutes } from "@/lib/tz/format"; + +import { HOURLY_MEAN_DAY_TYPES, usesHourlyMeanDay } from "./day-statistic"; + +const MS_PER_MINUTE = 60_000; +const MS_PER_HOUR = 3_600_000; +const MS_PER_DAY = 86_400_000; +/** Zone offsets change on quarter hours at the finest, so a quarter-hour slot shares one offset. */ +const OFFSET_SLOT_MS = 15 * MS_PER_MINUTE; + +export interface DayReading { + value: number; + measuredAt: Date; +} + +// ─── TypeScript ─────────────────────────────────────────── + +/** Local wall-clock milliseconds of an instant, with a per-call offset cache. */ +function makeLocalMs(tz: string): (at: Date) => number { + const offsets = new Map(); + return (at: Date) => { + const t = at.getTime(); + const slot = Math.floor(t / OFFSET_SLOT_MS); + let offset = offsets.get(slot); + if (offset === undefined) { + offset = tzOffsetMinutes(at, tz); + offsets.set(slot, offset); + } + return t + offset * MS_PER_MINUTE; + }; +} + +function plainMean(rows: readonly DayReading[]): number | null { + if (rows.length === 0) return null; + return rows.reduce((s, r) => s + r.value, 0) / rows.length; +} + +/** Mean of the per-hour means of readings that share one day. */ +function meanOfHourMeans( + rows: readonly DayReading[], + localMs: (at: Date) => number, +): number { + const hours = new Map(); + for (const r of rows) { + const hour = Math.floor(localMs(r.measuredAt) / MS_PER_HOUR); + const acc = hours.get(hour); + if (acc) { + acc.sum += r.value; + acc.n += 1; + } else { + hours.set(hour, { sum: r.value, n: 1 }); + } + } + let sum = 0; + for (const h of hours.values()) sum += h.sum / h.n; + return sum / hours.size; +} + +/** Group readings by local day (days since the epoch in `tz`). */ +function groupByLocalDay( + rows: readonly T[], + localMs: (at: Date) => number, +): Map { + const days = new Map(); + for (const r of rows) { + const day = Math.floor(localMs(r.measuredAt) / MS_PER_DAY); + const list = days.get(day); + if (list) list.push(r); + else days.set(day, [r]); + } + return days; +} + +/** + * The value of ONE day's readings: the mean of its local hours' means for an + * hourly-mean type, the plain mean otherwise. `null` for no readings. + */ +export function dayValue( + type: string, + rows: readonly DayReading[], + tz: string, +): number | null { + if (rows.length === 0) return null; + if (!usesHourlyMeanDay(type)) return plainMean(rows); + return meanOfHourMeans(rows, makeLocalMs(tz)); +} + +/** + * The mean of a window of readings: for an hourly-mean type the mean of its + * local days' values (each day the mean of its hours' means), otherwise the + * plain mean over every reading, summed left to right as the readers always + * did. `null` for no readings. + */ +export function readingsMean( + type: string, + rows: readonly DayReading[], + tz: string, +): number | null { + if (rows.length === 0) return null; + if (!usesHourlyMeanDay(type)) return plainMean(rows); + const localMs = makeLocalMs(tz); + let sum = 0; + let days = 0; + for (const dayRows of groupByLocalDay(rows, localMs).values()) { + sum += meanOfHourMeans(dayRows, localMs); + days += 1; + } + return sum / days; +} + +/** + * Readings of an hourly-mean type replaced by one row per local hour carrying + * the hour's mean, stamped at the hour's first reading. A reader that already + * folds its rows into day buckets by plain mean (and a window by mean of + * those) then yields the hourly-mean day without knowing the statistic. Every + * other type comes back untouched (the same array). + */ +export function hourMeanRows( + type: string, + rows: readonly T[], + tz: string, +): readonly DayReading[] { + if (!usesHourlyMeanDay(type) || rows.length === 0) return rows; + const localMs = makeLocalMs(tz); + const hours = new Map(); + for (const r of rows) { + const hour = Math.floor(localMs(r.measuredAt) / MS_PER_HOUR); + const acc = hours.get(hour); + if (acc) { + acc.sum += r.value; + acc.n += 1; + if (r.measuredAt < acc.at) acc.at = r.measuredAt; + } else { + hours.set(hour, { sum: r.value, n: 1, at: r.measuredAt }); + } + } + return [...hours.values()] + .map((h) => ({ value: h.sum / h.n, measuredAt: h.at })) + .sort((a, b) => a.measuredAt.getTime() - b.measuredAt.getTime()); +} + +/** + * Readings of an hourly-mean type replaced by one row per local day carrying + * the day value, stamped at the day's first reading; a reader that folds rows + * into a window by plain mean then weighs every day once. Every other type + * comes back untouched (the same array). + */ +export function dayValueRows( + type: string, + rows: readonly T[], + tz: string, +): readonly DayReading[] { + if (!usesHourlyMeanDay(type) || rows.length === 0) return rows; + const localMs = makeLocalMs(tz); + const out: DayReading[] = []; + for (const dayRows of groupByLocalDay(rows, localMs).values()) { + let first = dayRows[0].measuredAt; + for (const r of dayRows) if (r.measuredAt < first) first = r.measuredAt; + out.push({ value: meanOfHourMeans(dayRows, localMs), measuredAt: first }); + } + return out.sort((a, b) => a.measuredAt.getTime() - b.measuredAt.getTime()); +} + +// ─── SQL ────────────────────────────────────────────────── + +const ENUM_RE = /^[A-Z0-9_]+$/; + +/** `'PULSE'` — the hourly-mean types as a spliceable text-literal list. */ +export function hourlyMeanTypeLiterals(): string { + return [...HOURLY_MEAN_DAY_TYPES] + .map((t) => { + if (!ENUM_RE.test(t)) { + throw new Error(`unsafe enum literal for SQL splice: ${t}`); + } + return `'${t}'`; + }) + .join(","); +} + +/** SQL predicate: the type column names an hourly-mean type. */ +export function isHourlyMeanTypeSql(typeColumn: string): string { + return `(${typeColumn})::text IN (${hourlyMeanTypeLiterals()})`; +} + +/** + * The frame a reader buckets its days in. `SESSION_DAY_FRAME` is + * `date_trunc('day', measured_at)` on the timestamptz itself, the frame of the + * rollup tier and of every reader that groups by that expression. A zone frame + * names a bound parameter (`$3`) or a Prisma placeholder carrying an IANA zone. + */ +export type DayFrame = { kind: "session" } | { kind: "zone"; tzSql: string }; + +export const SESSION_DAY_FRAME: DayFrame = { kind: "session" }; + +/** A zone frame whose zone is the caller's bound parameter `tzSql` (e.g. `$3`). */ +export function zoneDayFrame(tzSql: string): DayFrame { + if (!/^\$\d+$/.test(tzSql) && tzSql !== "'UTC'") { + throw new Error(`zone frame must be a bound parameter: ${tzSql}`); + } + return { kind: "zone", tzSql }; +} + +function frameTs(frame: DayFrame, column: string): string { + return frame.kind === "session" + ? column + : `((${column} AT TIME ZONE 'UTC') AT TIME ZONE ${frame.tzSql})`; +} + +function weightedRowsSql( + source: string, + frame: DayFrame, + bySource: boolean, +): string { + if (!/^[a-z_][a-z0-9_]*$/.test(source)) { + throw new Error(`source must be a CTE name: ${source}`); + } + const hourOf = (alias: string) => + `date_trunc('hour', ${frameTs(frame, `${alias}."measured_at"`)})`; + const dayOf = (alias: string) => + `date_trunc('day', ${frameTs(frame, `${alias}."measured_at"`)})`; + const src = bySource ? `, t."source"` : ""; + const part = bySource ? `, x."source"` : ""; + const join = bySource ? ` AND hw."source" = s."source"` : ""; + return `( + SELECT s.*, + (CASE WHEN ${isHourlyMeanTypeSql('s."type"')} + THEN 1.0 / (hw.n * hw.hours) + ELSE 1.0 END)::double precision AS day_weight + FROM ${source} s + LEFT JOIN ( + SELECT x."type"${part}, x.local_hour, x.n, + COUNT(*) OVER (PARTITION BY x."type"${part}, x.local_day) AS hours + FROM ( + SELECT t."type"${src}, + ${hourOf("t")} AS local_hour, + ${dayOf("t")} AS local_day, + COUNT(*) AS n + FROM ${source} t + WHERE ${isHourlyMeanTypeSql('t."type"')} + GROUP BY ${bySource ? "1, 2, 3, 4" : "1, 2, 3"} + ) x + ) hw + ON hw."type" = s."type"${join} + AND hw.local_hour = ${hourOf("s")} + )`; +} + +/** + * A subquery over the CTE `source` (which must expose `type`, `measured_at` + * and, with `bySource`, `source`) that adds `day_weight` to every row. Alias + * it at the call site: `FROM ${dayWeightedRows("cm", frame)} m`. + * + * `bySource` keys the hours per source, for a reader that folds each source's + * days before it picks one; a reader over one canonical source per day leaves + * it off. + */ +export function dayWeightedRows( + source: string, + frame: DayFrame, + opts: { bySource?: boolean } = {}, +): string { + return weightedRowsSql(source, frame, opts.bySource === true); +} + +/** + * The Prisma tagged-template twin of `dayWeightedRows`, for readers written + * with `prisma.$queryRaw`: the zone, when given, is a bound parameter. + */ +export function dayWeightedRowsSql( + source: string, + tz: string | null, + opts: { bySource?: boolean } = {}, +): Prisma.Sql { + if (tz === null) { + return Prisma.raw( + weightedRowsSql(source, SESSION_DAY_FRAME, opts.bySource === true), + ); + } + // Split around a placeholder so the zone binds as a parameter. + const marker = "__DAY_MEAN_TZ__"; + const pieces = weightedRowsSql( + source, + { kind: "zone", tzSql: marker }, + opts.bySource === true, + ).split(marker); + let sql = Prisma.raw(pieces[0]); + for (const piece of pieces.slice(1)) { + sql = Prisma.sql`${sql}${tz}${Prisma.raw(piece)}`; + } + return sql; +} + +/** + * Window mean over weighted rows (`dayWeightedRows`): the mean of the day + * values for an hourly-mean type, the unchanged `AVG(value)` otherwise. The + * query groups by the type column (or filters to one type). `filter` is an + * optional SQL predicate applied as an aggregate FILTER. + */ +export function windowMeanSql(opts: { + typeColumn: string; + value: string; + weight: string; + filter?: string; +}): string { + const f = opts.filter ? ` FILTER (WHERE ${opts.filter})` : ""; + return `(CASE WHEN ${isHourlyMeanTypeSql(opts.typeColumn)} + THEN SUM(${opts.value} * ${opts.weight})${f} / NULLIF(SUM(${opts.weight})${f}, 0) + ELSE AVG(${opts.value})${f} END)`; +} + +/** + * Fold of per-day rows into a window mean: `SUM(wsum) / SUM(wdays)` (each day + * once) for an hourly-mean type, the unchanged `SUM(total) / SUM(cnt)` + * otherwise. `wsum` / `wdays` are `SUM(value * day_weight)` / + * `SUM(day_weight)` per day. + */ +export function foldMeanSql(opts: { + typeColumn: string; + total: string; + count: string; + weightedSum: string; + weightSum: string; +}): string { + return `(CASE WHEN ${isHourlyMeanTypeSql(opts.typeColumn)} + THEN SUM(${opts.weightedSum}) / NULLIF(SUM(${opts.weightSum}), 0) + ELSE SUM(${opts.total}) / SUM(${opts.count}) END)`; +} From a5ffde31c2651f0335f2955d6f88c3db1d6fb3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 11:10:55 +0200 Subject: [PATCH 03/10] Recompute stored pulse rollup means to the hourly-mean statistic Rows written before this change keep the reading mean: the boot backfill and the freshness check fill only missing days, so nothing rewrites them. Migration 0373 recomputes the mean of every stored PULSE row from the readings, DAY as the mean of its hours' means and WEEK / MONTH / YEAR as the mean of their days, and leaves count, min, max and the sums alone. The writer stores the coarse rows the same way, so a refold and the migration agree. --- .../0373_pulse_hourly_mean_day/migration.sql | 113 ++++++++++ src/lib/rollups/measurement-rollups.ts | 50 +++-- .../pulse-hourly-mean-migration.test.ts | 201 ++++++++++++++++++ 3 files changed, 346 insertions(+), 18 deletions(-) create mode 100644 prisma/migrations/0373_pulse_hourly_mean_day/migration.sql create mode 100644 tests/integration/pulse-hourly-mean-migration.test.ts diff --git a/prisma/migrations/0373_pulse_hourly_mean_day/migration.sql b/prisma/migrations/0373_pulse_hourly_mean_day/migration.sql new file mode 100644 index 000000000..4b7110762 --- /dev/null +++ b/prisma/migrations/0373_pulse_hourly_mean_day/migration.sql @@ -0,0 +1,113 @@ +-- Pulse: a day is the mean of its hours' means, a coarser bucket the mean of +-- its days. +-- +-- A watch samples pulse about every 5 to 10 minutes at rest and about every +-- 5 seconds in a workout, so the plain mean over a day's readings let one +-- workout hour outweigh the other twenty-three. The rollup writer now stores a +-- PULSE DAY bucket's `mean` as the mean of its per-hour means, and a WEEK / +-- MONTH / YEAR bucket's as the mean of its days' values (see +-- `src/lib/measurements/day-statistic.ts`). Nothing rewrites a stored bucket +-- on its own: the boot backfill and the freshness check fill only missing +-- days, so rows written before this release would keep the reading mean +-- until a write happened to touch their day. This migration recomputes them. +-- +-- Scope, exactly: +-- - only rows of type PULSE, in every granularity; +-- - only the `mean` column. `count`, `min_value`, `max_value`, `sum_value`, +-- `sd`, `slope`, `r2` and the regression sums stay over every reading, as +-- the writer keeps them; +-- - hours, days, weeks, months and years are `date_trunc` on `measured_at` +-- in the session zone, exactly as the writer groups them, per user / source +-- over non-deleted readings. +-- +-- Recomputing from `measurements` makes the migration idempotent: a rerun +-- derives the same means whatever the column held before. A stored row whose +-- readings are all gone is left as it is; the writer prunes such rows on the +-- next fold of their range. +-- +-- The hour means are built once into a temporary table, so the readings are +-- scanned once for all four granularities. +-- +-- Reversibility (down): recompute `mean` as `AVG(value)` over the same groups, +-- or refold the PULSE rollups on a release before this one. No column changes +-- shape. + +CREATE TEMPORARY TABLE pulse_hour_means AS +SELECT + m."user_id" AS user_id, + m."source" AS source, + date_trunc('hour', m."measured_at") AS hr, + AVG(m."value") AS hour_mean +FROM measurements m +WHERE m."type" = 'PULSE' + AND m."deleted_at" IS NULL +GROUP BY m."user_id", m."source", date_trunc('hour', m."measured_at"); + +CREATE TEMPORARY TABLE pulse_day_means AS +SELECT + h.user_id, + h.source, + date_trunc('day', h.hr) AS day, + AVG(h.hour_mean) AS day_mean +FROM pulse_hour_means h +GROUP BY h.user_id, h.source, date_trunc('day', h.hr); + +CREATE INDEX pulse_day_means_key ON pulse_day_means (user_id, source, day); + +-- DAY buckets: the mean of the day's hour means. +UPDATE "measurement_rollups" r +SET "mean" = d.day_mean +FROM pulse_day_means d +WHERE r."granularity" = 'DAY' + AND r."type" = 'PULSE' + AND r."user_id" = d.user_id + AND r."source" = d.source + AND r."bucket_start" = d.day; + +-- WEEK buckets: the mean of the week's day values. +UPDATE "measurement_rollups" r +SET "mean" = s.mean +FROM ( + SELECT d.user_id, d.source, date_trunc('week', d.day) AS bucket_start, + AVG(d.day_mean) AS mean + FROM pulse_day_means d + GROUP BY d.user_id, d.source, date_trunc('week', d.day) +) s +WHERE r."granularity" = 'WEEK' + AND r."type" = 'PULSE' + AND r."user_id" = s.user_id + AND r."source" = s.source + AND r."bucket_start" = s.bucket_start; + +-- MONTH buckets. +UPDATE "measurement_rollups" r +SET "mean" = s.mean +FROM ( + SELECT d.user_id, d.source, date_trunc('month', d.day) AS bucket_start, + AVG(d.day_mean) AS mean + FROM pulse_day_means d + GROUP BY d.user_id, d.source, date_trunc('month', d.day) +) s +WHERE r."granularity" = 'MONTH' + AND r."type" = 'PULSE' + AND r."user_id" = s.user_id + AND r."source" = s.source + AND r."bucket_start" = s.bucket_start; + +-- YEAR buckets. +UPDATE "measurement_rollups" r +SET "mean" = s.mean +FROM ( + SELECT d.user_id, d.source, date_trunc('year', d.day) AS bucket_start, + AVG(d.day_mean) AS mean + FROM pulse_day_means d + GROUP BY d.user_id, d.source, date_trunc('year', d.day) +) s +WHERE r."granularity" = 'YEAR' + AND r."type" = 'PULSE' + AND r."user_id" = s.user_id + AND r."source" = s.source + AND r."bucket_start" = s.bucket_start; + +DROP TABLE pulse_day_means; +DROP TABLE pulse_hour_means; diff --git a/src/lib/rollups/measurement-rollups.ts b/src/lib/rollups/measurement-rollups.ts index f5982bd4b..df8e52bb2 100644 --- a/src/lib/rollups/measurement-rollups.ts +++ b/src/lib/rollups/measurement-rollups.ts @@ -588,11 +588,16 @@ async function runRollupAggregate( /** * For a type whose DAY value is the mean of its per-hour means (see - * `day-statistic.ts`), replace the DAY bucket's `mean`. Only `mean` changes: + * `day-statistic.ts`), replace the bucket's `mean`: a DAY bucket's is the mean + * of its hours' means, a WEEK / MONTH / YEAR bucket's the mean of its days' + * values, each day once. The coarse rows are not read for their mean (every + * coarse read folds the DAY tier, `readCanonicalRollupBuckets`), but a stored + * row should not contradict the tier it is folded from. Only `mean` changes: * `count`, `min`, `max` and the sums the regression composes from stay over * every reading, so the band and the windowed slope are untouched. * - * Hours are UTC hours, matching the UTC day this tier buckets by. + * Hours and days are UTC, matching the UTC day this tier buckets by; migration + * 0373 applies the same statistic to rows stored before it. */ async function applyHourlyMeanDay( db: Pick, @@ -604,7 +609,10 @@ async function applyHourlyMeanDay( }, rows: RollupRow[], ): Promise { - if (input.granularity !== "DAY") return rows; + const truncUnit = DATE_TRUNC_UNIT[input.granularity]; + if (!["day", "week", "month", "year"].includes(truncUnit)) { + throw new Error(`unexpected granularity: ${input.granularity}`); + } const types = [ ...new Set( rows @@ -623,22 +631,28 @@ async function applyHourlyMeanDay( Array<{ type: string; source: string; bucket_start: Date; mean: number }> >( ` - SELECT h."type"::text AS type, - h."source"::text AS source, - date_trunc('day', h.hr) AS bucket_start, - AVG(h.hour_mean)::double precision AS mean + SELECT d."type"::text AS type, + d."source"::text AS source, + date_trunc('${truncUnit}', d.day) AS bucket_start, + AVG(d.day_mean)::double precision AS mean FROM ( - SELECT m."type", m."source", - date_trunc('hour', m."measured_at") AS hr, - AVG(m."value") AS hour_mean - FROM measurements m - WHERE m."user_id" = $1 - AND m."type" IN (${typeList}) - AND m."measured_at" >= $2 - AND m."measured_at" < $3 - AND m."deleted_at" IS NULL - GROUP BY m."type", m."source", date_trunc('hour', m."measured_at") - ) h + SELECT h."type", h."source", + date_trunc('day', h.hr) AS day, + AVG(h.hour_mean) AS day_mean + FROM ( + SELECT m."type", m."source", + date_trunc('hour', m."measured_at") AS hr, + AVG(m."value") AS hour_mean + FROM measurements m + WHERE m."user_id" = $1 + AND m."type" IN (${typeList}) + AND m."measured_at" >= $2 + AND m."measured_at" < $3 + AND m."deleted_at" IS NULL + GROUP BY m."type", m."source", date_trunc('hour', m."measured_at") + ) h + GROUP BY 1, 2, 3 + ) d GROUP BY 1, 2, 3 `, input.userId, diff --git a/tests/integration/pulse-hourly-mean-migration.test.ts b/tests/integration/pulse-hourly-mean-migration.test.ts new file mode 100644 index 000000000..040aa53fa --- /dev/null +++ b/tests/integration/pulse-hourly-mean-migration.test.ts @@ -0,0 +1,201 @@ +/** + * Migration 0373 rewrites the `mean` of stored PULSE rollup rows to the + * hourly-mean statistic, and touches nothing else. + * + * The rows are first built by the writer, then their `mean` is set to a + * sentinel the way a row written before this release still holds the reading + * mean. The migration must restore exactly what the writer stores now: DAY = + * mean of the day's hour means, WEEK / MONTH / YEAR = mean of their days. A + * second type (HRV) with the same readings keeps its sentinel, every other + * column keeps its value, and a rerun changes nothing. + */ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { beforeAll, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { Prisma } from "@/generated/prisma/client"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; + +const prisma = getPrismaClient(); +const USER = "pulse-mean-migration"; +const SENTINEL = 999; + +const MIGRATION = readFileSync( + resolve( + process.cwd(), + "prisma", + "migrations", + "0373_pulse_hourly_mean_day", + "migration.sql", + ), + "utf8", +); + +function steps(): string[] { + return MIGRATION.split("\n") + .filter((line) => !line.trimStart().startsWith("--")) + .join("\n") + .split(";") + .map((s) => s.trim()) + .filter((s) => s.length > 0); +} + +async function runMigration(): Promise { + // One connection: the temporary tables live in the session that made them. + await prisma.$transaction(async (tx) => { + for (const step of steps()) await tx.$executeRawUnsafe(step); + }); +} + +/** Tuesday 2026-03-03: one hard hour of 12 readings at 150, three hours of one reading at 60. */ +const TUE = Date.UTC(2026, 2, 3); +/** Wednesday: two hours at 70. */ +const WED = Date.UTC(2026, 2, 4); +const TUE_VALUE = (150 + 3 * 60) / 4; +const WED_VALUE = 70; + +function rows( + type: "PULSE" | "HEART_RATE_VARIABILITY", +): Prisma.MeasurementCreateManyInput[] { + const out: Prisma.MeasurementCreateManyInput[] = []; + const add = (at: number, value: number) => + out.push({ + id: `${type}-${at}`, + userId: USER, + type, + unit: type === "PULSE" ? "bpm" : "ms", + source: "APPLE_HEALTH", + value, + measuredAt: new Date(at), + }); + for (let i = 0; i < 12; i += 1) add(TUE + 10 * 3_600_000 + i * 300_000, 150); + for (const h of [12, 14, 16]) add(TUE + h * 3_600_000, 60); + for (const h of [8, 9]) add(WED + h * 3_600_000, 70); + return out; +} + +type Row = { + type: string; + granularity: string; + bucket_start: Date; + count: number; + mean: number; + min_value: number; + max_value: number; + sum_value: number | null; + sd: number | null; + sum_x: number | null; + sum_xy: number | null; + sum_xx: number | null; + sum_yy: number | null; +}; + +const readRows = () => + prisma.$queryRaw` + SELECT "type"::text AS type, "granularity"::text AS granularity, + "bucket_start", "count", "mean", "min_value", "max_value", + "sum_value", "sd", "sum_x", "sum_xy", "sum_xx", "sum_yy" + FROM measurement_rollups + WHERE "user_id" = ${USER} + ORDER BY 1, 2, 3 + `; + +let before: Row[] = []; + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, timezone: "UTC" }, + }); + await prisma.measurement.createMany({ + data: [...rows("PULSE"), ...rows("HEART_RATE_VARIABILITY")], + }); + await recomputeUserRollups(USER, { + types: ["PULSE", "HEART_RATE_VARIABILITY"], + from: new Date(Date.UTC(2026, 0, 1)), + to: new Date(Date.UTC(2027, 0, 1)), + }); + await prisma.$executeRaw` + UPDATE measurement_rollups SET "mean" = ${SENTINEL} WHERE "user_id" = ${USER} + `; + before = await readRows(); + await runMigration(); +}, 120_000); + +const meanOf = (all: Row[], type: string, granularity: string, at?: number) => + all.find( + (r) => + r.type === type && + r.granularity === granularity && + (at === undefined || r.bucket_start.getTime() === at), + )?.mean; + +describe("migration 0373", () => { + it("covers all four granularities of both types", () => { + for (const type of ["PULSE", "HEART_RATE_VARIABILITY"]) { + for (const g of ["DAY", "WEEK", "MONTH", "YEAR"]) { + expect(meanOf(before, type, g)).toBe(SENTINEL); + } + } + }); + + it("gives a PULSE day the mean of its hours' means", async () => { + const after = await readRows(); + expect(meanOf(after, "PULSE", "DAY", TUE)).toBeCloseTo(TUE_VALUE, 9); + expect(meanOf(after, "PULSE", "DAY", WED)).toBeCloseTo(WED_VALUE, 9); + }); + + it("gives a PULSE week, month and year the mean of their days", async () => { + const after = await readRows(); + for (const g of ["WEEK", "MONTH", "YEAR"]) { + expect(meanOf(after, "PULSE", g)).toBeCloseTo( + (TUE_VALUE + WED_VALUE) / 2, + 9, + ); + } + }); + + it("matches what the writer stores now", async () => { + const migrated = await readRows(); + await recomputeUserRollups(USER, { + types: ["PULSE"], + from: new Date(Date.UTC(2026, 0, 1)), + to: new Date(Date.UTC(2027, 0, 1)), + }); + const written = await readRows(); + for (const r of written.filter((x) => x.type === "PULSE")) { + const m = migrated.find( + (x) => + x.type === r.type && + x.granularity === r.granularity && + x.bucket_start.getTime() === r.bucket_start.getTime(), + ); + expect(m?.mean).toBeCloseTo(r.mean, 9); + } + // Restore the migrated state for the remaining assertions. + await prisma.$executeRaw` + UPDATE measurement_rollups SET "mean" = ${SENTINEL} + WHERE "user_id" = ${USER} AND "type" = 'HEART_RATE_VARIABILITY' + `; + }); + + it("leaves every other type's mean alone", async () => { + const after = await readRows(); + for (const r of after.filter((x) => x.type !== "PULSE")) { + expect(r.mean).toBe(SENTINEL); + } + }); + + it("leaves every column but mean alone", async () => { + const after = await readRows(); + const strip = ({ mean: _mean, ...rest }: Row) => rest; + expect(after.map(strip)).toEqual(before.map(strip)); + }); + + it("is idempotent", async () => { + const first = await readRows(); + await runMigration(); + expect(await readRows()).toEqual(first); + }); +}); From 3634f6c5206e602253ac5bf70dc8afa233cbe16b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 11:22:17 +0200 Subject: [PATCH 04/10] Average pulse by day in the dashboard, insights and series stats The dashboard summaries, the insights aggregate and the series stats strip still took a plain mean over pulse readings for their 7-day, 30-day, prior-month and all-time figures, while the year-ago figure and the chart already weighed each day once, so the year delta compared two different statistics. They now go through the day-mean helper: each day is the mean of its hours' means and a window the mean of its days, in the frame each reader buckets its days by. Count, min, max, the median and the spread stay over every reading; every other type is unchanged. --- src/app/api/measurements/series/route.ts | 86 +++++-- .../__tests__/summaries-slice.test.ts | 29 +-- src/lib/analytics/summaries-slice.ts | 140 +++++++---- src/lib/insights/comprehensive-aggregator.ts | 64 +++-- .../integration/pulse-day-mean-zones.test.ts | 221 ++++++++++++++++++ .../pulse-window-mean-readers.test.ts | 209 +++++++++++++++++ 6 files changed, 658 insertions(+), 91 deletions(-) create mode 100644 tests/integration/pulse-day-mean-zones.test.ts create mode 100644 tests/integration/pulse-window-mean-readers.test.ts diff --git a/src/app/api/measurements/series/route.ts b/src/app/api/measurements/series/route.ts index 794660abe..f91091ced 100644 --- a/src/app/api/measurements/series/route.ts +++ b/src/app/api/measurements/series/route.ts @@ -27,6 +27,12 @@ import { reconstructSleepNights } from "@/lib/analytics/sleep-night"; import { VALUE_RANGES } from "@/lib/validations/measurement"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; +import { + dayWeightedRows, + readingsMean, + windowMeanSql, + zoneDayFrame, +} from "@/lib/measurements/day-mean"; import { seriesRowsFrom } from "@/lib/measurements/series-canonical"; import { resolveUserTimezone, userDayKey } from "@/lib/tz/resolver"; import { convertGlucose, resolveGlucoseUnit } from "@/lib/glucose"; @@ -487,17 +493,50 @@ export const GET = apiHandler(async (request: NextRequest) => { // strip's mean/min/max/stdDev/count are the same figures the raw path // computed: `AVG`/`MIN`/`MAX` over `value`, `STDDEV_POP` matching the // JS population-variance helper, exact row count. Aggregate-only — - // no row transfer. - const [rawAgg] = await prisma.$queryRawUnsafe< - Array<{ - n: number; - mean: number | null; - min: number | null; - max: number | null; - sd: number | null; - }> - >( - ` + // no row transfer. For an hourly-mean type (pulse) the mean is the mean + // of the local days' values, each day the mean of its local hours' + // means, so it is the mean of the day points the chart draws; count, + // min, max and the spread stay over every reading. + const [rawAgg] = usesHourlyMeanDay(type) + ? await prisma.$queryRawUnsafe< + Array<{ + n: number; + mean: number | null; + min: number | null; + max: number | null; + sd: number | null; + }> + >( + ` + WITH src AS ( + SELECT m.* + FROM ${seriesRows} + WHERE m."measured_at" >= $2 + ) + SELECT + COUNT(*)::int AS n, + ${windowMeanSql({ typeColumn: 'm."type"', value: 'm."value"', weight: "m.day_weight" })}::double precision + AS mean, + MIN(m."value")::double precision AS min, + MAX(m."value")::double precision AS max, + STDDEV_POP(m."value")::double precision AS sd + FROM ${dayWeightedRows("src", zoneDayFrame("$3"))} m + GROUP BY m."type" + `, + user.id, + since, + userTz, + ) + : await prisma.$queryRawUnsafe< + Array<{ + n: number; + mean: number | null; + min: number | null; + max: number | null; + sd: number | null; + }> + >( + ` SELECT COUNT(*)::int AS n, AVG(m."value")::double precision AS mean, @@ -507,9 +546,9 @@ export const GET = apiHandler(async (request: NextRequest) => { FROM ${seriesRows} WHERE m."measured_at" >= $2 `, - user.id, - since, - ); + user.id, + since, + ); const round2 = (v: number) => Math.round(v * 100) / 100; const dayId = (d: Date) => grain === "day" @@ -687,6 +726,25 @@ export const GET = apiHandler(async (request: NextRequest) => { valueMax: row.valueMax ?? null, }; }); + // An hourly-mean type's strip mean is the mean of its local days' + // values, each day the mean of its local hours' means, so a dense + // workout hour does not outweigh the rest of the window. Count, min, + // max and the spread stay over every reading. + if (usesHourlyMeanDay(type) && rows.length > 0) { + const userTz = await resolveUserTimezone(user.id); + const values = rows.map((r) => r.value); + const rawSummary = summarize( + rows.map((r) => ({ date: r.measuredAt, value: r.value })), + ); + const mean = readingsMean(type, rows, userTz); + statsOverride = { + mean: mean === null ? 0 : Math.round(mean * 100) / 100, + min: rawSummary.min ?? 0, + max: rawSummary.max ?? 0, + stdDev: stdDev(values), + count: rawSummary.count, + }; + } } else { points = rows.map((r) => ({ id: r.id, diff --git a/src/lib/analytics/__tests__/summaries-slice.test.ts b/src/lib/analytics/__tests__/summaries-slice.test.ts index 9425ee5cf..f6b58107c 100644 --- a/src/lib/analytics/__tests__/summaries-slice.test.ts +++ b/src/lib/analytics/__tests__/summaries-slice.test.ts @@ -45,11 +45,9 @@ vi.mock("@/lib/logging/context", () => ({ // 90-day FILTER caps the slice itself writes — is exercised without // coupling the slice unit test to the rank builder's enum-whitelist // internals. -vi.mock("@/lib/analytics/source-rank-sql", () => ({ - buildSourceRankCase: vi.fn(() => "90"), - canonicalMeasurementsFrom: vi.fn( - (_rank: string, sinceInterval?: string) => - `( +vi.mock("@/lib/analytics/source-rank-sql", () => { + const cte = (_rank: string, sinceInterval?: string) => + ` SELECT mm.* FROM measurements mm WHERE mm."user_id" = $1 @@ -58,13 +56,15 @@ vi.mock("@/lib/analytics/source-rank-sql", () => ({ sinceInterval ? `AND mm."measured_at" >= NOW() - INTERVAL '${sinceInterval}'` : "" - } - ) m`, - ), -})); + }`; + return { + buildSourceRankCase: vi.fn(() => "90"), + canonicalMeasurementsCte: vi.fn(cte), + }; +}); import { prisma } from "@/lib/db"; -import { canonicalMeasurementsFrom } from "@/lib/analytics/source-rank-sql"; +import { canonicalMeasurementsCte } from "@/lib/analytics/source-rank-sql"; import { startOfUtcDay } from "@/lib/tz/start-of-utc-day"; import { computeSummariesSlice } from "../summaries-slice"; @@ -79,7 +79,7 @@ const ROLLUP_FIND_MANY = prisma.measurementRollup .findMany as unknown as ReturnType; const ROLLUP_FIND_FIRST = prisma.measurementRollup .findFirst as unknown as ReturnType; -const CANONICAL_FROM = canonicalMeasurementsFrom as unknown as ReturnType< +const CANONICAL_FROM = canonicalMeasurementsCte as unknown as ReturnType< typeof vi.fn >; @@ -609,11 +609,12 @@ describe("computeSummariesSlice", () => { expect(narrowsCap?.length).toBeGreaterThanOrEqual(1); // v1.11.1 — the cold-fallback `windowed` scan delegates its outer - // 90-day cap to `canonicalMeasurementsFrom(rank, "90 days")`. The + // 90-day cap to `canonicalMeasurementsCte(rank, "90 days")` (a CTE + // now, so the day weights can read it twice). The // helper lives in `@/lib/analytics/source-rank-sql` (stubbed // above), so pin the contract at the call boundary: the slice // must ask for the 90-day window. - expect(canonicalMeasurementsFrom).toHaveBeenCalledWith( + expect(canonicalMeasurementsCte).toHaveBeenCalledWith( expect.any(String), "90 days", ); @@ -621,7 +622,7 @@ describe("computeSummariesSlice", () => { // 15-year scan-DoS floor (far beyond any real history) so a // coverage-miss cold read can't trigger an unbounded full-partition // scan. It no longer calls the helper without an interval. - expect(canonicalMeasurementsFrom).toHaveBeenCalledWith( + expect(canonicalMeasurementsCte).toHaveBeenCalledWith( expect.any(String), "15 years", ); diff --git a/src/lib/analytics/summaries-slice.ts b/src/lib/analytics/summaries-slice.ts index 76dc34e6d..0eb7ea343 100644 --- a/src/lib/analytics/summaries-slice.ts +++ b/src/lib/analytics/summaries-slice.ts @@ -60,7 +60,18 @@ import type { MeasurementType } from "@/generated/prisma/client"; import { Prisma } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; -import { windowWeighting } from "@/lib/measurements/day-statistic"; +import { + usesHourlyMeanDay, + windowWeighting, +} from "@/lib/measurements/day-statistic"; +import { + SESSION_DAY_FRAME, + dayWeightedRows, + dayWeightedRowsSql, + foldMeanSql, + isHourlyMeanTypeSql, + windowMeanSql, +} from "@/lib/measurements/day-mean"; import type { DataSummary } from "@/lib/analytics/trends"; import { measurementTypeEnum } from "@/lib/validations/measurement"; import { annotate } from "@/lib/logging/context"; @@ -80,7 +91,7 @@ import { import { startOfUtcDay } from "@/lib/tz/start-of-utc-day"; import { buildSourceRankCase, - canonicalMeasurementsFrom, + canonicalMeasurementsCte, } from "@/lib/analytics/source-rank-sql"; import { isFullyCovered, @@ -137,11 +148,33 @@ interface AllTimeAggregateRow { interface RollupAggregateRow { type: string; count: number; + /** Days that hold a reading: the weight of the mean for an hourly-mean type. */ + days: number; min: number; max: number; mean: number; } +/** + * The window means over canonical readings that carry a `day_weight` (see + * `day-mean.ts`): the mean of the day values for an hourly-mean type (pulse), + * the unchanged `AVG(value)` for every other type. Days are UTC days, the + * frame the canonical-source pick and the rollup tier bucket by. + */ +function windowedMean(filter: string): string { + return windowMeanSql({ + typeColumn: 'm."type"', + value: 'm."value"', + weight: "m.day_weight", + filter, + }); +} +const AVG7_SQL = windowedMean(`m."measured_at" >= NOW() - INTERVAL '7 days'`); +const AVG30_SQL = windowedMean(`m."measured_at" >= NOW() - INTERVAL '30 days'`); +const AVG30_LAST_MONTH_SQL = windowedMean( + `m."measured_at" >= NOW() - INTERVAL '60 days' AND m."measured_at" < NOW() - INTERVAL '30 days'`, +); + interface NarrowAggregateRow { type: string; avg7: number | null; @@ -455,26 +488,23 @@ async function computeFromRollups(userId: string): Promise { // inner DISTINCT ON picks the ladder-winning source for each day, and the // join keeps only that source's readings so the 90-day AVG / median / slope // never blend two devices that both reported the same vital. + // The window means weigh each day once for an hourly-mean type + // (`windowedMean`); the median stays over every reading. prisma.$queryRawUnsafe( ` + WITH cm AS (${canonicalMeasurementsCte(rankUnqualified, "90 days")} + ) SELECT m."type"::text AS type, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '7 days' - )::double precision AS avg7, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '30 days' - )::double precision AS avg30, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '60 days' - AND m."measured_at" < NOW() - INTERVAL '30 days' - )::double precision AS avg30_last_month, + ${AVG7_SQL}::double precision AS avg7, + ${AVG30_SQL}::double precision AS avg30, + ${AVG30_LAST_MONTH_SQL}::double precision AS avg30_last_month, PERCENTILE_CONT(0.5) WITHIN GROUP ( ORDER BY m."value" ) FILTER ( WHERE m."measured_at" >= NOW() - INTERVAL '90 days' )::double precision AS median - FROM ${canonicalMeasurementsFrom(rankUnqualified, "90 days")} + FROM ${dayWeightedRows("cm", SESSION_DAY_FRAME)} m GROUP BY m."type" `, userId, @@ -527,12 +557,19 @@ async function computeFromRollups(userId: string): Promise { SELECT "type"::text AS type, SUM("count")::int AS count, + COUNT(*)::int AS days, MIN("min_value")::double precision AS min, MAX("max_value")::double precision AS max, - ( - SUM("count" * "mean")::double precision - / NULLIF(SUM("count")::double precision, 0) - ) AS mean + -- An hourly-mean type (pulse): every day once, the DAY mean already + -- being the mean of its hours' means. Every other type: every + -- reading once. + (CASE WHEN ${isHourlyMeanTypeSql('"type"')} + THEN AVG("mean")::double precision + ELSE ( + SUM("count" * "mean")::double precision + / NULLIF(SUM("count")::double precision, 0) + ) + END) AS mean FROM collapsed GROUP BY "type" `, @@ -576,16 +613,27 @@ async function computeFromRollups(userId: string): Promise { // day first (a hash aggregate, a few rows per day however dense the // stream), each day then keeps its ladder-canonical source, so the // sort never sees more than one row per source per day. + // An hourly-mean type folds each source's day as the mean of its + // hours' means (`wsum` / `wdays`, see `day-mean.ts`) and the days into + // the remainder once each, the statistic the DAY rollup stores. prisma.$queryRaw< Array<{ type: string; count: number; + days: number; min: number; max: number; mean: number; }> >` - WITH per_source AS ( + WITH src AS ( + SELECT * + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."measured_at" < ${foldSplit} + ), + per_source AS ( SELECT m."type", m."source", @@ -593,25 +641,33 @@ async function computeFromRollups(userId: string): Promise { COUNT(*)::int AS cnt, SUM(m."value")::double precision AS total, MIN(m."value")::double precision AS min_value, - MAX(m."value")::double precision AS max_value - FROM measurements m - WHERE m."user_id" = ${userId} - AND m."deleted_at" IS NULL - AND m."measured_at" < ${foldSplit} + MAX(m."value")::double precision AS max_value, + SUM(m."value" * m.day_weight)::double precision AS wsum, + SUM(m.day_weight)::double precision AS wdays + FROM ${dayWeightedRowsSql("src", null, { bySource: true })} m GROUP BY m."type", m."source", 3 ), canon AS ( SELECT DISTINCT ON (p."type", p.day) - p."type", p.cnt, p.total, p.min_value, p.max_value + p."type", p.cnt, p.total, p.min_value, p.max_value, p.wsum, p.wdays FROM per_source p ORDER BY p."type", p.day, (${Prisma.raw(rankPerSource)}), p."source"::text ) SELECT c."type"::text AS type, SUM(c.cnt)::int AS count, + COUNT(*)::int AS days, MIN(c.min_value)::double precision AS min, MAX(c.max_value)::double precision AS max, - (SUM(c.total) / SUM(c.cnt))::double precision AS mean + ${Prisma.raw( + foldMeanSql({ + typeColumn: 'c."type"', + total: "c.total", + count: "c.cnt", + weightedSum: "c.wsum", + weightSum: "c.wdays", + }), + )}::double precision AS mean FROM canon c GROUP BY c."type" `, @@ -619,11 +675,12 @@ async function computeFromRollups(userId: string): Promise { const preFoldByType = new Map< string, - { count: number; min: number; max: number; mean: number } + { count: number; days: number; min: number; max: number; mean: number } >(); for (const row of preFoldRemainder) { preFoldByType.set(row.type, { count: Number(row.count), + days: Number(row.days), min: Number(row.min), max: Number(row.max), mean: Number(row.mean), @@ -754,8 +811,13 @@ async function computeFromRollups(userId: string): Promise { const allCount = row.count + (pre?.count ?? 0); const allMin = pre ? Math.min(row.min, pre.min) : row.min; const allMax = pre ? Math.max(row.max, pre.max) : row.max; + // Both halves weigh their days once for an hourly-mean type (pulse), so + // they combine by day count; every other type combines by readings. const allMean = pre - ? (row.mean * row.count + pre.mean * pre.count) / allCount + ? usesHourlyMeanDay(row.type) + ? (row.mean * Number(row.days) + pre.mean * pre.days) / + (Number(row.days) + pre.days) + : (row.mean * row.count + pre.mean * pre.count) / allCount : row.mean; summaries[row.type] = { count: allCount, @@ -889,31 +951,29 @@ async function computeFromLiveAggregate( const [allTime, windowed, latests] = await Promise.all([ prisma.$queryRawUnsafe( ` + WITH cm AS (${canonicalMeasurementsCte(rankUnqualified, ALL_TIME_LIVE_CAP_INTERVAL)} + ) SELECT m."type"::text AS type, COUNT(*) AS count, MIN(m."value")::double precision AS min_value, MAX(m."value")::double precision AS max_value, - AVG(m."value")::double precision AS mean_value - FROM ${canonicalMeasurementsFrom(rankUnqualified, ALL_TIME_LIVE_CAP_INTERVAL)} + ${windowMeanSql({ typeColumn: 'm."type"', value: 'm."value"', weight: "m.day_weight" })}::double precision + AS mean_value + FROM ${dayWeightedRows("cm", SESSION_DAY_FRAME)} m GROUP BY m."type" `, userId, ), prisma.$queryRawUnsafe( ` + WITH cm AS (${canonicalMeasurementsCte(rankUnqualified, "90 days")} + ) SELECT m."type"::text AS type, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '7 days' - )::double precision AS avg7, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '30 days' - )::double precision AS avg30, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '60 days' - AND m."measured_at" < NOW() - INTERVAL '30 days' - )::double precision AS avg30_last_month, + ${AVG7_SQL}::double precision AS avg7, + ${AVG30_SQL}::double precision AS avg30, + ${AVG30_LAST_MONTH_SQL}::double precision AS avg30_last_month, PERCENTILE_CONT(0.5) WITHIN GROUP ( ORDER BY m."value" ) FILTER ( @@ -956,7 +1016,7 @@ async function computeFromLiveAggregate( ) FILTER ( WHERE m."measured_at" >= (date_trunc('day', NOW() AT TIME ZONE 'UTC') - INTERVAL '30 days') AT TIME ZONE 'UTC' )::double precision AS r2_30 - FROM ${canonicalMeasurementsFrom(rankUnqualified, "90 days")} + FROM ${dayWeightedRows("cm", SESSION_DAY_FRAME)} m GROUP BY m."type" `, userId, diff --git a/src/lib/insights/comprehensive-aggregator.ts b/src/lib/insights/comprehensive-aggregator.ts index e828eb189..8d562abd0 100644 --- a/src/lib/insights/comprehensive-aggregator.ts +++ b/src/lib/insights/comprehensive-aggregator.ts @@ -114,6 +114,11 @@ import { type AccumulatorBucketRow, } from "@/lib/rollups/measurement-read"; import { windowWeighting } from "@/lib/measurements/day-statistic"; +import { + SESSION_DAY_FRAME, + dayWeightedRows, + windowMeanSql, +} from "@/lib/measurements/day-mean"; import { startOfUtcDay } from "@/lib/tz/start-of-utc-day"; import { buildSourceRankCase, @@ -126,6 +131,26 @@ import { } from "@/lib/rollups/measurement-coverage"; import { dateOnlyKey } from "@/lib/tz/date-only"; +/** + * Window means over canonical readings carrying a `day_weight` (see + * `day-mean.ts`): the mean of the day values for an hourly-mean type (pulse), + * the unchanged `AVG(value)` otherwise. Days are UTC days, the frame the + * canonical-source pick buckets by. + */ +function windowedMean(filter: string): string { + return windowMeanSql({ + typeColumn: 'm."type"', + value: 'm."value"', + weight: "m.day_weight", + filter, + }); +} +const AVG7_SQL = windowedMean(`m."measured_at" >= NOW() - INTERVAL '7 days'`); +const AVG30_SQL = windowedMean(`m."measured_at" >= NOW() - INTERVAL '30 days'`); +const AVG30_LAST_MONTH_SQL = windowedMean( + `m."measured_at" >= NOW() - INTERVAL '60 days' AND m."measured_at" < NOW() - INTERVAL '30 days'`, +); + /** * Heavy aggregate row — count/min/max/mean alongside the non-composable * windowed columns. Used on the cold-mount fallback path where the @@ -419,6 +444,9 @@ async function buildFromRollups( -- instead of twice per rebuild. Output is identical. WITH cm AS (${canonicalMeasurementsCte(rankUnqualified, "90 days")} ), + -- The z-score's mean and spread stay over every reading: they measure + -- how far one reading sits from the others, a per-reading dispersion, + -- not the level the window means below report. window_stats AS ( SELECT m."type", @@ -440,18 +468,11 @@ async function buildFromRollups( (ROUND(((m."value" - ws.mean_value) / ws.stddev_value)::numeric, 2)) ) > 2 ) AS anomaly_count, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '7 days' - )::double precision AS avg7, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '30 days' - )::double precision AS avg30, + ${AVG7_SQL}::double precision AS avg7, + ${AVG30_SQL}::double precision AS avg30, -- Prior 30d window (days [30, 60) ago) used by tile delta callout. - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '60 days' - AND m."measured_at" < NOW() - INTERVAL '30 days' - )::double precision AS avg30_last_month - FROM cm m + ${AVG30_LAST_MONTH_SQL}::double precision AS avg30_last_month + FROM ${dayWeightedRows("cm", SESSION_DAY_FRAME)} m JOIN window_stats ws ON ws."type" = m."type" GROUP BY m."type", ws.stddev_value `, @@ -662,6 +683,9 @@ async function buildFromLiveAggregate( -- so the DISTINCT-ON pick runs once. Output is identical. WITH cm AS (${canonicalMeasurementsCte(rankUnqualified, "90 days")} ), + -- The z-score's mean and spread stay over every reading: they measure + -- how far one reading sits from the others, a per-reading dispersion, + -- not the level the window means below report. window_stats AS ( SELECT m."type", @@ -675,7 +699,8 @@ async function buildFromLiveAggregate( COUNT(*) AS count, MIN(m."value")::double precision AS min_value, MAX(m."value")::double precision AS max_value, - AVG(m."value")::double precision AS mean_value, + ${windowMeanSql({ typeColumn: 'm."type"', value: 'm."value"', weight: "m.day_weight" })}::double precision + AS mean_value, ws.stddev_value::double precision AS stddev_value, COUNT(*) FILTER ( WHERE ws.stddev_value IS NOT NULL @@ -684,16 +709,9 @@ async function buildFromLiveAggregate( (ROUND(((m."value" - ws.mean_value) / ws.stddev_value)::numeric, 2)) ) > 2 ) AS anomaly_count, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '7 days' - )::double precision AS avg7, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '30 days' - )::double precision AS avg30, - AVG(m."value") FILTER ( - WHERE m."measured_at" >= NOW() - INTERVAL '60 days' - AND m."measured_at" < NOW() - INTERVAL '30 days' - )::double precision AS avg30_last_month, + ${AVG7_SQL}::double precision AS avg7, + ${AVG30_SQL}::double precision AS avg30, + ${AVG30_LAST_MONTH_SQL}::double precision AS avg30_last_month, -- A2-M2 — the regression windows anchor on the UTC-midnight day -- boundary, NOT the wall-clock instant, so this cold-fallback path -- returns the SAME boundary-day membership the warm rollup path @@ -731,7 +749,7 @@ async function buildFromLiveAggregate( ) FILTER ( WHERE m."measured_at" >= (date_trunc('day', NOW() AT TIME ZONE 'UTC') - INTERVAL '30 days') AT TIME ZONE 'UTC' )::double precision AS r2_30 - FROM cm m + FROM ${dayWeightedRows("cm", SESSION_DAY_FRAME)} m JOIN window_stats ws ON ws."type" = m."type" GROUP BY m."type", ws.stddev_value `, diff --git a/tests/integration/pulse-day-mean-zones.test.ts b/tests/integration/pulse-day-mean-zones.test.ts new file mode 100644 index 000000000..da0bdc286 --- /dev/null +++ b/tests/integration/pulse-day-mean-zones.test.ts @@ -0,0 +1,221 @@ +/** + * A day of pulse is the mean of its LOCAL hours' means, and the days are LOCAL + * days, for a profile zone that is not UTC. + * + * Two zones that each break a UTC assumption: + * + * - Asia/Kolkata (UTC+05:30). Its hours start at half past a UTC hour, so + * 10:20Z, 10:25Z (both 15:5x local) and 10:40Z (16:10 local) are two local + * hours (100 and 40, day value 70), though one UTC hour (80). 18:45Z is + * already 00:15 the next local day. + * - Europe/Berlin. 23:30Z is the next local day, so 10:00Z, 11:00Z (three + * readings) and 23:30Z are two local days (100 and 40), though one UTC day + * (80). + * + * Asserted through the live per-day read the chart serves, the series route's + * day bucket, and the shared SQL day weight every other live reader uses. + */ +import { NextRequest } from "next/server"; +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { cookieJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { Prisma } from "@/generated/prisma/client"; +import { readLiveBuckets } from "@/lib/measurements/daily-series-read"; +import { + dayWeightedRows, + dayWeightedRowsSql, + readingsMean, + windowMeanSql, + zoneDayFrame, +} from "@/lib/measurements/day-mean"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const DAY = 86_400_000; +const now = new Date(); +/** A UTC midnight three weeks back: inside every window the reads use. */ +const D = Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() - 21, +); +const iso = (ms: number) => new Date(ms).toISOString(); +const hm = (h: number, m: number) => D + h * 3_600_000 + m * 60_000; + +const KOLKATA = { + user: "pulse-zone-kolkata", + tz: "Asia/Kolkata", + readings: [ + [hm(10, 20), 100], + [hm(10, 25), 100], + [hm(10, 40), 40], + [hm(18, 45), 90], + ] as const, + /** Local days: 70 (two local hours) and 90. */ + days: [70, 90], + /** What a UTC frame would give: one day, hours 80 and 90. */ + utcDay: 85, +}; + +const BERLIN = { + user: "pulse-zone-berlin", + tz: "Europe/Berlin", + readings: [ + [hm(10, 0), 100], + [hm(11, 0), 100], + [hm(11, 10), 100], + [hm(11, 20), 100], + [hm(23, 30), 40], + ] as const, + /** Local days: 100 and 40. */ + days: [100, 40], + /** What a UTC frame would give: one day, hours 100, 100 and 40. */ + utcDay: 80, +}; + +const CASES = [KOLKATA, BERLIN]; + +beforeAll(async () => { + await truncateAllTables(prisma); + for (const c of CASES) { + await prisma.user.create({ + data: { id: c.user, username: c.user, timezone: c.tz }, + }); + invalidateUserTimezone(c.user); + const rows: Prisma.MeasurementCreateManyInput[] = c.readings.map( + ([at, value], i) => ({ + id: `${c.user}-${i}`, + userId: c.user, + type: "PULSE", + unit: "bpm", + source: "APPLE_HEALTH", + value, + measuredAt: new Date(at), + }), + ); + await prisma.measurement.createMany({ data: rows }); + } +}, 120_000); + +describe.each(CASES)("profile zone $tz", (c) => { + it("the TypeScript helper reads local hours and local days", () => { + const rows = c.readings.map(([at, value]) => ({ + value, + measuredAt: new Date(at), + })); + expect(readingsMean("PULSE", rows, c.tz)).toBeCloseTo( + (c.days[0] + c.days[1]) / 2, + 9, + ); + expect(readingsMean("PULSE", rows, "UTC")).toBeCloseTo(c.utcDay, 9); + }); + + it("the live per-day read buckets by local hour and local day", async () => { + const rows = await readLiveBuckets({ + userId: c.user, + type: "PULSE", + from: new Date(D - DAY), + to: new Date(D + 2 * DAY), + cap: 400, + priorityJson: null, + grain: "daily", + timeZone: c.tz, + }); + expect(rows.map((r) => r.value)).toEqual( + c.days.map((v) => expect.closeTo(v, 6)), + ); + // Count and band stay over every reading. + expect(rows.reduce((s, r) => s + (r.count ?? 0), 0)).toBe( + c.readings.length, + ); + }); + + it("the shared SQL day weight gives the mean of the local days", async () => { + const [row] = await prisma.$queryRawUnsafe>( + ` + WITH src AS ( + SELECT * FROM measurements + WHERE "user_id" = $1 AND "deleted_at" IS NULL + ) + SELECT ${windowMeanSql({ + typeColumn: 'm."type"', + value: 'm."value"', + weight: "m.day_weight", + })}::double precision AS mean + FROM ${dayWeightedRows("src", zoneDayFrame("$2"))} m + GROUP BY m."type" + `, + c.user, + c.tz, + ); + expect(row.mean).toBeCloseTo((c.days[0] + c.days[1]) / 2, 9); + + const [tagged] = await prisma.$queryRaw>` + WITH src AS ( + SELECT * FROM measurements + WHERE "user_id" = ${c.user} AND "deleted_at" IS NULL + ) + SELECT SUM(m.day_weight)::double precision AS days + FROM ${dayWeightedRowsSql("src", c.tz)} m + `; + // Each local day's weights add up to one. + expect(tagged.days).toBeCloseTo(2, 9); + }); + + it("the series route's day bucket is the local day of local hours", async () => { + const s = await prisma.session.create({ + data: { userId: c.user, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.clear(); + cookieJar.set("healthlog_session", s.id); + const { GET } = await import("@/app/api/measurements/series/route"); + const res = await GET( + new NextRequest( + "http://localhost/api/measurements/series?kind=pulse&days=365", + ), + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { + points: Array<{ id: string; value: number }>; + stats: { mean: number; count: number }; + }; + }; + expect(body.data.points.every((p) => p.id.startsWith("day:"))).toBe(true); + expect(body.data.points.map((p) => p.value)).toEqual( + c.days.map((v) => expect.closeTo(v, 2)), + ); + // The stats strip: the mean of the day values, the count of readings. + expect(body.data.stats.mean).toBeCloseTo((c.days[0] + c.days[1]) / 2, 2); + expect(body.data.stats.count).toBe(c.readings.length); + }); +}); + +it("the fixture really separates the frames", () => { + expect(iso(hm(18, 45))).toMatch(/T18:45/); + for (const c of CASES) { + expect((c.days[0] + c.days[1]) / 2).not.toBeCloseTo(c.utcDay, 2); + } +}); diff --git a/tests/integration/pulse-window-mean-readers.test.ts b/tests/integration/pulse-window-mean-readers.test.ts new file mode 100644 index 000000000..086adfef8 --- /dev/null +++ b/tests/integration/pulse-window-mean-readers.test.ts @@ -0,0 +1,209 @@ +/** + * The dashboard's and the insights' window means of pulse weigh each day once, + * each day the mean of its hours' means; heart-rate variability, seeded with + * the very same readings, keeps the plain mean over its readings. + * + * A workout day: twelve readings at 150 bpm inside one hour, then three + * resting hours of one reading at 60 bpm. Its day value is (150 + 3 x 60) / 4 + * = 82.5, its readings' mean (12 x 150 + 3 x 60) / 15 = 132. A quiet day: one + * reading at 60. + * + * last 30 days workout day + quiet day 71.25 (plain 127.5) + * days 30 to 60 workout day + quiet day 71.25 (plain 127.5) + * six years ago workout day alone 82.5 (plain 132) + * all time five days 73.5 (plain 6060 / 47) + * + * Asserted on both paths of the dashboard summaries slice (the rollup tier, + * with the six-year-old day in the pre-fold remainder, and the live + * fallback), both paths of the insights aggregate, and the stats strip of the + * series route's raw window. + */ +import { NextRequest } from "next/server"; +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { cookieJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { Prisma } from "@/generated/prisma/client"; +import { computeSummariesSlice } from "@/lib/analytics/summaries-slice"; +import { buildComprehensiveAggregate } from "@/lib/insights/comprehensive-aggregator"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => cookieJar.set(name, value), + delete: (name: string) => cookieJar.delete(name), + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +// The reads kick off a background rollup refresh; keep it off so the live +// user stays on the live path for the whole file. +vi.mock("@/lib/rollups/measurement-rollups", async (importOriginal) => { + const actual = + await importOriginal(); + return { ...actual, ensureUserRollupsFresh: vi.fn(async () => undefined) }; +}); + +const prisma = getPrismaClient(); +const DAY = 86_400_000; +const now = new Date(); +const T0 = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()); + +const LIVE = "pulse-window-live"; +const ROLLUP = "pulse-window-rollup"; + +const WORKOUT_DAYS = [T0 - 3 * DAY, T0 - 45 * DAY, T0 - 6 * 365 * DAY]; +const QUIET_DAYS = [T0 - 2 * DAY, T0 - 44 * DAY]; + +const HOURLY = { + window: (82.5 + 60) / 2, + allTime: (82.5 * 3 + 60 * 2) / 5, +}; +const PLAIN = { + window: (12 * 150 + 3 * 60 + 60) / 16, + allTime: (3 * (12 * 150 + 3 * 60) + 2 * 60) / 47, +}; + +function rows( + userId: string, + type: "PULSE" | "HEART_RATE_VARIABILITY", +): Prisma.MeasurementCreateManyInput[] { + const unit = type === "PULSE" ? "bpm" : "ms"; + const out: Prisma.MeasurementCreateManyInput[] = []; + const push = (at: number, value: number) => + out.push({ + id: `${userId}-${type}-${at}`, + userId, + type, + unit, + source: "APPLE_HEALTH", + value, + measuredAt: new Date(at), + }); + for (const d of WORKOUT_DAYS) { + for (let i = 0; i < 12; i += 1) push(d + 10 * 3_600_000 + i * 300_000, 150); + for (const h of [12, 14, 16]) push(d + h * 3_600_000, 60); + } + for (const d of QUIET_DAYS) push(d + 8 * 3_600_000, 60); + return out; +} + +beforeAll(async () => { + await truncateAllTables(prisma); + for (const user of [LIVE, ROLLUP]) { + await prisma.user.create({ + data: { id: user, username: user, timezone: "UTC" }, + }); + invalidateUserTimezone(user); + await prisma.measurement.createMany({ + data: [...rows(user, "PULSE"), ...rows(user, "HEART_RATE_VARIABILITY")], + }); + } + // The rollup user's DAY tier covers the last 400 days; the six-year-old + // workout day stays outside it, in the pre-fold remainder. + await recomputeUserRollups(ROLLUP, { + types: ["PULSE", "HEART_RATE_VARIABILITY"], + from: new Date(T0 - 400 * DAY), + to: new Date(T0 + DAY), + }); +}, 120_000); + +describe.each([ + { path: "rollup", user: ROLLUP }, + { path: "live", user: LIVE }, +])("dashboard summaries slice, $path path", ({ user }) => { + it("weighs each day once for pulse", async () => { + const { summaries } = await computeSummariesSlice(user); + const pulse = summaries.PULSE; + expect(pulse.avg7).toBeCloseTo(HOURLY.window, 2); + expect(pulse.avg30).toBeCloseTo(HOURLY.window, 2); + expect(pulse.avg30LastMonth).toBeCloseTo(HOURLY.window, 2); + expect(pulse.mean).toBeCloseTo(HOURLY.allTime, 2); + // Count, min and max stay over every reading. + expect(pulse.count).toBe(47); + expect(pulse.min).toBe(60); + expect(pulse.max).toBe(150); + }); + + it("keeps the plain mean for HRV", async () => { + const { summaries } = await computeSummariesSlice(user); + const hrv = summaries.HEART_RATE_VARIABILITY; + expect(hrv.avg7).toBeCloseTo(PLAIN.window, 2); + expect(hrv.avg30).toBeCloseTo(PLAIN.window, 2); + expect(hrv.avg30LastMonth).toBeCloseTo(PLAIN.window, 2); + expect(hrv.mean).toBeCloseTo(PLAIN.allTime, 2); + }); +}); + +describe.each([ + { path: "rollup", user: ROLLUP }, + { path: "live", user: LIVE }, +])("insights aggregate, $path path", ({ user }) => { + it("weighs each day once for pulse and keeps the plain mean for HRV", async () => { + const { summaries } = await buildComprehensiveAggregate(user); + expect(summaries.PULSE.avg7).toBeCloseTo(HOURLY.window, 2); + expect(summaries.PULSE.avg30).toBeCloseTo(HOURLY.window, 2); + expect(summaries.PULSE.avg30LastMonth).toBeCloseTo(HOURLY.window, 2); + // The 90-day window holds two workout days and two quiet days. + expect(summaries.PULSE.mean).toBeCloseTo(HOURLY.window, 2); + expect(summaries.PULSE.count).toBe(32); + const hrv = summaries.HEART_RATE_VARIABILITY; + expect(hrv.avg7).toBeCloseTo(PLAIN.window, 2); + expect(hrv.avg30LastMonth).toBeCloseTo(PLAIN.window, 2); + expect(hrv.mean).toBeCloseTo(PLAIN.window, 2); + }); +}); + +describe("series route, raw window", () => { + async function stats(kind: string) { + const s = await prisma.session.create({ + data: { userId: LIVE, expiresAt: new Date(Date.now() + 3_600_000) }, + }); + cookieJar.clear(); + cookieJar.set("healthlog_session", s.id); + const { GET } = await import("@/app/api/measurements/series/route"); + const res = await GET( + new NextRequest( + `http://localhost/api/measurements/series?kind=${kind}&days=30`, + ), + ); + expect(res.status).toBe(200); + return ( + (await res.json()) as { + data: { + points: unknown[]; + stats: { mean: number; count: number; min: number; max: number }; + }; + } + ).data; + } + + it("gives pulse the mean of its days on the stats strip", async () => { + const data = await stats("pulse"); + // Raw points: every reading of the last 30 days. + expect(data.points).toHaveLength(16); + expect(data.stats.mean).toBeCloseTo(HOURLY.window, 2); + expect(data.stats.count).toBe(16); + expect(data.stats.min).toBe(60); + expect(data.stats.max).toBe(150); + }); + + it("keeps the plain mean for HRV", async () => { + const data = await stats("heartRateVariability"); + expect(data.stats.mean).toBeCloseTo(PLAIN.window, 2); + }); +}); From db524ded98c9847d1449c86a34481061a695436c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 11:33:42 +0200 Subject: [PATCH 05/10] Average pulse by day in the doctor report, correlations and baselines The doctor report's pulse average and day points, the correlation channels, the illness correlation, the personal baseline and the Coach's tiered context each folded pulse readings into a day by plain mean. They now take the day as the mean of its local hours' means and a window as the mean of its days, in the zone each reader buckets by. The day aggregates carry the weighted sums for an hourly-mean type, so every caller reads one day value. Every other type is unchanged. --- .../day-bucket-latest-and-count.test.ts | 2 + src/lib/doctor-report/collect.ts | 14 +- src/lib/doctor-report/dense-buckets.ts | 63 ++++++- src/lib/doctor-report/measurement-series.ts | 29 ++- .../correlation-read-pulse-hourly.test.ts | 86 +++++++++ src/lib/illness/correlation-read.ts | 17 +- ...elation-channel-series-rollup-swap.test.ts | 44 +++++ .../correlation-channel-series.test.ts | 33 ++++ .../insights/correlation-channel-series.ts | 25 ++- src/lib/insights/derived/baseline.ts | 7 +- .../__tests__/fake-day-aggregates.ts | 27 ++- src/lib/measurements/day-aggregates.ts | 89 +++++++++ src/lib/rollups/tiered-context.ts | 82 +++++++- .../day-aggregates-pulse-hourly.test.ts | 175 ++++++++++++++++++ tests/integration/dense-read-bounded.test.ts | 28 ++- .../doctor-report-pulse-hourly-mean.test.ts | 116 ++++++++++++ 16 files changed, 819 insertions(+), 18 deletions(-) create mode 100644 src/lib/illness/__tests__/correlation-read-pulse-hourly.test.ts create mode 100644 tests/integration/day-aggregates-pulse-hourly.test.ts create mode 100644 tests/integration/doctor-report-pulse-hourly-mean.test.ts diff --git a/src/lib/doctor-report/__tests__/day-bucket-latest-and-count.test.ts b/src/lib/doctor-report/__tests__/day-bucket-latest-and-count.test.ts index feb000741..b83ac5e05 100644 --- a/src/lib/doctor-report/__tests__/day-bucket-latest-and-count.test.ts +++ b/src/lib/doctor-report/__tests__/day-bucket-latest-and-count.test.ts @@ -45,6 +45,8 @@ function bucket( clinicalTar2Count: 0, clinicalLowRiskSum: null, clinicalHighRiskSum: null, + hourMeanSum: null, + hourCount: null, }; } diff --git a/src/lib/doctor-report/collect.ts b/src/lib/doctor-report/collect.ts index 79aaacad5..0f850b658 100644 --- a/src/lib/doctor-report/collect.ts +++ b/src/lib/doctor-report/collect.ts @@ -78,6 +78,7 @@ import { loadLabResults, } from "./clinical-records"; import { dayKeyAsUtcMidnight } from "@/lib/tz/date-only"; +import { readingsMean } from "@/lib/measurements/day-mean"; const DENSE_REPORT_RAW_WINDOW_DAYS = 90; @@ -446,8 +447,19 @@ export async function collectDoctorReportData( const stats: Record = {}; for (const [type, entries] of Object.entries(byType)) { const values = entries.map((e) => e.value); + // A day of pulse is the mean of its local hours' means and the period + // average the mean of its days (`day-mean.ts`); every other type keeps the + // mean of its readings. The dense types' stats are replaced below. stats[type] = { - avg: values.reduce((a, b) => a + b, 0) / values.length, + avg: + readingsMean( + type, + entries.map((e) => ({ + value: e.value, + measuredAt: new Date(e.measuredAt), + })), + reportTz, + ) ?? 0, ...minMaxOf(values), count: values.length, latest: values[values.length - 1], diff --git a/src/lib/doctor-report/dense-buckets.ts b/src/lib/doctor-report/dense-buckets.ts index bd89d4dc1..8d05fdab9 100644 --- a/src/lib/doctor-report/dense-buckets.ts +++ b/src/lib/doctor-report/dense-buckets.ts @@ -13,6 +13,7 @@ */ import { Prisma } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; +import { isHourlyMeanTypeSql } from "@/lib/measurements/day-mean"; import type { GlucoseContext, MeasurementSource, @@ -44,6 +45,13 @@ export interface DenseMeasurementBucket { clinicalTar2Count: number; clinicalLowRiskSum: number | null; clinicalHighRiskSum: number | null; + /** + * For an hourly-mean type (pulse, see `day-mean.ts`): the sum of the + * bucket's local hours' means and the number of those hours, so the day + * value is their ratio. `null` for every other type. + */ + hourMeanSum: number | null; + hourCount: number | null; } /** @@ -65,6 +73,8 @@ export async function loadDenseMeasurementBuckets(params: { const typeList = Prisma.join( denseTypes.map((type) => Prisma.sql`${type}::"measurement_type"`), ); + // Closed code constant (the hourly-mean type list), spliced as literals. + const hourlyMeanType = Prisma.raw(isHourlyMeanTypeSql('"type"')); return prisma.$queryRaw` WITH localized AS ( SELECT @@ -78,7 +88,11 @@ export async function loadDenseMeasurementBuckets(params: { date_trunc( 'day', (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${reportTz} - ) AS local_day + ) AS local_day, + date_trunc( + 'hour', + (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${reportTz} + ) AS local_hour FROM measurements m WHERE m."user_id" = ${userId} AND m."measured_at" >= ${start} @@ -121,6 +135,43 @@ export async function loadDenseMeasurementBuckets(params: { "measured_at" DESC, "id" DESC ), + -- The hourly-mean day (pulse): each local hour's mean, then their sum + -- and count per bucket. Empty for every other type. + hour_means AS ( + SELECT + local_day, + "type", + "source", + "device_type", + "glucose_context", + AVG("value") AS hour_mean + FROM localized + WHERE ${hourlyMeanType} + GROUP BY + local_day, + local_hour, + "type", + "source", + "device_type", + "glucose_context" + ), + hour_agg AS ( + SELECT + local_day, + "type", + "source", + "device_type", + "glucose_context", + SUM(hour_mean)::double precision AS hour_mean_sum, + COUNT(*)::int AS hour_count + FROM hour_means + GROUP BY + local_day, + "type", + "source", + "device_type", + "glucose_context" + ), grouped AS ( SELECT local_day, @@ -196,7 +247,9 @@ export async function loadDenseMeasurementBuckets(params: { grouped.clinical_tar1_count AS "clinicalTar1Count", grouped.clinical_tar2_count AS "clinicalTar2Count", grouped.clinical_low_risk_sum AS "clinicalLowRiskSum", - grouped.clinical_high_risk_sum AS "clinicalHighRiskSum" + grouped.clinical_high_risk_sum AS "clinicalHighRiskSum", + hour_agg.hour_mean_sum AS "hourMeanSum", + hour_agg.hour_count AS "hourCount" FROM grouped INNER JOIN latest_rows ON latest_rows.local_day = grouped.local_day @@ -204,6 +257,12 @@ export async function loadDenseMeasurementBuckets(params: { AND latest_rows."source" = grouped."source" AND latest_rows."device_type" IS NOT DISTINCT FROM grouped."device_type" AND latest_rows."glucose_context" IS NOT DISTINCT FROM grouped."glucose_context" + LEFT JOIN hour_agg + ON hour_agg.local_day = grouped.local_day + AND hour_agg."type" = grouped."type" + AND hour_agg."source" = grouped."source" + AND hour_agg."device_type" IS NOT DISTINCT FROM grouped."device_type" + AND hour_agg."glucose_context" IS NOT DISTINCT FROM grouped."glucose_context" ORDER BY "bucketStart" ASC, grouped."type" ASC `; diff --git a/src/lib/doctor-report/measurement-series.ts b/src/lib/doctor-report/measurement-series.ts index 1f0d0a6ef..c00d76eb0 100644 --- a/src/lib/doctor-report/measurement-series.ts +++ b/src/lib/doctor-report/measurement-series.ts @@ -10,6 +10,7 @@ */ import { collapseMeasurementsToCanonical } from "@/lib/doctor-report-helpers"; import { glucoseContextBucket } from "@/lib/glucose"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import type { DoctorReportStats } from "@/lib/doctor-report-types"; import type { GlucoseClinicalMetrics } from "@/lib/analytics/glucose-metrics"; import type { DenseMeasurementBucket } from "./dense-buckets"; @@ -61,6 +62,8 @@ export function summariseDenseBuckets( max: number; latestAt: Date; latest: number; + hourMeanSum: number; + hourCount: number; } >(); for (const row of canonical) { @@ -76,11 +79,15 @@ export function summariseDenseBuckets( max: row.maxValue, latestAt: row.latestAt, latest: row.latestValue, + hourMeanSum: row.hourMeanSum ?? 0, + hourCount: row.hourCount ?? 0, }); continue; } current.count += row.count; current.sum += row.sumValue; + current.hourMeanSum += row.hourMeanSum ?? 0; + current.hourCount += row.hourCount ?? 0; current.min = Math.min(current.min, row.minValue); current.max = Math.max(current.max, row.maxValue); if (row.latestAt > current.latestAt) { @@ -109,12 +116,26 @@ export function summariseDenseBuckets( } }; + // For an hourly-mean type (pulse, see `day-mean.ts`) a day is the mean of + // its local hours' means and the period average the mean of its days, so a + // workout hour's dense samples do not outweigh the rest of the day, nor a + // workout day the rest of the period. Count, min and max stay over every + // reading. Every other type keeps the mean of its readings. + const dayMeans = new Map(); for (const row of byTypeDay.values()) { + const value = + usesHourlyMeanDay(row.type) && row.hourCount > 0 + ? row.hourMeanSum / row.hourCount + : row.sum / row.count; (byType[row.type] ??= []).push({ - value: row.sum / row.count, + value, measuredAt: row.bucketStart.toISOString(), }); accumulate(statsAcc, row.type, row); + const dm = dayMeans.get(row.type) ?? { sum: 0, days: 0 }; + dm.sum += value; + dm.days += 1; + dayMeans.set(row.type, dm); } for (const row of canonical) { @@ -137,9 +158,13 @@ export function summariseDenseBuckets( } const stats: DenseMeasurementSummary["stats"] = {}; for (const [key, value] of statsAcc) { + const dm = dayMeans.get(key); stats[key] = { count: value.count, - avg: value.sum / value.count, + avg: + usesHourlyMeanDay(key) && dm !== undefined && dm.days > 0 + ? dm.sum / dm.days + : value.sum / value.count, min: value.min, max: value.max, latest: value.latest, diff --git a/src/lib/illness/__tests__/correlation-read-pulse-hourly.test.ts b/src/lib/illness/__tests__/correlation-read-pulse-hourly.test.ts new file mode 100644 index 000000000..502ea27e0 --- /dev/null +++ b/src/lib/illness/__tests__/correlation-read-pulse-hourly.test.ts @@ -0,0 +1,86 @@ +/** + * The illness read's live per-day pulse is the mean of the day's local hours' + * means, so a workout hour's dense samples do not read as a fever-day pulse + * spike. Other vitals keep the mean of their readings. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const NOW = new Date("2026-01-21T12:00:00Z"); + +const db = vi.hoisted(() => ({ + measurement: { findMany: vi.fn() }, + illnessDayLog: { findMany: vi.fn() }, +})); +const engine = vi.hoisted(() => ({ compute: vi.fn() })); + +vi.mock("@/lib/db", () => ({ prisma: db })); +vi.mock("@/lib/rollups/measurement-coverage", () => ({ + probeRollupCoverage: vi.fn(async () => new Map()), +})); +vi.mock("@/lib/rollups/measurement-read-wmy", () => ({ + readBestGranularityRollups: vi.fn(async () => null), +})); +vi.mock("@/lib/rollups/measurement-read", () => ({ + loadUserSourcePriority: vi.fn(async () => null), +})); +vi.mock("../correlation", async (orig) => ({ + ...(await orig()), + computeIllnessCorrelation: (input: unknown) => { + engine.compute(input); + return { status: "insufficient" }; + }, +})); + +import { computeEpisodeCorrelation } from "../correlation-read"; + +/** A workout hour (six readings at 150) and two resting hours at 60, in Kolkata local time. */ +const DAY_ROWS = [ + ...[0, 5, 10, 15, 20, 25].map((m) => ({ + value: 150, + // 04:45Z + m = 10:15 + m local (one local hour, two UTC hours). + measuredAt: new Date(Date.parse("2026-01-15T04:45:00Z") + m * 60_000), + })), + { value: 60, measuredAt: new Date("2026-01-15T06:30:00Z") }, + { value: 60, measuredAt: new Date("2026-01-15T08:30:00Z") }, +]; + +beforeEach(() => { + vi.clearAllMocks(); + db.illnessDayLog.findMany.mockResolvedValue([]); + db.measurement.findMany.mockImplementation( + async ({ where }: { where: { type: string } }) => + where.type === "PULSE" || where.type === "HEART_RATE_VARIABILITY" + ? DAY_ROWS + : [], + ); +}); + +describe("illness read, live per-day means", () => { + it("makes a pulse day the mean of its local hours' means; HRV keeps the reading mean", async () => { + await computeEpisodeCorrelation( + "u1", + { + id: "ep", + onsetAt: new Date("2026-01-16T08:00:00Z"), + resolvedAt: null, + lifecycle: "ACUTE", + }, + "Asia/Kolkata", + NOW, + ); + const { series } = engine.compute.mock.calls[0][0] as { + series: Array<{ + type: string; + episodeDays: Array<{ day: string; mean: number }>; + episodeDayMax: Array<{ day: string; mean: number }>; + }>; + }; + const pulse = series.find((s) => s.type === "PULSE")!; + // Hours' means 150, 60, 60 → 90 (the readings average 127.5). + expect(pulse.episodeDays).toEqual([{ day: "2026-01-15", mean: 90 }]); + // The day's max stays over every reading. + expect(pulse.episodeDayMax).toEqual([{ day: "2026-01-15", mean: 150 }]); + const hrv = series.find((s) => s.type === "HEART_RATE_VARIABILITY")!; + expect(hrv.episodeDays).toEqual([{ day: "2026-01-15", mean: 127.5 }]); + }); +}); diff --git a/src/lib/illness/correlation-read.ts b/src/lib/illness/correlation-read.ts index 08b5840bf..aa50005d1 100644 --- a/src/lib/illness/correlation-read.ts +++ b/src/lib/illness/correlation-read.ts @@ -30,6 +30,7 @@ import { prisma } from "@/lib/db"; import { readBestGranularityRollups } from "@/lib/rollups/measurement-read-wmy"; import { probeRollupCoverage } from "@/lib/rollups/measurement-coverage"; import { dayKeyForUserTz } from "@/lib/measurements/consolidation-tz"; +import { dayValue } from "@/lib/measurements/day-mean"; import { resolveUserTimezone } from "@/lib/measurements/consolidation-base"; import { reconstructNights } from "@/lib/insights/derived/sleep-score"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; @@ -250,18 +251,24 @@ async function readWindowMeans( select: { value: true, measuredAt: true }, }); if (rows.length === 0) return { points: [], source: "none" }; - const byDay = new Map(); + const byDay = new Map(); for (const row of rows) { const day = dayKeyForUserTz(row.measuredAt, tz); - const acc = byDay.get(day) ?? { sum: 0, count: 0, max: -Infinity }; - acc.sum += row.value; - acc.count += 1; + const acc = byDay.get(day) ?? { rows: [], max: -Infinity }; + acc.rows.push(row); acc.max = Math.max(acc.max, row.value); byDay.set(day, acc); } + // The day's mean is the type's day value (`day-mean.ts`): the mean of its + // local hours' means for pulse, the mean of its readings otherwise. The + // max stays over every reading. const points = [...byDay.entries()] .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - .map(([day, acc]) => ({ day, mean: acc.sum / acc.count, max: acc.max })); + .map(([day, acc]) => ({ + day, + mean: dayValue(type, acc.rows, tz) as number, + max: acc.max, + })); return { points, source: "live" }; } diff --git a/src/lib/insights/__tests__/correlation-channel-series-rollup-swap.test.ts b/src/lib/insights/__tests__/correlation-channel-series-rollup-swap.test.ts index de6d72f18..e5a301014 100644 --- a/src/lib/insights/__tests__/correlation-channel-series-rollup-swap.test.ts +++ b/src/lib/insights/__tests__/correlation-channel-series-rollup-swap.test.ts @@ -239,6 +239,50 @@ describe("rollup ↔ raw parity — multi-source day (source collapse)", () => { }); }); +describe("pulse from the DAY rollup tier", () => { + it("reads each source's stored DAY mean (the hourly-mean day), not its readings' sum over count", async () => { + mockCoverage([["PULSE", true]]); + // The stored DAY mean of pulse is the mean of the day's hours' means (90), + // while its readings average 127.5. A second source on another day is + // single-source and kept as is. + rollupFindMany.mockResolvedValue([ + { + type: "PULSE", + bucketStart: new Date(`${utcDayKey(at(8, 7))}T00:00:00.000Z`), + count: 8, + mean: 90, + sumValue: 1020, + }, + ]); + const result = await fetchMeasurementDailySeriesTiered( + USER, + "Europe/Berlin", + SINCE(), + ["PULSE"], + ); + expect(result.rollupTypes).toEqual(["PULSE"]); + expect(result.byType.get("PULSE")).toEqual([ + { day: utcDayKey(at(8, 7)), value: 90 }, + ]); + }); + + it("weighs each source's day once on a day two sources report", async () => { + mockCoverage([["PULSE", true]]); + const bucketStart = new Date(`${utcDayKey(at(8, 7))}T00:00:00.000Z`); + rollupFindMany.mockResolvedValue([ + { type: "PULSE", bucketStart, count: 900, mean: 70, sumValue: 900 * 120 }, + { type: "PULSE", bucketStart, count: 3, mean: 60, sumValue: 180 }, + ]); + const result = await fetchMeasurementDailySeriesTiered( + USER, + "Europe/Berlin", + SINCE(), + ["PULSE"], + ); + expect(result.byType.get("PULSE")![0].value).toBeCloseTo(65, 10); + }); +}); + describe("rollup ↔ raw parity — far-from-UTC profile timezone", () => { const savedTz = process.env.TZ; afterAll(() => { diff --git a/src/lib/insights/__tests__/correlation-channel-series.test.ts b/src/lib/insights/__tests__/correlation-channel-series.test.ts index 14a1dce74..e2026f620 100644 --- a/src/lib/insights/__tests__/correlation-channel-series.test.ts +++ b/src/lib/insights/__tests__/correlation-channel-series.test.ts @@ -145,6 +145,39 @@ describe("buildMeasurementDailySeries — grain consistency (v1.29.6)", () => { expect(points).toEqual([{ day: "2026-06-04", value: 62 }]); }); + + it("makes a pulse day the mean of its local hours' means, not of its readings", () => { + // A dense workout hour (six readings at 150) and two resting hours at 60. + // The plain mean would be (6 * 150 + 120) / 8 = 127.5; each hour once + // gives (150 + 60 + 60) / 3 = 90. + const rows: MeasurementSeriesRow[] = [ + ...[0, 10, 20, 30, 40, 50].map((m) => + row(`2026-06-04T10:${String(m).padStart(2, "0")}:00.000Z`, 150), + ), + row("2026-06-04T12:00:00.000Z", 60), + row("2026-06-04T14:00:00.000Z", 60), + ]; + expect(buildMeasurementDailySeries("PULSE", rows, "UTC", null)).toEqual([ + { day: "2026-06-04", value: 90 }, + ]); + // HRV keeps the plain mean of its readings. + expect( + buildMeasurementDailySeries("HEART_RATE_VARIABILITY", rows, "UTC", null), + ).toEqual([{ day: "2026-06-04", value: 127.5 }]); + }); + + it("reads the pulse hours on the profile zone's own clock", () => { + // Asia/Kolkata (UTC+05:30): 10:20Z and 10:40Z are 15:50 and 16:10 local, + // two local hours (100 and 40), though one UTC hour. + const rows: MeasurementSeriesRow[] = [ + row("2026-06-04T10:20:00.000Z", 100), + row("2026-06-04T10:25:00.000Z", 100), + row("2026-06-04T10:40:00.000Z", 40), + ]; + expect( + buildMeasurementDailySeries("PULSE", rows, "Asia/Kolkata", null), + ).toEqual([{ day: "2026-06-04", value: 70 }]); + }); }); describe("fetchMeasurementWindowSeries — desc+cap+resort (v1.30.3 QA F1/F2/F3)", () => { diff --git a/src/lib/insights/correlation-channel-series.ts b/src/lib/insights/correlation-channel-series.ts index 55d37773f..ecbca30b3 100644 --- a/src/lib/insights/correlation-channel-series.ts +++ b/src/lib/insights/correlation-channel-series.ts @@ -63,6 +63,8 @@ import type { } from "@/generated/prisma/client"; import { TRACKED_INTAKE_WHERE } from "@/lib/medications/intake-tracking"; import { dateOnlyKey } from "@/lib/tz/date-only"; +import { hourMeanRows } from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { decryptSymptomText } from "@/lib/symptoms/server"; import { MAX_ACTIVE_SYMPTOM_DEFINITIONS, @@ -689,8 +691,15 @@ export function buildMeasurementDailySeries( if (CUMULATIVE_HK_TYPES.has(type)) { return buildCumulativeDailySeries(type, rows, tz, priorityJson); } + // A day of pulse is the mean of its local hours' means (`day-mean.ts`): + // collapse each local hour to its mean first, so the daily mean below + // weighs every hour once. Every other type passes through unchanged. return toDailyMeans( - rows.map((r) => ({ value: r.value, at: r.at })), + hourMeanRows( + type, + rows.map((r) => ({ value: r.value, measuredAt: r.at })), + tz, + ).map((r) => ({ value: r.value, at: r.measuredAt })), tz, ); } @@ -808,6 +817,7 @@ function isRollupSwapEligible(type: MeasurementType): boolean { * mean exactly up to float addition order. */ function composeRollupDailyMeans( + type: MeasurementType, rows: Array<{ bucketStart: Date; count: number; @@ -815,10 +825,14 @@ function composeRollupDailyMeans( sumValue: number | null; }>, ): DailySeriesPoint[] { + // A pulse DAY row's stored `mean` is the mean of its hours' means + // (`day-mean.ts`), which `sumValue / count` would undo; each source's day + // then weighs one. Every other type keeps the count-weighted compose. + const hourly = usesHourlyMeanDay(type); const byDay = new Map(); for (const r of rows) { if (r.count <= 0) continue; - const sum = r.sumValue ?? r.mean * r.count; + const sum = hourly ? r.mean : (r.sumValue ?? r.mean * r.count); if (!Number.isFinite(sum)) continue; // DAY buckets are minted at UTC midnight; the UTC date IS the bucket's // day key (the same convention graded-series and the derived baselines @@ -826,7 +840,7 @@ function composeRollupDailyMeans( const day = dateOnlyKey(r.bucketStart); const acc = byDay.get(day) ?? { sum: 0, count: 0 }; acc.sum += sum; - acc.count += r.count; + acc.count += hourly ? 1 : r.count; byDay.set(day, acc); } return [...byDay.entries()] @@ -920,7 +934,10 @@ export async function fetchMeasurementDailySeriesTiered( rowsByType.set(bucket.type, list); } for (const type of covered) { - const series = composeRollupDailyMeans(rowsByType.get(type) ?? []); + const series = composeRollupDailyMeans( + type, + rowsByType.get(type) ?? [], + ); if (series.length === 0) { // Coverage said "has buckets" but the window resolved empty — // backfill pending or all data older than the window. Fall back diff --git a/src/lib/insights/derived/baseline.ts b/src/lib/insights/derived/baseline.ts index c267e44c4..a53f5675b 100644 --- a/src/lib/insights/derived/baseline.ts +++ b/src/lib/insights/derived/baseline.ts @@ -282,7 +282,12 @@ export async function readDayMeanSeries( timeZone: timeZone ?? "UTC", valueRange: plausibleMetricRange(type), }); - const points = days.map((d) => ({ day: d.day, mean: d.sum / d.n })); + // A pulse day is the mean of its local hours' means (`dayMean`, see + // `day-mean.ts`); every other type's day is the mean of its readings. + const points = days.map((d) => ({ + day: d.day, + mean: d.dayMean ?? d.sum / d.n, + })); if (points.length === 0) { return { points: [], source: "none" }; } diff --git a/src/lib/measurements/__tests__/fake-day-aggregates.ts b/src/lib/measurements/__tests__/fake-day-aggregates.ts index 096727fb9..dd17f37ef 100644 --- a/src/lib/measurements/__tests__/fake-day-aggregates.ts +++ b/src/lib/measurements/__tests__/fake-day-aggregates.ts @@ -19,6 +19,8 @@ import type { SourceDayAggregateRow, } from "@/lib/measurements/day-aggregates"; import { userDayKey } from "@/lib/tz/format"; +import { dayValue } from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; export function foldDayAggregates( rows: ReadonlyArray<{ measuredAt: Date; value: number }>, @@ -74,7 +76,30 @@ export async function fakeReadDayAggregates( orderBy: { measuredAt: "asc" }, select: { measuredAt: true, value: true }, })) ?? []) as Array<{ measuredAt: Date; value: number }>; - return foldDayAggregates(rows, opts); + const days = foldDayAggregates(rows, opts); + if (!usesHourlyMeanDay(opts.type)) return days; + // Mirror the SQL's hourly-mean day: the mean of the (day, segment)'s local + // hours' means. + const starts = opts.segmentStarts ?? []; + return days.map((d) => { + const members = rows.filter((r) => { + const t = r.measuredAt.getTime(); + if (t < opts.since.getTime()) return false; + if (opts.until && t > opts.until.getTime()) return false; + if ( + opts.valueRange && + (r.value < opts.valueRange.min || r.value > opts.valueRange.max) + ) { + return false; + } + return ( + userDayKey(r.measuredAt, opts.timeZone) === d.day && + starts.filter((s) => t < s.getTime()).length === d.segment + ); + }); + const dayMean = dayValue(opts.type, members, opts.timeZone); + return dayMean === null ? d : { ...d, dayMean }; + }); } /** diff --git a/src/lib/measurements/day-aggregates.ts b/src/lib/measurements/day-aggregates.ts index 48d2966b1..d29b71cfa 100644 --- a/src/lib/measurements/day-aggregates.ts +++ b/src/lib/measurements/day-aggregates.ts @@ -27,6 +27,8 @@ import type { import { Prisma } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import { isValidTimezone } from "@/lib/tz/format"; +import { dayWeightedRowsSql } from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; export interface DayAggregateRow { /** Calendar day in the requested zone, `YYYY-MM-DD`. */ @@ -37,6 +39,21 @@ export interface DayAggregateRow { sum: number; min: number; max: number; + /** + * For a type whose day is the mean of its hours' means (pulse, see + * `day-mean.ts`): `SUM(value * day_weight)` and `SUM(day_weight)` over the + * row's readings. A whole day's weights add up to one, so `weightedSum / + * weightSum` is the day value and the same ratio over several rows the mean + * of their days. Absent for every other type. + */ + weightedSum?: number; + weightSum?: number; + /** + * The row's value for an hourly-mean type, `weightedSum / weightSum`: the + * mean of its local hours' means. Absent for every other type, whose value + * stays `sum / n`. Read it as `dayMean ?? sum / n`. + */ + dayMean?: number; } export interface ReadDayAggregatesOptions { @@ -78,6 +95,10 @@ export async function readDayAggregates( " + ", )})`; + if (usesHourlyMeanDay(opts.type)) { + return readHourlyMeanDayAggregates(opts, timeZone, until, range, segment); + } + // The day and segment are computed once per row in `src` and grouped by // name, so the bound zone / boundary parameters never have to match // themselves across SELECT and GROUP BY. @@ -125,6 +146,74 @@ export async function readDayAggregates( })); } +/** + * `readDayAggregates` for an hourly-mean type: the same rows, plus the day + * weights of `day-mean.ts` cut in the same zone, so a day's hours count once + * each however densely they were sampled. + */ +async function readHourlyMeanDayAggregates( + opts: ReadDayAggregatesOptions, + timeZone: string, + until: Prisma.Sql, + range: Prisma.Sql, + segment: Prisma.Sql, +): Promise { + const rows = await (opts.db ?? prisma).$queryRaw< + Array<{ + day: string; + segment: number; + n: number; + sum: number; + min: number; + max: number; + wsum: number; + wdays: number; + }> + >` + WITH base AS ( + SELECT m.* + FROM measurements m + WHERE m."user_id" = ${opts.userId} + AND m."type" = ${opts.type}::measurement_type + AND m."deleted_at" IS NULL + AND m."measured_at" >= ${opts.since} + ${until} + ${range} + ), + src AS ( + SELECT + to_char((m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}, 'YYYY-MM-DD') AS day, + ${segment} AS segment, + m."value" AS value, + m.day_weight AS day_weight + FROM ${dayWeightedRowsSql("base", timeZone)} m + ) + SELECT + day, + segment::int AS segment, + COUNT(*)::int AS n, + SUM(value)::double precision AS sum, + MIN(value)::double precision AS min, + MAX(value)::double precision AS max, + SUM(value * day_weight)::double precision AS wsum, + SUM(day_weight)::double precision AS wdays + FROM src + GROUP BY day, segment + ORDER BY day ASC, segment DESC + `; + return rows.map((r) => ({ + day: r.day, + segment: Number(r.segment), + n: Number(r.n), + sum: Number(r.sum), + min: Number(r.min), + max: Number(r.max), + weightedSum: Number(r.wsum), + weightSum: Number(r.wdays), + dayMean: Number(r.wsum) / Number(r.wdays), + })); +} + export interface SourceDayAggregateRow { type: MeasurementType; /** Calendar day in the requested zone, `YYYY-MM-DD`. */ diff --git a/src/lib/rollups/tiered-context.ts b/src/lib/rollups/tiered-context.ts index a5b29605f..d9d815c2e 100644 --- a/src/lib/rollups/tiered-context.ts +++ b/src/lib/rollups/tiered-context.ts @@ -40,6 +40,9 @@ import { } from "./measurement-rollups"; import { probeRollupCoverage } from "./measurement-coverage"; import { readDayAggregates } from "@/lib/measurements/day-aggregates"; +import { dayWeightedRowsSql, windowMeanSql } from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; +import { Prisma } from "@/generated/prisma/client"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -270,9 +273,11 @@ async function readRecentDaily( since, timeZone: tz, }); + // A pulse day is the mean of its local hours' means (`dayMean`, see + // `day-mean.ts`); every other type's day is the mean of its readings. return days.map((d) => ({ date: d.day, - value: round(d.sum / d.n), + value: round(d.dayMean ?? d.sum / d.n), count: d.n, })); } @@ -316,6 +321,9 @@ async function readBandLive( tzKey?: string, ): Promise { const unit = BAND_TRUNC_UNIT[granularity]; + if (usesHourlyMeanDay(type)) { + return readHourlyMeanBandLive(userId, type, unit, from, to, tzKey); + } try { const rows = tzKey ? await prisma.$queryRaw< @@ -387,6 +395,78 @@ async function readBandLive( } } +/** + * `readBandLive` for a type whose day is the mean of its hours' means (pulse, + * `day-mean.ts`): a bucket's mean is the mean of its days, each the mean of + * its hours, so a workout's dense samples do not outweigh the rest of the + * bucket. Count, min, max and the spread stay over every reading. The bucket + * boundary is the one the plain branch uses; hours and days are cut in the + * user's zone when `tzKey` is set, in the session's (UTC) frame otherwise. + */ +async function readHourlyMeanBandLive( + userId: string, + type: MeasurementType, + unit: "day" | "week" | "month" | "year", + from: Date, + to: Date, + tzKey?: string, +): Promise { + const mean = Prisma.raw( + windowMeanSql({ + typeColumn: 'm."type"', + value: 'm."value"', + weight: "m.day_weight", + }), + ); + const bucket = tzKey + ? Prisma.sql`date_trunc(${unit}, m."measured_at" AT TIME ZONE ${tzKey})` + : Prisma.sql`date_trunc(${unit}, m."measured_at")`; + try { + const rows = await prisma.$queryRaw< + Array<{ + bucket_start: Date; + count: number; + mean: number; + min_value: number; + max_value: number; + sd: number | null; + }> + >` + WITH src AS ( + SELECT * FROM measurements m + WHERE m."user_id" = ${userId} + AND m."type" = ${type}::"measurement_type" + AND m."deleted_at" IS NULL + AND m."measured_at" >= ${from} + AND m."measured_at" < ${to} + ) + SELECT + ${bucket} AS bucket_start, + COUNT(*)::int AS count, + ${mean}::double precision AS mean, + MIN(m."value")::double precision AS min_value, + MAX(m."value")::double precision AS max_value, + STDDEV_POP(m."value")::double precision AS sd + FROM ${dayWeightedRowsSql("src", tzKey ?? null)} m + GROUP BY 1, m."type" + ORDER BY 1 + `; + return rows.map((r) => ({ + bucketStart: new Date(r.bucket_start), + count: Number(r.count), + mean: Number(r.mean), + minValue: Number(r.min_value), + maxValue: Number(r.max_value), + sd: r.sd === null ? null : Number(r.sd), + slope: null, + r2: null, + })); + } catch { + // Degrades to empty, like the plain branch. + return []; + } +} + async function readBand( userId: string, type: MeasurementType, diff --git a/tests/integration/day-aggregates-pulse-hourly.test.ts b/tests/integration/day-aggregates-pulse-hourly.test.ts new file mode 100644 index 000000000..238dcaeaf --- /dev/null +++ b/tests/integration/day-aggregates-pulse-hourly.test.ts @@ -0,0 +1,175 @@ +/** + * The per-day fold the background readers share (`readDayAggregates`) carries + * the hourly-mean day for pulse, cut on the requested zone's own hours and + * days; the vitals baseline and the tiered context read their live pulse days + * from it. + * + * Asia/Kolkata (UTC+05:30). Day one: six readings at 150 from 04:45Z to + * 04:55Z and two at 150 at 05:05Z and 05:10Z (all local 10:15 to 10:40, one + * local hour, two UTC hours), then one reading at 60 in each of local hours + * 12, 14 and 16. Hours' means 150, 60, 60, 60, day value 82.5; the readings + * average (8 x 150 + 180) / 11 = 125.45. Day two: one reading at 60. + * + * A second workout day ten days earlier (12:45Z to 13:10Z, local 18:15 to + * 18:40, then one reading at 60 at 14:00Z, 15:00Z and 16:00Z) sits in the + * tiered context's day band, which a zone this far from UTC reads live. + */ +import { beforeAll, describe, expect, it } from "vitest"; + +import type { Prisma } from "@/generated/prisma/client"; +import { readDayAggregates } from "@/lib/measurements/day-aggregates"; +import { readDayMeanSeries } from "@/lib/insights/derived/baseline"; +import { buildTieredSeries } from "@/lib/rollups/tiered-context"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const prisma = getPrismaClient(); +const USER = "day-aggregates-pulse-hourly"; +const TZ = "Asia/Kolkata"; +const now = new Date(); +const D = Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() - 10, +); +const at = (h: number, m: number) => new Date(D + h * 3_600_000 + m * 60_000); +const dayKey = (ms: number) => + new Intl.DateTimeFormat("en-CA", { timeZone: TZ }).format(new Date(ms)); + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, timezone: TZ }, + }); + invalidateUserTimezone(USER); + const readings: Array<[Date, number]> = [ + ...[45, 47, 49, 51, 53, 55].map((m): [Date, number] => [at(4, m), 150]), + [at(5, 5), 150], + [at(5, 10), 150], + [at(6, 30), 60], + [at(8, 30), 60], + [at(10, 30), 60], + [at(24 + 4, 0), 60], + ...[45, 47, 49, 51, 53, 55, 65, 70].map((m): [Date, number] => [ + at(-10 * 24 + 12, m), + 150, + ]), + [at(-10 * 24 + 14, 0), 60], + [at(-10 * 24 + 15, 0), 60], + [at(-10 * 24 + 16, 0), 60], + ]; + const rows: Prisma.MeasurementCreateManyInput[] = []; + for (const type of ["PULSE", "HEART_RATE_VARIABILITY"] as const) { + readings.forEach(([measuredAt, value], i) => + rows.push({ + id: `${type}-${i}`, + userId: USER, + type, + unit: type === "PULSE" ? "bpm" : "ms", + source: "APPLE_HEALTH", + value, + measuredAt, + }), + ); + } + await prisma.measurement.createMany({ data: rows }); +}, 120_000); + +describe("readDayAggregates", () => { + it("carries the hourly-mean day for pulse, on the zone's own hours", async () => { + const days = await readDayAggregates({ + userId: USER, + type: "PULSE", + since: new Date(D - 86_400_000), + timeZone: TZ, + }); + expect(days.map((d) => d.day)).toEqual([ + dayKey(D + 5 * 3_600_000), + dayKey(D + 28 * 3_600_000), + ]); + expect(days[0]).toMatchObject({ n: 11, min: 60, max: 150 }); + expect(days[0].sum).toBe(8 * 150 + 3 * 60); + expect(days[0].dayMean).toBeCloseTo(82.5, 9); + expect(days[1].dayMean).toBeCloseTo(60, 9); + }); + + it("splits the hours per segment", async () => { + const days = await readDayAggregates({ + userId: USER, + type: "PULSE", + since: new Date(D - 86_400_000), + timeZone: TZ, + // Readings before 08:00Z (the workout and local hour 12) fall into + // segment 1, the rest of the day into segment 0. + segmentStarts: [at(8, 0)], + }); + const first = days.filter((d) => d.day === dayKey(D + 5 * 3_600_000)); + expect(first.map((d) => [d.segment, d.dayMean])).toEqual([ + [1, expect.closeTo(105, 9)], + [0, expect.closeTo(60, 9)], + ]); + }); + + it("leaves every other type without a day mean", async () => { + const days = await readDayAggregates({ + userId: USER, + type: "HEART_RATE_VARIABILITY", + since: new Date(D - 86_400_000), + timeZone: TZ, + }); + expect(days[0].dayMean ?? null).toBeNull(); + }); +}); + +describe("live pulse days of the readers built on it", () => { + it("the vitals baseline", async () => { + const { points, source } = await readDayMeanSeries( + USER, + "PULSE", + 30, + now, + new Map(), + TZ, + ); + expect(source).toBe("live"); + expect(points.map((p) => p.mean)).toEqual([ + expect.closeTo(82.5, 9), + expect.closeTo(82.5, 9), + expect.closeTo(60, 9), + ]); + const hrv = await readDayMeanSeries( + USER, + "HEART_RATE_VARIABILITY", + 30, + now, + new Map(), + TZ, + ); + expect(hrv.points[0].mean).toBeCloseTo((8 * 150 + 3 * 60) / 11, 9); + }); + + it("the tiered context's recent days and its live day band", async () => { + const series = await buildTieredSeries(USER, "PULSE", { + now: now.getTime(), + tz: TZ, + skipEnsureFresh: true, + }); + expect(series.recentDaily.map((p) => [p.value, p.count])).toEqual([ + [82.5, 11], + [60, 1], + ]); + const workout = series.dayBand.find((b) => b.count === 11); + expect(workout?.mean).toBeCloseTo(82.5, 2); + const hrv = await buildTieredSeries(USER, "HEART_RATE_VARIABILITY", { + now: now.getTime(), + tz: TZ, + skipEnsureFresh: true, + }); + expect(hrv.recentDaily[0].value).toBeCloseTo((8 * 150 + 180) / 11, 2); + expect(hrv.dayBand.find((b) => b.count === 11)?.mean).toBeCloseTo( + (8 * 150 + 180) / 11, + 2, + ); + }); +}); diff --git a/tests/integration/dense-read-bounded.test.ts b/tests/integration/dense-read-bounded.test.ts index 6357877b7..8fefb1505 100644 --- a/tests/integration/dense-read-bounded.test.ts +++ b/tests/integration/dense-read-bounded.test.ts @@ -29,6 +29,8 @@ import { buildGradedSeriesWithRollups } from "@/lib/insights/graded-series"; import { readDayAggregates } from "@/lib/measurements/day-aggregates"; import { foldDayAggregates } from "@/lib/measurements/__tests__/fake-day-aggregates"; import { localHmAsUtc } from "@/lib/tz/local-day"; +import { dayValue } from "@/lib/measurements/day-mean"; +import { userDayKey } from "@/lib/tz/format"; vi.mock("@/lib/db-compat", () => ({ ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), @@ -202,7 +204,31 @@ describe("pulse status over a dense stream (#1023)", () => { rows.map((r) => ({ measuredAt: r.at, value: r.value })), opts, ); - expect(sql).toEqual(memory); + // Pulse rows also carry the hourly-mean day (`day-mean.ts`); the plain + // fold fields agree with the in-memory fold, and each row's day value is + // the mean of its local hours' means over the same readings. + expect( + sql.map( + ({ dayMean: _d, weightedSum: _w, weightSum: _s, ...plain }) => plain, + ), + ).toEqual(memory); + for (const row of sql) { + const members = rows.filter( + (r) => + r.value >= 20 && + r.value <= 300 && + userDayKey(r.at, tz) === row.day && + opts.segmentStarts.filter((s) => r.at < s).length === row.segment, + ); + expect(row.dayMean).toBeCloseTo( + dayValue( + "PULSE", + members.map((r) => ({ value: r.value, measuredAt: r.at })), + tz, + )!, + 9, + ); + } // Sanity: the impossible value was dropped and the edge row split off. expect(sql.reduce((n, r) => n + r.n, 0)).toBe(5); }); diff --git a/tests/integration/doctor-report-pulse-hourly-mean.test.ts b/tests/integration/doctor-report-pulse-hourly-mean.test.ts new file mode 100644 index 000000000..050a10dd3 --- /dev/null +++ b/tests/integration/doctor-report-pulse-hourly-mean.test.ts @@ -0,0 +1,116 @@ +/** + * The doctor report's pulse figures use the hourly-mean day: a day is the mean + * of its local hours' means and the period average is the mean of its days, + * on both the raw path (a short window) and the day-bucket path (a long one). + * + * The account sits in Asia/Kolkata (UTC+05:30), so a local hour starts at half + * past a UTC hour. The workout runs 04:45Z to 05:14:30Z: one local hour + * (10:15 to 10:44:30), two UTC hours. + * + * day one 60 readings at 150 in local 10:xx, one at 60 in local 12, 14, 16 + * hours' means 150, 60, 60, 60, day value 82.5 + * (plain mean 145.7; UTC hours would give 96) + * day two one reading at 60, day value 60 + * period (82.5 + 60) / 2 = 71.25 (plain mean over readings 144.375) + * + * Count, min and max stay over every reading. + */ +import { beforeAll, describe, expect, it } from "vitest"; + +import type { Prisma } from "@/generated/prisma/client"; +import { collectDoctorReportData } from "@/lib/doctor-report-data"; +import { selectionFromLeaves } from "@/lib/report-selection/selection"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const MODULES = { + cycle: false, + mood: false, + sleep: false, + glucose: false, + workouts: false, + recovery: false, + labs: false, + illness: false, + achievements: true, + coach: true, + insights: true, + medications: true, + doctorReport: true, + environment: true, + mcp: true, + inboundDocuments: true, + mentalHealth: true, + nutrients: true, + vaccinations: true, +} as const; + +const SELECTION = selectionFromLeaves(["PULSE"]); +const END = new Date("2026-05-01T00:00:00.000Z"); +const LONG = { + start: new Date("2026-01-01T00:00:00.000Z"), + end: END, + days: 120, +}; +const SHORT = { + start: new Date("2026-02-15T00:00:00.000Z"), + end: END, + days: 75, +}; + +let userId = ""; + +beforeAll(async () => { + const prisma = getPrismaClient(); + await truncateAllTables(prisma); + const user = await prisma.user.create({ + data: { + username: "doctor-report-pulse-hourly", + email: "doctor-report-pulse-hourly@example.test", + timezone: "Asia/Kolkata", + }, + }); + userId = user.id; + const rows: Prisma.MeasurementCreateManyInput[] = []; + const add = (iso: string, value: number) => + rows.push({ + userId, + type: "PULSE", + value, + unit: "bpm", + source: "APPLE_HEALTH", + measuredAt: new Date(iso), + }); + const workoutStart = Date.parse("2026-03-10T04:45:00.000Z"); + for (let i = 0; i < 60; i += 1) { + add(new Date(workoutStart + i * 30_000).toISOString(), 150); + } + add("2026-03-10T06:30:00.000Z", 60); + add("2026-03-10T08:30:00.000Z", 60); + add("2026-03-10T10:30:00.000Z", 60); + add("2026-03-11T03:30:00.000Z", 60); + await prisma.measurement.createMany({ data: rows }); +}, 120_000); + +describe.each([ + ["day-bucket path (long window)", LONG], + ["raw path (short window)", SHORT], +] as const)("doctor report pulse, %s", (_label, range) => { + it("averages the days, each the mean of its local hours", async () => { + const data = await collectDoctorReportData(userId, range, SELECTION, { + moduleMap: { ...MODULES }, + }); + expect(data.stats.PULSE).toMatchObject({ count: 64, min: 60, max: 150 }); + expect(data.stats.PULSE.avg).toBeCloseTo(71.25, 9); + }); +}); + +it("draws the long window's day points as hourly-mean days", async () => { + const data = await collectDoctorReportData(userId, LONG, SELECTION, { + moduleMap: { ...MODULES }, + }); + expect(data.measurements.PULSE.map((p) => p.value)).toEqual([ + expect.closeTo(82.5, 9), + expect.closeTo(60, 9), + ]); +}); From 11e35a6ba15a07ee6a72e5796547004b60b41cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 11:33:45 +0200 Subject: [PATCH 06/10] Average pulse by day in the insights, Coach and report trend The feature extraction, the signals of the day, the status cards, the graded series, the Coach snapshot, the mood correlations and the report's trend arrow each still averaged pulse over its readings, so a dense workout hour decided the day and a workout day the window. They now read a pulse day as the mean of its hours' means and a window as the mean of its days, through the day-mean helper. Counts, extremes and spreads stay over every reading; every other type is unchanged. --- .../__tests__/core-metrics-pulse-day.test.ts | 68 +++++++ .../snapshot-blocks/core-metrics-block.ts | 10 +- src/lib/ai/coach/snapshot-series.ts | 55 +++++- .../clinical-summary-pulse-trend.test.ts | 60 +++++++ src/lib/doctor-report-pdf/clinical-summary.ts | 18 +- src/lib/insights/__tests__/features.test.ts | 58 ++++++ .../insights/__tests__/general-status.test.ts | 49 +++++ .../__tests__/mood-status-pulse-day.test.ts | 37 ++++ .../__tests__/pulse-hourly-mean-folds.test.ts | 167 ++++++++++++++++++ .../insights/__tests__/pulse-status.test.ts | 49 +++++ .../__tests__/signals-of-day-pulse.test.ts | 58 ++++++ src/lib/insights/bucket-series.ts | 45 ++++- src/lib/insights/feature-blocks.ts | 44 ++++- src/lib/insights/features.ts | 43 ++++- src/lib/insights/general-status.ts | 6 +- src/lib/insights/graded-series.ts | 78 ++++++-- src/lib/insights/mood-status.ts | 56 ++++-- src/lib/insights/pulse-status.ts | 7 +- src/lib/insights/signals-of-day.ts | 21 ++- .../feature-blocks-pulse-day-mean.test.ts | 75 ++++++++ .../graded-series-pulse-day-mean.test.ts | 117 ++++++++++++ 21 files changed, 1060 insertions(+), 61 deletions(-) create mode 100644 src/lib/ai/coach/snapshot-blocks/__tests__/core-metrics-pulse-day.test.ts create mode 100644 src/lib/doctor-report-pdf/__tests__/clinical-summary-pulse-trend.test.ts create mode 100644 src/lib/insights/__tests__/mood-status-pulse-day.test.ts create mode 100644 src/lib/insights/__tests__/pulse-hourly-mean-folds.test.ts create mode 100644 src/lib/insights/__tests__/signals-of-day-pulse.test.ts create mode 100644 tests/integration/feature-blocks-pulse-day-mean.test.ts create mode 100644 tests/integration/graded-series-pulse-day-mean.test.ts diff --git a/src/lib/ai/coach/snapshot-blocks/__tests__/core-metrics-pulse-day.test.ts b/src/lib/ai/coach/snapshot-blocks/__tests__/core-metrics-pulse-day.test.ts new file mode 100644 index 000000000..d901a64b7 --- /dev/null +++ b/src/lib/ai/coach/snapshot-blocks/__tests__/core-metrics-pulse-day.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; + +import { buildCoreMetricsBlocks } from "../core-metrics-block"; + +const HOUR = 3_600_000; +const DAY = 24 * HOUR; + +/** + * The Coach's pulse timeline: a day is the mean of its hours' means and a week + * the mean of its days (`day-mean.ts`). A workout day of twelve readings at + * 150 in one hour and three resting hours at 60 reads 82.5, not 132. + */ +describe("Coach pulse block", () => { + it("states a pulse day as the mean of its hours, a week as its days", () => { + const first = Date.UTC(2026, 2, 4); + const rows: Array<{ type: string; value: number; measuredAt: Date }> = []; + const day = (start: number) => { + for (let i = 0; i < 12; i += 1) { + rows.push({ + type: "PULSE", + value: 150, + measuredAt: new Date(start + 10 * HOUR + i * 300_000), + }); + } + for (const h of [12, 14, 16]) { + rows.push({ + type: "PULSE", + value: 60, + measuredAt: new Date(start + h * HOUR), + }); + } + rows.push({ + type: "PULSE", + value: 60, + measuredAt: new Date(start + DAY + 8 * HOUR), + }); + }; + day(first - 14 * DAY); // older: lands in the weekly fold + day(first); // recent + const snapshot: Record = {}; + buildCoreMetricsBlocks({ + sources: new Set(["pulse"]), + features: { pulse: { coverage: { count: rows.length } } } as never, + measurementRows: rows, + moodRows: null, + recentCutoff: new Date(first), + userTz: "UTC", + coarseTails: {}, + snapshot, + windows: new Set(), + metrics: new Set(), + counts: {}, + registerBlock: () => {}, + groundingValues: new Map(), + units: {} as never, + }); + const pulse = snapshot.pulse as { + timeline: { + recent: Array<{ value: number }>; + weekly: Array<{ mean: number; count: number }>; + }; + }; + expect(pulse.timeline.recent.map((d) => d.value)).toEqual([82.5, 60]); + expect(pulse.timeline.weekly).toHaveLength(1); + expect(pulse.timeline.weekly[0].mean).toBe(71.3); + expect(pulse.timeline.weekly[0].count).toBe(16); + }); +}); diff --git a/src/lib/ai/coach/snapshot-blocks/core-metrics-block.ts b/src/lib/ai/coach/snapshot-blocks/core-metrics-block.ts index eda25227a..d5586ea8a 100644 --- a/src/lib/ai/coach/snapshot-blocks/core-metrics-block.ts +++ b/src/lib/ai/coach/snapshot-blocks/core-metrics-block.ts @@ -133,13 +133,21 @@ export function buildCoreMetricsBlocks( if (ctx.sources.has("pulse") && ctx.features.pulse) { const rows = byType("PULSE"); + // A pulse day is the mean of its hours' means, a week the mean of its + // days (`day-mean.ts`). ctx.snapshot.pulse = { aggregate: ctx.features.pulse, timeline: { - recent: buildDailyValueRows(rows, ctx.recentCutoff, ctx.userTz), + recent: buildDailyValueRows( + rows, + ctx.recentCutoff, + ctx.userTz, + "PULSE", + ), weekly: bucketWeekly( rows.filter((row) => row.measuredAt < ctx.recentCutoff), ctx.userTz, + "PULSE", ), ...(ctx.coarseTails.pulse ? { coarse: ctx.coarseTails.pulse } : {}), }, diff --git a/src/lib/ai/coach/snapshot-series.ts b/src/lib/ai/coach/snapshot-series.ts index f06fc3328..a62f14f88 100644 --- a/src/lib/ai/coach/snapshot-series.ts +++ b/src/lib/ai/coach/snapshot-series.ts @@ -8,6 +8,8 @@ * is one-way (snapshot → series), never back. */ import { userDayKey } from "@/lib/tz/format"; +import { dayValue, readingsMean } from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { applyDisplayTransform, type DisplayTransform, @@ -135,11 +137,33 @@ function dailyMeans( return grouped; } +function groupRowsByDay( + rows: Array<{ measuredAt: Date; value: number }>, + tz: string, +): Map> { + const out = new Map>(); + for (const r of rows) { + const key = tzDayKey(r.measuredAt, tz); + const list = out.get(key); + if (list) list.push(r); + else out.set(key, [r]); + } + return out; +} + +/** + * `type`, when given, names the measurement type: for a type whose day is the + * mean of its hours' means (pulse, `day-mean.ts`) a week's mean is the mean of + * its days and a day's value that day value. `count` stays the readings. + */ export function bucketWeekly( rows: Array<{ measuredAt: Date; value: number }>, tz: string, + type?: string, ): WeeklyBucket[] { + const hourly = type !== undefined && usesHourlyMeanDay(type); const grouped = new Map(); + const members = new Map>(); for (const r of rows) { const key = isoWeekKey(r.measuredAt, tz); const list = grouped.get(key); @@ -148,15 +172,23 @@ export function bucketWeekly( } else { grouped.set(key, [r.value]); } + if (hourly) { + const kept = members.get(key); + if (kept) kept.push(r); + else members.set(key, [r]); + } } return Array.from(grouped.entries()) - .map(([weekISO, values]) => ({ - weekISO, - mean: - Math.round((values.reduce((s, v) => s + v, 0) / values.length) * 10) / - 10, - count: values.length, - })) + .map(([weekISO, values]) => { + const mean = hourly + ? (readingsMean(type!, members.get(weekISO) ?? [], tz) ?? 0) + : values.reduce((s, v) => s + v, 0) / values.length; + return { + weekISO, + mean: Math.round(mean * 10) / 10, + count: values.length, + }; + }) .sort((a, b) => a.weekISO.localeCompare(b.weekISO)); } @@ -164,12 +196,19 @@ export function buildDailyValueRows( rows: Array<{ measuredAt: Date; value: number }>, recentCutoff: Date, tz: string, + type?: string, ): DailyValueRow[] { const recent = rows.filter((r) => r.measuredAt >= recentCutoff); const grouped = dailyMeans(recent, tz); + const hourly = type !== undefined && usesHourlyMeanDay(type); + const members = hourly + ? groupRowsByDay(recent, tz) + : new Map>(); return Array.from(grouped.entries()) .map(([date, info]) => { - const mean = info.values.reduce((s, v) => s + v, 0) / info.values.length; + const mean = hourly + ? (dayValue(type!, members.get(date) ?? [], tz) ?? 0) + : info.values.reduce((s, v) => s + v, 0) / info.values.length; return { date, weekday: tzWeekday(info.date, tz), diff --git a/src/lib/doctor-report-pdf/__tests__/clinical-summary-pulse-trend.test.ts b/src/lib/doctor-report-pdf/__tests__/clinical-summary-pulse-trend.test.ts new file mode 100644 index 000000000..4ceda5fbc --- /dev/null +++ b/src/lib/doctor-report-pdf/__tests__/clinical-summary-pulse-trend.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; + +import { buildClinicalSummaryLines } from "../clinical-summary"; +import type { DoctorReportData } from "../../doctor-report-data"; + +/** + * The cover sheet's trend arrow compares the first half of a series with the + * second. On the raw path a pulse series holds every reading, so its halves + * were halves by reading count and a dense workout hour decided the arrow. It + * now compares day values, each the mean of its hours' means. + * + * Day 1: three resting readings at 60. Day 2: a workout hour of twelve + * readings at 150, then three resting hours at 60. By readings the halves are + * equal (120 and 120, "→"); by day the pulse rose from 60 to 82.5 ("↑"). + */ +function series(type: string) { + const at = (d: number, h: number, m = 0) => + new Date(Date.UTC(2026, 2, d, h, m)).toISOString(); + return [ + ...[8, 9, 10].map((h) => ({ value: 60, measuredAt: at(2, h) })), + ...Array.from({ length: 12 }, (_, i) => ({ + value: 150, + measuredAt: at(3, 10, i * 5), + })), + ...[12, 14, 16].map((h) => ({ value: 60, measuredAt: at(3, h) })), + ].map((p) => ({ ...p, type })); +} + +function arrowFor(type: string): string | undefined { + const points = series(type).map(({ value, measuredAt }) => ({ + value, + measuredAt, + })); + const data = { + period: { days: 2, since: "", start: "", end: "" }, + measurements: { [type]: points }, + stats: { + [type]: { avg: 0, min: 60, max: 150, count: points.length, latest: 60 }, + }, + compliance: {}, + } as unknown as DoctorReportData; + const lines = buildClinicalSummaryLines( + data, + (key, vars) => `${key}|${String(vars?.arrow ?? "")}`, + (n) => String(n), + ); + return lines + .find((l) => l.startsWith("doctorReport.summaryTrend")) + ?.split("|")[1]; +} + +describe("clinical summary trend arrow", () => { + it("reads pulse by day, not by reading count", () => { + expect(arrowFor("PULSE")).toBe("↑"); + }); + + it("keeps the reading halves for every other type", () => { + expect(arrowFor("WEIGHT")).toBe("→"); + }); +}); diff --git a/src/lib/doctor-report-pdf/clinical-summary.ts b/src/lib/doctor-report-pdf/clinical-summary.ts index 013847f6a..037a765d5 100644 --- a/src/lib/doctor-report-pdf/clinical-summary.ts +++ b/src/lib/doctor-report-pdf/clinical-summary.ts @@ -8,6 +8,7 @@ import type { DoctorReportNumberFormatter, DoctorReportTranslator, } from "./render-context"; +import { dayValueRows } from "@/lib/measurements/day-mean"; export function getBmiClassificationKey(bmi: number): string { if (bmi < 18.5) return "doctorReport.bmiUnderweight"; @@ -76,7 +77,22 @@ export function buildClinicalSummaryLines( const series = data.measurements[type]; const stat = data.stats[type]; if (!series || series.length === 0 || !stat) continue; - const arrow = trendArrow(series.map((point) => point.value)); + // A pulse series on the raw path holds every reading, so halves of it + // are halves by reading count and a dense workout hour tips the arrow. + // Compare day values instead, each the mean of its hours' means (see + // `day-mean.ts`); the report's day points already are one per day. Days + // are UTC days: the arrow needs only their order. Every other type keeps + // its points as they are. + const arrow = trendArrow( + dayValueRows( + type, + series.map((point) => ({ + value: point.value, + measuredAt: new Date(point.measuredAt), + })), + "UTC", + ).map((point) => point.value), + ); lines.push( t("doctorReport.summaryTrend", { label: t(MEASUREMENT_TYPE_LABEL_KEYS[type as MeasurementType] ?? ""), diff --git a/src/lib/insights/__tests__/features.test.ts b/src/lib/insights/__tests__/features.test.ts index 94ad4694e..f43d56adf 100644 --- a/src/lib/insights/__tests__/features.test.ts +++ b/src/lib/insights/__tests__/features.test.ts @@ -624,3 +624,61 @@ describe("extractFeatures — S10 ECG device-verdict descriptor", () => { expect(call?.select?.rhythmClassification).toBe(true); }); }); + +describe("extractFeatures — a day of pulse is the mean of its hours' means", () => { + /** + * Two days: one with a workout hour of twelve readings at 150 and three + * resting hours of one reading at 60 (day value 82.5), one with a single + * resting reading at 60. Every window weighs each day once: 71.25. The plain + * mean over the seventeen readings would be about 123.5. + */ + function pulseRows() { + const now = new Date(); + const midnight = (daysAgo: number) => + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()) - + daysAgo * dayMs; + const workout = midnight(2); + const rows: Array<{ type: string; value: number; measuredAt: Date }> = []; + for (let i = 0; i < 12; i += 1) { + rows.push({ + type: "PULSE", + value: 150, + measuredAt: new Date(workout + 10 * 3_600_000 + i * 5 * 60_000), + }); + } + for (const h of [12, 14, 16]) { + rows.push({ + type: "PULSE", + value: 60, + measuredAt: new Date(workout + h * 3_600_000), + }); + } + rows.push({ + type: "PULSE", + value: 60, + measuredAt: new Date(midnight(3) + 8 * 3_600_000), + }); + // The bulk read is newest-first; the builder reverses it. + return rows + .map((r) => ({ + ...r, + sleepStage: null, + source: "APPLE_HEALTH", + deviceType: null, + })) + .sort((a, b) => b.measuredAt.getTime() - a.measuredAt.getTime()); + } + + it("weighs each day once in every pulse window and in the all-time mean", async () => { + prismaMock.measurement.findMany.mockResolvedValue(pulseRows()); + const f = await extractFeatures("user-pulse-day", false); + expect(f.pulse?.avg7).toBe(71.25); + expect(f.pulse?.avg30).toBe(71.25); + expect(f.pulse?.avg90).toBe(71.25); + expect(f.pulse?.allTimeAvg).toBe(71.25); + // Extremes stay over every reading. + expect(f.pulse?.allTimeMin).toBe(60); + expect(f.pulse?.allTimeMax).toBe(150); + expect(f.historicalComparison?.pulse?.current7dAvg).toBe(71.25); + }); +}); diff --git a/src/lib/insights/__tests__/general-status.test.ts b/src/lib/insights/__tests__/general-status.test.ts index 5dded1ca4..1c3c74ada 100644 --- a/src/lib/insights/__tests__/general-status.test.ts +++ b/src/lib/insights/__tests__/general-status.test.ts @@ -259,3 +259,52 @@ describe("generateGeneralStatusForUser — token-leak hardening (v1.4.27 F16)", expect(notes[0].text).not.toContain("metric:"); }); }); + +describe("generateGeneralStatusForUser — a pulse day is the mean of its hours", () => { + it("folds pulse days and buckets by hours and days, other types unchanged", async () => { + // Yesterday: a workout hour of twelve readings at 150 and three resting + // hours at 60 (day value 82.5); the day before one reading at 60. The + // plain mean of yesterday's fifteen readings would be 132. + const now = new Date(); + const midnight = (daysAgo: number) => + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()) - + daysAgo * dayMs; + const at = (daysAgo: number, h: number, m = 0) => + new Date(midnight(daysAgo) + h * 3_600_000 + m * 60_000); + const rows: Array<{ type: string; value: number; measuredAt: Date }> = []; + for (const type of ["PULSE", "WEIGHT"]) { + for (let i = 0; i < 12; i++) { + rows.push({ type, value: 150, measuredAt: at(1, 10, i * 5) }); + } + for (const h of [12, 14, 16]) { + rows.push({ type, value: 60, measuredAt: at(1, h) }); + } + rows.push({ type, value: 60, measuredAt: at(2, 8) }); + } + rows.sort((a, b) => b.measuredAt.getTime() - a.measuredAt.getTime()); + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + dateOfBirth: null, + timezone: "UTC", + } as never); + vi.mocked(prisma.measurement.findMany).mockResolvedValue(rows as never); + vi.mocked(prisma.medicationIntakeEvent.findMany).mockResolvedValue( + [] as never, + ); + vi.mocked(prisma.moodEntry.findMany).mockResolvedValue([] as never); + const captured: { userPrompt: string | null } = { userPrompt: null }; + stubCompletion('{"summary":"OK"}', captured); + + await generateGeneralStatusForUser("user-pulse-hours", { locale: "en" }); + + const snapshot = JSON.parse(captured.userPrompt!.match(/\{[\s\S]*\}/)![0]); + const pulse = snapshot.measurementSeries.PULSE; + expect(pulse.summary.mean).toBe(71.25); + expect(pulse.series.recent.map((d: { mean: number }) => d.mean)).toEqual([ + 60, 82.5, + ]); + const weight = snapshot.measurementSeries.WEIGHT; + expect(weight.series.recent.map((d: { mean: number }) => d.mean)).toEqual([ + 60, 132, + ]); + }); +}); diff --git a/src/lib/insights/__tests__/mood-status-pulse-day.test.ts b/src/lib/insights/__tests__/mood-status-pulse-day.test.ts new file mode 100644 index 000000000..3342c2728 --- /dev/null +++ b/src/lib/insights/__tests__/mood-status-pulse-day.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; + +import { crossMetricDailySeries } from "../mood-status"; + +/** + * The mood correlations pair each mood day with that day's pulse. A pulse + * day is the mean of its hours' means, so a workout hour of many readings does + * not decide the value the mood is paired with. + */ +describe("crossMetricDailySeries", () => { + const now = new Date("2026-03-05T12:00:00Z"); + const workoutDay = [ + ...Array.from({ length: 12 }, (_, i) => ({ + type: "PULSE", + value: 150, + measuredAt: new Date(Date.UTC(2026, 2, 3, 10, i * 5)), + })), + ...[12, 14, 16].map((h) => ({ + type: "PULSE", + value: 60, + measuredAt: new Date(Date.UTC(2026, 2, 3, h)), + })), + ]; + + it("gives a pulse day the mean of its hours' means", () => { + const series = crossMetricDailySeries(workoutDay, "PULSE", now, "UTC"); + expect(series.daily.map((d) => d.value)).toEqual([82.5]); + // The reading count stays. + expect(series.daily[0].n).toBe(15); + }); + + it("keeps the plain daily mean for every other type", () => { + const weight = workoutDay.map((m) => ({ ...m, type: "WEIGHT" })); + const series = crossMetricDailySeries(weight, "WEIGHT", now, "UTC"); + expect(series.daily.map((d) => d.value)).toEqual([132]); + }); +}); diff --git a/src/lib/insights/__tests__/pulse-hourly-mean-folds.test.ts b/src/lib/insights/__tests__/pulse-hourly-mean-folds.test.ts new file mode 100644 index 000000000..eadcbebaf --- /dev/null +++ b/src/lib/insights/__tests__/pulse-hourly-mean-folds.test.ts @@ -0,0 +1,167 @@ +/** + * A day of pulse is the mean of its hours' means and a longer bucket the mean + * of its days (`day-mean.ts`), in every in-memory fold the status cards and the + * Coach snapshot build. One workout day (twelve readings at 150 in one hour, + * three resting hours at 60: day value 82.5) and one resting day (one reading + * at 60): a bucket holding both reads 71.25; the plain mean of the sixteen + * readings is 127.5. Every other type keeps the plain mean, and the counts, + * minima and maxima stay over the readings. + */ +import { describe, expect, it } from "vitest"; + +import { bucketSeries } from "../bucket-series"; +import { + buildGradedSeriesFromDayAggregates, + buildGradedSeriesFromPoints, +} from "../graded-series"; +import { + bucketWeekly, + buildDailyValueRows, +} from "@/lib/ai/coach/snapshot-series"; + +const HOUR = 3_600_000; +const DAY = 24 * HOUR; +const TZ = "UTC"; + +function twoDays(firstDay: number) { + const rows: Array<{ measuredAt: Date; value: number }> = []; + for (let i = 0; i < 12; i += 1) { + rows.push({ + measuredAt: new Date(firstDay + 10 * HOUR + i * 300_000), + value: 150, + }); + } + for (const h of [12, 14, 16]) { + rows.push({ measuredAt: new Date(firstDay + h * HOUR), value: 60 }); + } + rows.push({ measuredAt: new Date(firstDay + DAY + 8 * HOUR), value: 60 }); + return rows; +} + +describe("graded series from points", () => { + // Wednesday 2026-03-04 and Thursday 2026-03-05, both in ISO week 10. + const first = Date.UTC(2026, 2, 4); + const rows = twoDays(first); + + it("gives a recent pulse day the mean of its hours", () => { + const g = buildGradedSeriesFromPoints( + rows, + new Date(first + 2 * DAY), + TZ, + "PULSE", + ); + const day = g.recent.find((d) => d.date === "2026-03-04")!; + expect(day.mean).toBe(82.5); + expect(day.n).toBe(15); + expect(day.min).toBe(60); + expect(day.max).toBe(150); + }); + + it("gives a pulse week the mean of its days", () => { + // 30 days later both days fall in the weekly slice. + const g = buildGradedSeriesFromPoints( + rows, + new Date(first + 30 * DAY), + TZ, + "PULSE", + ); + expect(g.weekly).toHaveLength(1); + expect(g.weekly[0].mean).toBe(71.25); + expect(g.weekly[0].n).toBe(16); + }); + + it("keeps the plain mean for any other type", () => { + const g = buildGradedSeriesFromPoints( + rows, + new Date(first + 30 * DAY), + TZ, + "BLOOD_PRESSURE_SYS", + ); + expect(g.weekly[0].mean).toBe(127.5); + }); +}); + +describe("graded series from day aggregates", () => { + it("takes a bucket's mean from the day weights when the rows carry them", () => { + const g = buildGradedSeriesFromDayAggregates([ + { + day: "2026-03-04", + segment: 1, + n: 15, + sum: 12 * 150 + 3 * 60, + min: 60, + max: 150, + weightedSum: 82.5, + weightSum: 1, + }, + { + day: "2026-03-05", + segment: 1, + n: 1, + sum: 60, + min: 60, + max: 60, + weightedSum: 60, + weightSum: 1, + }, + ]); + expect(g.weekly[0].mean).toBe(71.25); + expect(g.weekly[0].n).toBe(16); + }); +}); + +describe("bucketSeries", () => { + const now = new Date(Date.UTC(2026, 2, 6, 12)); + const rows = twoDays(Date.UTC(2026, 2, 4)); + + it("gives a pulse day its hours' mean and keeps n the readings", () => { + const s = bucketSeries(rows, { now, tz: TZ, type: "PULSE" }); + const workout = s.daily.find((d) => d.dayOffset === 2)!; + expect(workout.value).toBe(82.5); + expect(workout.n).toBe(15); + }); + + it("gives a pulse month the mean of its days", () => { + const s = bucketSeries(rows, { + now: new Date(Date.UTC(2026, 2, 4) + 20 * DAY), + tz: TZ, + type: "PULSE", + dailyDays: 10, + }); + expect(s.monthly).toHaveLength(1); + expect(s.monthly[0].value).toBe(71.25); + expect(s.monthly[0].n).toBe(16); + }); + + it("keeps the plain mean without a type or for any other type", () => { + expect( + bucketSeries(rows, { now, tz: TZ }).daily.find((d) => d.dayOffset === 2)! + .value, + ).toBe(132); + expect( + bucketSeries(rows, { now, tz: TZ, type: "WEIGHT" }).daily.find( + (d) => d.dayOffset === 2, + )!.value, + ).toBe(132); + }); +}); + +describe("Coach snapshot timeline", () => { + const first = Date.UTC(2026, 2, 4); + const rows = twoDays(first); + + it("gives a recent pulse day the mean of its hours", () => { + const daily = buildDailyValueRows(rows, new Date(first), TZ, "PULSE"); + expect(daily.map((d) => d.value)).toEqual([82.5, 60]); + expect( + buildDailyValueRows(rows, new Date(first), TZ).map((d) => d.value), + ).toEqual([132, 60]); + }); + + it("gives a pulse week the mean of its days, count the readings", () => { + const [week] = bucketWeekly(rows, TZ, "PULSE"); + expect(week.mean).toBe(71.3); + expect(week.count).toBe(16); + expect(bucketWeekly(rows, TZ)[0].mean).toBe(127.5); + }); +}); diff --git a/src/lib/insights/__tests__/pulse-status.test.ts b/src/lib/insights/__tests__/pulse-status.test.ts index 1f76a76a6..6965abddc 100644 --- a/src/lib/insights/__tests__/pulse-status.test.ts +++ b/src/lib/insights/__tests__/pulse-status.test.ts @@ -347,3 +347,52 @@ describe("generatePulseStatusForUser — token-leak hardening (v1.4.27 F16)", () expect(notes[0].text).toContain("Your pulse is stable."); }); }); + +describe("generatePulseStatusForUser — a pulse day is the mean of its hours", () => { + it("gives the latest day and the summary the hours' and days' means", async () => { + // Yesterday: a workout hour of twelve readings at 150 and three resting + // hours at 60 (day value 82.5). The day before: one reading at 60. The + // plain mean of yesterday's fifteen readings would be 132. + const now = new Date(); + const midnight = (daysAgo: number) => + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()) - + daysAgo * dayMs; + const records: Array<{ value: number; measuredAt: Date }> = []; + for (let i = 0; i < 12; i++) { + records.push({ + value: 150, + measuredAt: new Date(midnight(1) + 10 * 3_600_000 + i * 300_000), + }); + } + for (const h of [12, 14, 16]) { + records.push({ + value: 60, + measuredAt: new Date(midnight(1) + h * 3_600_000), + }); + } + records.push({ + value: 60, + measuredAt: new Date(midnight(2) + 8 * 3_600_000), + }); + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + dateOfBirth: null, + gender: null, + timezone: "UTC", + } as never); + vi.mocked(prisma.measurement.findMany).mockImplementation(((args: { + where: { type: string }; + }) => + Promise.resolve( + args.where.type === "PULSE" ? [...records].reverse() : [], + )) as never); + vi.mocked(prisma.moodEntry.findMany).mockResolvedValue([] as never); + const captured: { userPrompt: string | null } = { userPrompt: null }; + stubCompletion('{"summary":"OK"}', captured); + + await generatePulseStatusForUser("user-pulse-hours", { locale: "en" }); + + const snapshot = JSON.parse(captured.userPrompt!.match(/\{[\s\S]*\}/)![0]); + expect(snapshot.pulse.latestDayFocus.value).toBe(82.5); + expect(snapshot.pulse.summary.mean).toBe(71.25); + }); +}); diff --git a/src/lib/insights/__tests__/signals-of-day-pulse.test.ts b/src/lib/insights/__tests__/signals-of-day-pulse.test.ts new file mode 100644 index 000000000..7aea4e5d1 --- /dev/null +++ b/src/lib/insights/__tests__/signals-of-day-pulse.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; + +import { computeSignalsOfDay } from "../signals-of-day"; + +const DAY = 86_400_000; + +/** + * A day of pulse is the mean of its hours' means and a window the mean of its + * days (`day-mean.ts`). Two days: a workout hour of twelve readings at 150 and + * three resting hours at 60 (day value 82.5), then a day with one reading at + * 60. The 7- and 30-day means are 71.25, not the 127.5 of the readings. + */ +function rows(now: number) { + const today = new Date(now); + const midnight = (daysAgo: number) => + Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate()) - + daysAgo * DAY; + const out: Array<{ value: number; measuredAt: Date }> = []; + const workout = midnight(3); + for (let i = 0; i < 12; i += 1) { + out.push({ + value: 150, + measuredAt: new Date(workout + 10 * 3_600_000 + i * 300_000), + }); + } + for (const h of [12, 14, 16]) { + out.push({ value: 60, measuredAt: new Date(workout + h * 3_600_000) }); + } + out.push({ value: 60, measuredAt: new Date(midnight(2) + 8 * 3_600_000) }); + return out; +} + +describe("the pulse signal of the day", () => { + it("weighs each day once in its 7- and 30-day means", () => { + const now = Date.now(); + const pulse = rows(now); + const [signal] = computeSignalsOfDay( + (type) => (type === "PULSE" ? pulse : []), + now, + "UTC", + ); + expect(signal.metric).toBe("pulse"); + expect(signal.avg7).toBe(71.25); + expect(signal.avg30).toBe(71.25); + expect(signal.deltaVs7).toBe(-11.25); + }); + + it("leaves every other metric on the plain mean of its readings", () => { + const now = Date.now(); + const weight = rows(now); + const [signal] = computeSignalsOfDay( + (type) => (type === "WEIGHT" ? weight : []), + now, + "UTC", + ); + expect(signal.avg7).toBe(127.5); + }); +}); diff --git a/src/lib/insights/bucket-series.ts b/src/lib/insights/bucket-series.ts index fa45092b9..74fe24c7f 100644 --- a/src/lib/insights/bucket-series.ts +++ b/src/lib/insights/bucket-series.ts @@ -17,6 +17,8 @@ */ import { DEFAULT_TIMEZONE, userDayKey } from "@/lib/tz/format"; +import { readingsMean } from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -61,6 +63,13 @@ export interface BucketOptions { * Berlin; this threads the user TZ the display surfaces already use. */ tz?: string; + /** + * The measurement type, when known. For a type whose day is the mean of its + * hours' means (pulse, `day-mean.ts`) a daily bucket's value is that day + * value and a monthly bucket's the mean of its days; `n` stays the number + * of readings. + */ + type?: string; } /** @@ -144,6 +153,26 @@ export function bucketSeries( const daily = new Map(); const monthly = new Map(); + // An hourly-mean type keeps each bucket's readings for its mean of days. + const hourly = options.type !== undefined && usesHourlyMeanDay(options.type); + const dailyRows = new Map< + number, + Array<{ measuredAt: Date; value: number }> + >(); + const monthlyRows = new Map< + number, + Array<{ measuredAt: Date; value: number }> + >(); + const keep = ( + rows: Map>, + key: number, + record: { measuredAt: Date; value: number }, + ) => { + if (!hourly) return; + const list = rows.get(key); + if (list) list.push(record); + else rows.set(key, [record]); + }; // monthOffset 12 → first 30-day window past the daily horizon. Keeping // the labelling stable regardless of `dailyDays` so prompts stay @@ -160,6 +189,7 @@ export function bucketSeries( bucket.sum += record.value; bucket.n += 1; daily.set(offset, bucket); + keep(dailyRows, offset, record); continue; } @@ -170,13 +200,24 @@ export function bucketSeries( bucket.sum += record.value; bucket.n += 1; monthly.set(monthOffset, bucket); + keep(monthlyRows, monthOffset, record); } + const meanOf = ( + rows: Map>, + key: number, + agg: { sum: number; n: number }, + ) => { + const list = rows.get(key); + return list + ? (readingsMean(options.type!, list, tz) ?? agg.sum / agg.n) + : agg.sum / agg.n; + }; const dailyOut: DailyBucket[] = Array.from(daily.entries()) .sort(([a], [b]) => a - b) .map(([dayOffset, agg]) => ({ dayOffset, - value: round(agg.sum / agg.n), + value: round(meanOf(dailyRows, dayOffset, agg)), n: agg.n, })); @@ -184,7 +225,7 @@ export function bucketSeries( .sort(([a], [b]) => a - b) .map(([monthOffset, agg]) => ({ monthOffset, - value: round(agg.sum / agg.n), + value: round(meanOf(monthlyRows, monthOffset, agg)), n: agg.n, })); diff --git a/src/lib/insights/feature-blocks.ts b/src/lib/insights/feature-blocks.ts index fcdf7ec74..9fdb1f270 100644 --- a/src/lib/insights/feature-blocks.ts +++ b/src/lib/insights/feature-blocks.ts @@ -12,6 +12,11 @@ import { prisma } from "@/lib/db"; import { Prisma } from "@/generated/prisma/client"; import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; +import { + dayWeightedRowsSql, + foldMeanSql, + isHourlyMeanTypeSql, +} from "@/lib/measurements/day-mean"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; import { classifyAgainstEffectiveRange } from "@/lib/labs/reference-range"; import { calendarDaysUntil } from "@/lib/measurement-reminders/due-day"; @@ -64,10 +69,41 @@ export async function readAllTimeExtremes( const typeList = Prisma.join( types.map((t) => Prisma.sql`${t}::measurement_type`), ); + // A day of pulse is the mean of its hours' means and the full history the + // mean of its days (`day-mean.ts`). `weighted` carries, per source and day, + // the day-weighted sum and the day's weight (one) for those types only; + // every other type keeps the reading-weighted mean below, unchanged. + const mean = Prisma.raw( + foldMeanSql({ + typeColumn: 'c."type"', + total: "c.total", + count: "c.cnt", + weightedSum: "c.wsum", + weightSum: "c.wdays", + }), + ); const rows = await prisma.$queryRaw< Array<{ type: string; mean: number; min: number; max: number }> >` - WITH per_source AS ( + WITH hourly_src AS ( + SELECT m.* + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${typeList}) + AND ${Prisma.raw(isHourlyMeanTypeSql('m."type"'))} + ), + weighted AS ( + SELECT + w."type", + w."source", + date_trunc('day', w."measured_at") AS day, + SUM(w."value" * w.day_weight)::double precision AS wsum, + SUM(w.day_weight)::double precision AS wdays + FROM ${dayWeightedRowsSql("hourly_src", null, { bySource: true })} w + GROUP BY w."type", w."source", 3 + ), + per_source AS ( SELECT m."type", m."source", @@ -84,13 +120,15 @@ export async function readAllTimeExtremes( ), canon AS ( SELECT DISTINCT ON (p."type", p.day) - p."type", p.cnt, p.total, p.min_value, p.max_value + p."type", p.cnt, p.total, p.min_value, p.max_value, w.wsum, w.wdays FROM per_source p + LEFT JOIN weighted w + ON w."type" = p."type" AND w."source" = p."source" AND w.day = p.day ORDER BY p."type", p.day, (${rank}), p."source"::text ) SELECT c."type"::text AS type, - (SUM(c.total) / SUM(c.cnt))::double precision AS mean, + ${mean}::double precision AS mean, MIN(c.min_value)::double precision AS min, MAX(c.max_value)::double precision AS max FROM canon c diff --git a/src/lib/insights/features.ts b/src/lib/insights/features.ts index 93b287ffb..48f9497d3 100644 --- a/src/lib/insights/features.ts +++ b/src/lib/insights/features.ts @@ -14,7 +14,9 @@ import { SCHEDULE_COMPLIANCE_SELECT, } from "@/lib/analytics/compliance"; import { resolveUserTimezone } from "@/lib/tz/resolver"; -import { userDayKey } from "@/lib/tz/format"; +import { DEFAULT_TIMEZONE, userDayKey } from "@/lib/tz/format"; +import { readingsMean } from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { isCurrentForTodayClaim } from "@/lib/insights/measurement-freshness"; import { computeMoodDimensionSeries } from "@/lib/insights/mood-dimension-series"; import { computeContextMoodComparison } from "@/lib/insights/mood-context-crosstab"; @@ -914,12 +916,19 @@ function computeCoverage( }; } -/** Compute average of values within a time window (days ago from now). */ +/** + * Compute average of values within a time window (days ago from now). + * + * `day` names the type and the zone its days are read in. For a type whose + * day is the mean of its hours' means (pulse, see `day-mean.ts`) the window + * is then the mean of its day values; every other type keeps the plain mean. + */ export function avgInWindow( records: Array<{ value: number; measuredAt: Date }>, now: number, fromDaysAgo: number, toDaysAgo: number = 0, + day?: { type: string; tz: string }, ): number | null { const fromMs = now - fromDaysAgo * 24 * 60 * 60 * 1000; const toMs = now - toDaysAgo * 24 * 60 * 60 * 1000; @@ -928,6 +937,10 @@ export function avgInWindow( return t >= fromMs && t <= toMs; }); if (filtered.length === 0) return null; + if (day && usesHourlyMeanDay(day.type)) { + const mean = readingsMean(day.type, filtered, day.tz); + return mean === null ? null : Math.round(mean * 100) / 100; + } const sum = filtered.reduce((s, r) => s + r.value, 0); return Math.round((sum / filtered.length) * 100) / 100; } @@ -1033,6 +1046,10 @@ export async function extractFeatures( : null; const byType = (type: string) => measurements.filter((m) => m.type === type); + // The zone pulse days are read in; resolved only when there is pulse to read. + const pulseTz = measurements.some((m) => m.type === "PULSE") + ? await resolveUserTimezone(userId) + : DEFAULT_TIMEZONE; const bpTargets = getBpTargets(user?.dateOfBirth ?? null); @@ -1231,16 +1248,23 @@ export async function extractFeatures( const pulseData = byType("PULSE"); if (pulseData.length > 0) { const summary = summarize(toDataPoints(pulseData)); + // A day of pulse is the mean of its local hours' means and a window the + // mean of its days (see `day-mean.ts`): `summarize` means every reading, + // so the means come from the shared helper over the same windows. + const pulseDays = { type: "PULSE", tz: pulseTz }; + const pulseMean = (fromDaysAgo: number) => + avgInWindow(pulseData, now, fromDaysAgo, 0, pulseDays); + const allPulse = readingsMean("PULSE", pulseData, pulseTz); features.pulse = { - avg7: summary.avg7, - avg30: summary.avg30, - avg90: avgInWindow(pulseData, now, 90), + avg7: pulseMean(7), + avg30: pulseMean(30), + avg90: pulseMean(90), // v1.18.11 P1 — full-history extremes when the bulk read is windowed. allTimeAvg: allTimeExtremes ? roundMean(allTimeExtremes.get("PULSE")?.mean ?? null) - : summary.count > 0 - ? summary.mean - : null, + : allPulse === null + ? null + : Math.round(allPulse * 100) / 100, allTimeMin: allTimeExtremes ? (allTimeExtremes.get("PULSE")?.min ?? null) : summary.count > 0 @@ -1650,6 +1674,7 @@ export async function extractFeatures( features.historicalComparison.pulse = computeHistoricalComparison( pulseData, now, + { type: "PULSE", tz: pulseTz }, ); } @@ -1768,7 +1793,7 @@ export async function extractFeatures( // in-memory measurement set (no extra DB round-trip). Lands inside the // compacted features payload, so it feeds both the briefing prompt AND the // content-hash gate — a fresh daily signal forces the briefing to refresh. - const signalsOfDay = computeSignalsOfDay(byType, now); + const signalsOfDay = computeSignalsOfDay(byType, now, pulseTz); if (signalsOfDay.length > 0) { features.signalsOfDay = signalsOfDay; } diff --git a/src/lib/insights/general-status.ts b/src/lib/insights/general-status.ts index 11f1fd5f1..de8420700 100644 --- a/src/lib/insights/general-status.ts +++ b/src/lib/insights/general-status.ts @@ -216,9 +216,11 @@ export async function prepareGeneralStatusForUser( })); if (records.length === 0) return []; - const series = applyPayloadBudget(records, { now, tz: userTz }); + // `type` makes a pulse day the mean of its hours' means and a pulse + // window the mean of its days (`day-mean.ts`); other types unchanged. + const series = applyPayloadBudget(records, { now, tz: userTz, type }); dailyByType.set(type, series); - const graded = buildGradedSeriesFromPoints(records, now, userTz); + const graded = buildGradedSeriesFromPoints(records, now, userTz, type); return [ [ diff --git a/src/lib/insights/graded-series.ts b/src/lib/insights/graded-series.ts index 73e1938f8..ed50332bb 100644 --- a/src/lib/insights/graded-series.ts +++ b/src/lib/insights/graded-series.ts @@ -57,6 +57,8 @@ import { readDayAggregates, type DayAggregateRow, } from "@/lib/measurements/day-aggregates"; +import { readingsMean } from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { userDayKey } from "@/lib/tz/format"; const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -169,6 +171,20 @@ interface Agg { min: number; max: number; n: number; + /** + * Day-weighted sums for a type whose day is the mean of its hours' means + * (`DayAggregateRow.weightedSum` / `weightSum`); the bucket mean is then + * their ratio, the mean of its days. Absent for every other type. + */ + ws?: number; + wd?: number; +} + +/** A bucket's mean: of its days for an hourly-mean type, of its readings otherwise. */ +function aggMean(a: Agg): number { + return a.wd !== undefined && a.wd > 0 && a.ws !== undefined + ? a.ws / a.wd + : a.sum / a.n; } function foldInto(map: Map, key: string, value: number): void { @@ -214,8 +230,24 @@ export function buildGradedSeriesFromPoints( points: Point[], now: Date, timeZone: string, + /** + * The measurement type, when known. For a type whose day is the mean of its + * hours' means (pulse, `day-mean.ts`) every bucket's mean is the mean of its + * days; min, max and n stay over the readings. + */ + type?: string, ): GradedSeries { const nowMs = now.getTime(); + const hourly = type !== undefined && usesHourlyMeanDay(type); + const members = new Map(); + const foldPoint = (map: Map, key: string, p: Point) => { + foldInto(map, key, p.value); + if (!hourly) return; + const agg = map.get(key)!; + const list = members.get(agg); + if (list) list.push(p); + else members.set(agg, [p]); + }; const recentAgg = new Map(); const weeklyAgg = new Map(); @@ -231,26 +263,31 @@ export function buildGradedSeriesFromPoints( // Every bucket is a period of the user's own calendar. const { year, month, day } = localYmd(p.measuredAt, timeZone); if (ageMs < RECENT_WINDOW_MS) { - foldInto(recentAgg, `${year}-${month}-${day}`, p.value); + foldPoint(recentAgg, `${year}-${month}-${day}`, p); } else if (ageMs < WEEKLY_WINDOW_MS) { - foldInto(weeklyAgg, isoWeekKeyOfYmd(year, month, day), p.value); + foldPoint(weeklyAgg, isoWeekKeyOfYmd(year, month, day), p); } else if (ageMs < MONTHLY_WINDOW_MS) { - foldInto(monthlyAgg, `${year}-${month}`, p.value); + foldPoint(monthlyAgg, `${year}-${month}`, p); } else { const yk = year; - foldInto(yearlyAgg, yk, p.value); + foldPoint(yearlyAgg, yk, p); const list = yearlyValues.get(yk); if (list) list.push(p.value); else yearlyValues.set(yk, [p.value]); } } + for (const [agg, list] of members) { + agg.ws = readingsMean(type!, list, timeZone) ?? agg.sum / agg.n; + agg.wd = 1; + } + const recent: RecentDayBucket[] = Array.from(recentAgg.entries()) .map(([date, a]) => ({ date, min: round(a.min), max: round(a.max), - mean: round(a.sum / a.n), + mean: round(aggMean(a)), n: a.n, })) .sort((x, y) => x.date.localeCompare(y.date)); @@ -260,7 +297,7 @@ export function buildGradedSeriesFromPoints( weekISO, min: round(a.min), max: round(a.max), - mean: round(a.sum / a.n), + mean: round(aggMean(a)), n: a.n, })) .sort((x, y) => x.weekISO.localeCompare(y.weekISO)); @@ -270,7 +307,7 @@ export function buildGradedSeriesFromPoints( month, min: round(a.min), max: round(a.max), - mean: round(a.sum / a.n), + mean: round(aggMean(a)), n: a.n, })) .sort((x, y) => x.month.localeCompare(y.month)); @@ -280,7 +317,7 @@ export function buildGradedSeriesFromPoints( year, min: round(a.min), max: round(a.max), - mean: round(a.sum / a.n), + mean: round(aggMean(a)), n: a.n, // Order the readings oldest → newest for the slope. The map kept // insertion order; `points` arrives caller-ordered, so re-sort by @@ -304,8 +341,20 @@ function mergeInto( existing.n += row.n; if (row.min < existing.min) existing.min = row.min; if (row.max > existing.max) existing.max = row.max; + if (row.weightedSum !== undefined && row.weightSum !== undefined) { + existing.ws = (existing.ws ?? 0) + row.weightedSum; + existing.wd = (existing.wd ?? 0) + row.weightSum; + } } else { - map.set(key, { sum: row.sum, min: row.min, max: row.max, n: row.n }); + map.set(key, { + sum: row.sum, + min: row.min, + max: row.max, + n: row.n, + ...(row.weightedSum !== undefined && row.weightSum !== undefined + ? { ws: row.weightedSum, wd: row.weightSum } + : {}), + }); } } @@ -359,7 +408,9 @@ async function readGradedDayAggregates( * The graded series folded from per-day aggregates (segment = age class). * * recent / weekly / monthly and every bucket's min / max / mean / n match - * the raw-row fold exactly. The yearly slope is taken over the day means in + * the raw-row fold exactly, except that a type whose day is the mean of its + * hours' means (pulse, `day-mean.ts`) takes each bucket's mean over its days + * (the rows' `weightedSum` / `weightSum`), each day the mean of its hours. The yearly slope is taken over the day means in * date order rather than over individual readings: a direction signal either * way, and the only form that stays bounded on a dense stream. */ @@ -386,7 +437,10 @@ export function buildGradedSeriesFromDayAggregates( } else { mergeInto(yearlyAgg, year, row); const list = yearlyValues.get(year); - const dayMean = row.sum / row.n; + const dayMean = + row.weightedSum !== undefined && row.weightSum + ? row.weightedSum / row.weightSum + : row.sum / row.n; if (list) list.push(dayMean); else yearlyValues.set(year, [dayMean]); } @@ -398,7 +452,7 @@ export function buildGradedSeriesFromDayAggregates( key, min: round(a.min), max: round(a.max), - mean: round(a.sum / a.n), + mean: round(aggMean(a)), n: a.n, })) .sort((x, y) => x.key.localeCompare(y.key)); diff --git a/src/lib/insights/mood-status.ts b/src/lib/insights/mood-status.ts index a827b1dc9..4c787f040 100644 --- a/src/lib/insights/mood-status.ts +++ b/src/lib/insights/mood-status.ts @@ -93,6 +93,30 @@ function stripFactorCrosstab( return rest; } +/** + * One cross metric's daily buckets for the mood correlations. The type is + * passed through, so a pulse day is the mean of its hours' means (see + * `day-mean.ts`) and a workout hour does not decide the day the mood is + * paired with; every other type keeps its plain daily mean. + */ +export function crossMetricDailySeries( + measurements: ReadonlyArray<{ + type: string; + measuredAt: Date; + value: number; + }>, + type: string, + now: Date, + tz: string, +): ReturnType { + return applyPayloadBudget( + measurements + .filter((m) => m.type === type) + .map((m) => ({ measuredAt: m.measuredAt, value: m.value })), + { now, tz, type }, + ); +} + /** * Public entry — unchanged signature. Prepares the card (cache-read, * read-only miss, snapshot build, hash gate) then, when the LLM is still @@ -389,25 +413,23 @@ export async function prepareMoodStatusForUser( annotate({ meta: { cross_metric_truncated: true } }); } - const weightSeries = applyPayloadBudget( - measurements - .filter((m) => m.type === "WEIGHT") - .map((m) => ({ measuredAt: m.measuredAt, value: m.value })), - { now, tz: userTz }, + const weightSeries = crossMetricDailySeries( + measurements, + "WEIGHT", + now, + userTz, ); - - const sysSeries = applyPayloadBudget( - measurements - .filter((m) => m.type === "BLOOD_PRESSURE_SYS") - .map((m) => ({ measuredAt: m.measuredAt, value: m.value })), - { now, tz: userTz }, + const sysSeries = crossMetricDailySeries( + measurements, + "BLOOD_PRESSURE_SYS", + now, + userTz, ); - - const pulseSeries = applyPayloadBudget( - measurements - .filter((m) => m.type === "PULSE") - .map((m) => ({ measuredAt: m.measuredAt, value: m.value })), - { now, tz: userTz }, + const pulseSeries = crossMetricDailySeries( + measurements, + "PULSE", + now, + userTz, ); // Correlations between mood and other metrics — pair on dayOffset so diff --git a/src/lib/insights/pulse-status.ts b/src/lib/insights/pulse-status.ts index 084e17aed..b6143d2e1 100644 --- a/src/lib/insights/pulse-status.ts +++ b/src/lib/insights/pulse-status.ts @@ -229,7 +229,12 @@ export async function preparePulseStatusForUser( // `applyPayloadBudget` daily buckets still drive the derived stats // below (latest, in-target %, delta). They are NOT embedded in the // prompt — the compact graded series replaces the full daily array. - const pulseSeries = applyPayloadBudget(pulsePoints, { now, tz: userTz }); + // A pulse day is the mean of its hours' means (`day-mean.ts`). + const pulseSeries = applyPayloadBudget(pulsePoints, { + now, + tz: userTz, + type: "PULSE", + }); // Primary metric: recent / weekly fold from the bounded raw read, the // monthly / yearly tail comes from the MONTH / YEAR rollup tier (with // a full-history in-memory fallback on a cold-tier coverage miss). diff --git a/src/lib/insights/signals-of-day.ts b/src/lib/insights/signals-of-day.ts index a6bf3f0b0..6f6c620e0 100644 --- a/src/lib/insights/signals-of-day.ts +++ b/src/lib/insights/signals-of-day.ts @@ -54,13 +54,14 @@ export interface SignalOfDay { export function computeHistoricalComparison( records: Array<{ value: number; measuredAt: Date }>, now: number, + day?: { type: string; tz: string }, ): { current7dAvg: number | null; previous30dAvg: number | null; change: number | null; } { - const current7dAvg = avgInWindow(records, now, 7, 0); - const previous30dAvg = avgInWindow(records, now, 37, 7); + const current7dAvg = avgInWindow(records, now, 7, 0, day); + const previous30dAvg = avgInWindow(records, now, 37, 7, day); const change = current7dAvg !== null && previous30dAvg !== null ? Math.round((current7dAvg - previous30dAvg) * 100) / 100 @@ -98,6 +99,7 @@ function buildSignal( records: Array<{ value: number; measuredAt: Date }>, now: number, unit?: string, + day?: { type: string; tz: string }, ): SignalOfDay | null { if (records.length === 0) return null; const newest = records[records.length - 1]; @@ -112,8 +114,10 @@ function buildSignal( ); if (win30.length < 3) return null; - const avg7 = avgInWindow(records, now, 7); - const avg30 = avgInWindow(records, now, 30); + // For pulse each window is the mean of its day values (`day-mean.ts`); the + // spread and the anomaly stay over the readings. + const avg7 = avgInWindow(records, now, 7, 0, day); + const avg30 = avgInWindow(records, now, 30, 0, day); const spread30 = stdDev(win30.map((rec) => rec.value)); const deltaVs7 = avg7 !== null ? r2(newest.value - avg7) : null; const deltaVs30 = avg30 !== null ? r2(newest.value - avg30) : null; @@ -186,6 +190,8 @@ function buildSignal( export function computeSignalsOfDay( byType: (type: string) => Array<{ value: number; measuredAt: Date }>, now: number, + /** The zone pulse days are read in (`day-mean.ts`). */ + pulseTz: string = "UTC", ): SignalOfDay[] { const candidates: SignalOfDay[] = []; const push = (s: SignalOfDay | null) => { @@ -213,7 +219,12 @@ export function computeSignalsOfDay( "bpm", ), ); - push(buildSignal("pulse", "pulse", byType("PULSE"), now, "bpm")); + push( + buildSignal("pulse", "pulse", byType("PULSE"), now, "bpm", { + type: "PULSE", + tz: pulseTz, + }), + ); // Weight and glucose are canonical here and carry no unit: the reader's // unit is attached where the signal is converted for the prompt // (`featuresInReaderUnits`), so no canonical symbol can leak into one. diff --git a/tests/integration/feature-blocks-pulse-day-mean.test.ts b/tests/integration/feature-blocks-pulse-day-mean.test.ts new file mode 100644 index 000000000..15bcffd6f --- /dev/null +++ b/tests/integration/feature-blocks-pulse-day-mean.test.ts @@ -0,0 +1,75 @@ +/** + * The full-history pulse mean the insight features carry (`allTimeAvg`) is the + * mean of the day values, each day the mean of its hours' means, like every + * other pulse mean (`day-mean.ts`). Count-weighted, a workout day's dense hour + * outweighed every resting hour of every other day. + * + * Two days: a workout hour of twelve readings at 150 and three resting hours + * at 60 (day value 82.5), and a day with one reading at 60: 71.25. The same + * readings as systolic pressure keep the plain mean (127.5) as the control. + */ +import { beforeAll, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { Prisma } from "@/generated/prisma/client"; +import { readAllTimeExtremes } from "@/lib/insights/feature-blocks"; + +const prisma = getPrismaClient(); +const USER = "feature-blocks-pulse-day"; +const DAY = 86_400_000; +const now = new Date(); +const midnight = (daysAgo: number) => + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()) - + daysAgo * DAY; + +function readings( + type: "PULSE" | "BLOOD_PRESSURE_SYS", + unit: string, +): Prisma.MeasurementCreateManyInput[] { + const workout = midnight(40); + const at: Array<[number, number]> = []; + for (let i = 0; i < 12; i += 1) { + at.push([workout + 10 * 3_600_000 + i * 300_000, 150]); + } + for (const h of [12, 14, 16]) at.push([workout + h * 3_600_000, 60]); + at.push([midnight(39) + 8 * 3_600_000, 60]); + return at.map(([ms, value], i) => ({ + id: `${USER}-${type}-${i}`, + userId: USER, + type, + unit, + source: "APPLE_HEALTH", + value, + measuredAt: new Date(ms), + })); +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ data: { id: USER, username: USER } }); + await prisma.measurement.createMany({ + data: [ + ...readings("PULSE", "bpm"), + ...readings("BLOOD_PRESSURE_SYS", "mmHg"), + ], + }); +}, 120_000); + +describe("readAllTimeExtremes", () => { + it("gives pulse the mean of its days, each the mean of its hours", async () => { + const out = await readAllTimeExtremes(USER, [ + "PULSE", + "BLOOD_PRESSURE_SYS", + ]); + const pulse = out.get("PULSE"); + expect(pulse?.mean).toBeCloseTo(71.25, 9); + // The extremes stay over every reading. + expect(pulse?.min).toBe(60); + expect(pulse?.max).toBe(150); + }); + + it("keeps the plain mean of the readings for every other type", async () => { + const out = await readAllTimeExtremes(USER, ["BLOOD_PRESSURE_SYS"]); + expect(out.get("BLOOD_PRESSURE_SYS")?.mean).toBe(127.5); + }); +}); diff --git a/tests/integration/graded-series-pulse-day-mean.test.ts b/tests/integration/graded-series-pulse-day-mean.test.ts new file mode 100644 index 000000000..5371dc30b --- /dev/null +++ b/tests/integration/graded-series-pulse-day-mean.test.ts @@ -0,0 +1,117 @@ +/** + * The pulse status card's graded series reads per-day aggregates folded in + * Postgres (`readDayAggregates`). For pulse those rows carry the day weights + * of `day-mean.ts`, so a recent day is the mean of its hours' means and a + * week the mean of its days, against a real database. + * + * Each pair of days: a workout hour of twelve readings at 150 and three + * resting hours at 60 (day value 82.5), then a day with one reading at 60. + * A bucket over both reads 71.25; the plain mean of the readings is 127.5. + * Systolic pressure with the same readings keeps the plain mean. + */ +import { beforeAll, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { Prisma } from "@/generated/prisma/client"; +import { buildGradedSeriesWithRollups } from "@/lib/insights/graded-series"; +import { readDayAggregates } from "@/lib/measurements/day-aggregates"; + +const prisma = getPrismaClient(); +const USER = "graded-pulse-day"; +const DAY = 86_400_000; +const now = new Date(); +const today = Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate(), +); +/** A Monday about six weeks back: its Tuesday and Wednesday share a week. */ +const dow = (new Date(today).getUTCDay() + 6) % 7; +const OLD_MONDAY = today - dow * DAY - 6 * 7 * DAY; +const RECENT = today - 4 * DAY; + +function pair( + type: "PULSE" | "BLOOD_PRESSURE_SYS", + start: number, + tag: string, +): Prisma.MeasurementCreateManyInput[] { + const at: Array<[number, number]> = []; + for (let i = 0; i < 12; i += 1) { + at.push([start + 10 * 3_600_000 + i * 300_000, 150]); + } + for (const h of [12, 14, 16]) at.push([start + h * 3_600_000, 60]); + at.push([start + DAY + 8 * 3_600_000, 60]); + return at.map(([ms, value], i) => ({ + id: `${USER}-${type}-${tag}-${i}`, + userId: USER, + type, + unit: type === "PULSE" ? "bpm" : "mmHg", + source: "APPLE_HEALTH", + value, + measuredAt: new Date(ms), + })); +} + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ data: { id: USER, username: USER } }); + const data: Prisma.MeasurementCreateManyInput[] = []; + for (const type of ["PULSE", "BLOOD_PRESSURE_SYS"] as const) { + data.push(...pair(type, RECENT, "recent")); + data.push(...pair(type, OLD_MONDAY + DAY, "old")); + } + await prisma.measurement.createMany({ data }); +}, 120_000); + +describe("readDayAggregates for pulse", () => { + it("carries day weights whose ratio is the day value", async () => { + const rows = await readDayAggregates({ + userId: USER, + type: "PULSE", + since: new Date(RECENT), + timeZone: "UTC", + }); + const workout = rows.find( + (r) => r.day === new Date(RECENT).toISOString().slice(0, 10), + )!; + expect(workout.n).toBe(15); + expect(workout.min).toBe(60); + expect(workout.max).toBe(150); + expect(workout.weightSum).toBeCloseTo(1, 9); + expect(workout.weightedSum! / workout.weightSum!).toBeCloseTo(82.5, 9); + }); + + it("leaves every other type without them", async () => { + const rows = await readDayAggregates({ + userId: USER, + type: "BLOOD_PRESSURE_SYS", + since: new Date(RECENT), + timeZone: "UTC", + }); + expect(rows.every((r) => r.weightedSum === undefined)).toBe(true); + }); +}); + +describe("the pulse graded series", () => { + it("gives a recent day its hours' mean and a week its days' mean", async () => { + const g = await buildGradedSeriesWithRollups(USER, "PULSE", now, "UTC"); + const workout = g.recent.find( + (d) => d.date === new Date(RECENT).toISOString().slice(0, 10), + )!; + expect(workout.mean).toBe(82.5); + expect(workout.n).toBe(15); + expect(g.weekly).toHaveLength(1); + expect(g.weekly[0].mean).toBe(71.25); + expect(g.weekly[0].n).toBe(16); + }); + + it("keeps the plain mean for systolic pressure", async () => { + const g = await buildGradedSeriesWithRollups( + USER, + "BLOOD_PRESSURE_SYS", + now, + "UTC", + ); + expect(g.weekly[0].mean).toBe(127.5); + }); +}); From 8f730484e0e5a4e1be15ab3b0c2f1579d9c863f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 11:37:57 +0200 Subject: [PATCH 07/10] Average pulse by day in the comparison snapshot and Coach inventory The insights comparison snapshot took its current and baseline pulse figures from a reading AVG, and the Coach's data inventory told the model a reading mean of the whole pulse history. Both now read the mean of the user's local days, each the mean of its hours' means, through the day-mean helper. Every other type keeps its aggregate. --- .../coach-availability-window-honesty.test.ts | 31 +++++++ src/lib/ai/coach/tools/availability.ts | 55 ++++++++++- .../comparison-snapshot-window.test.ts | 34 ++++++- src/lib/insights/comprehensive-generate.ts | 58 +++++++++++- .../pulse-comparison-availability.test.ts | 91 +++++++++++++++++++ 5 files changed, 258 insertions(+), 11 deletions(-) create mode 100644 tests/integration/pulse-comparison-availability.test.ts diff --git a/src/lib/ai/coach/__tests__/coach-availability-window-honesty.test.ts b/src/lib/ai/coach/__tests__/coach-availability-window-honesty.test.ts index 06634c8c7..e34055329 100644 --- a/src/lib/ai/coach/__tests__/coach-availability-window-honesty.test.ts +++ b/src/lib/ai/coach/__tests__/coach-availability-window-honesty.test.ts @@ -48,6 +48,9 @@ interface GroupRow { } let measurementRows: GroupRow[] = []; let measurementThrows = false; +/** What the day-weighted pulse query returns, per `type|unit`. */ +let pulseDayMeans: Array<{ type: string; unit: string | null; mean: number }> = + []; const workoutAggregate = vi.fn(); const moodAggregate = vi.fn(); const intakeAggregate = vi.fn(); @@ -66,6 +69,7 @@ vi.mock("@/lib/db", () => ({ ); }, }, + $queryRaw: () => Promise.resolve(pulseDayMeans), workout: { aggregate: (a: unknown) => workoutAggregate(a) }, moodEntry: { aggregate: (a: unknown) => moodAggregate(a) }, medicationIntakeEvent: { aggregate: (a: unknown) => intakeAggregate(a) }, @@ -386,6 +390,33 @@ describe("classifyAvailability", () => { }); describe("probeCoachAvailability", () => { + // A pulse history's mean is the mean of its days, each the mean of its + // hours' means; the reading AVG a workout's dense samples tip is not what + // the Coach is told. + it("states the day-weighted pulse mean, not the reading mean", async () => { + measurementRows = [ + { + type: "PULSE", + unit: "bpm", + _count: { _all: 930 }, + _min: { measuredAt: new Date("2026-07-20T00:05:00Z"), value: 60 }, + _max: { measuredAt: new Date("2026-07-20T23:55:00Z"), value: 150 }, + _avg: { value: 127.7 }, + }, + ]; + pulseDayMeans = [{ type: "PULSE", unit: "bpm", mean: 63.75 }]; + const probed = await probeCoachAvailability( + "u1", + new Map([["pulse", { kind: "measurement", types: ["PULSE"] }]]), + { now: NOW }, + ); + pulseDayMeans = []; + const series = [...probed.values()][0]?.series; + expect(series).toEqual([ + expect.objectContaining({ series: "PULSE", mean: 63.8, count: 930 }), + ]); + }); + it("returns no entry for a domain the record is empty for", async () => { measurementRows = []; const probed = await probeCoachAvailability( diff --git a/src/lib/ai/coach/tools/availability.ts b/src/lib/ai/coach/tools/availability.ts index 623e5bb30..c7c35110c 100644 --- a/src/lib/ai/coach/tools/availability.ts +++ b/src/lib/ai/coach/tools/availability.ts @@ -44,7 +44,9 @@ * is enough for the Coach to answer the question instead of only naming it. */ import { prisma } from "@/lib/db"; -import type { MeasurementType } from "@/generated/prisma/client"; +import { Prisma, type MeasurementType } from "@/generated/prisma/client"; +import { dayWeightedRowsSql, windowMeanSql } from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { userDayKey } from "@/lib/tz/format"; import { resolveUserTimezone } from "@/lib/tz/resolver"; import { annotate } from "@/lib/logging/context"; @@ -155,13 +157,18 @@ async function readMeasurementBounds( _max: { measuredAt: true, value: true }, _avg: { value: true }, }); + const dayMeans = await readHourlyMeanTypeMeans(userId, types); const out = new Map(); for (const row of rows) { const firstAt = row._min.measuredAt; const lastAt = row._max.measuredAt; const min = row._min.value; const max = row._max.value; - const mean = row._avg.value; + // A pulse mean is the mean of its days, each the mean of its hours' + // means, so a dense workout hour does not stand in for the history. + const mean = usesHourlyMeanDay(row.type) + ? (dayMeans.get(`${row.type}|${row.unit}`) ?? null) + : row._avg.value; if (!firstAt || !lastAt) continue; const existing = out.get(row.type); const unitEntry = @@ -197,6 +204,50 @@ async function readMeasurementBounds( return out; } +/** + * The all-time mean per type and unit for the hourly-mean types among + * `types` (pulse, see `day-mean.ts`): the mean of the user's local days, each + * the mean of its local hours' means. Empty when none of `types` is one. + */ +async function readHourlyMeanTypeMeans( + userId: string, + types: readonly MeasurementType[], +): Promise> { + const hourly = types.filter((t) => usesHourlyMeanDay(t)); + const out = new Map(); + if (hourly.length === 0) return out; + const tz = await resolveUserTimezone(userId); + const rows = await prisma.$queryRaw< + Array<{ type: string; unit: string | null; mean: number | null }> + >` + WITH src AS ( + SELECT * + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."type" IN (${Prisma.join( + hourly.map((t) => Prisma.sql`${t}::measurement_type`), + )}) + ) + SELECT + m."type"::text AS type, + m."unit" AS unit, + ${Prisma.raw( + windowMeanSql({ + typeColumn: 'm."type"', + value: 'm."value"', + weight: "m.day_weight", + }), + )}::double precision AS mean + FROM ${dayWeightedRowsSql("src", tz)} m + GROUP BY m."type", m."unit" + `; + for (const r of rows) { + if (r.mean !== null) out.set(`${r.type}|${r.unit}`, Number(r.mean)); + } + return out; +} + /** One `aggregate()` per non-measurement subject kind. */ async function readTableBounds( userId: string, diff --git a/src/lib/insights/__tests__/comparison-snapshot-window.test.ts b/src/lib/insights/__tests__/comparison-snapshot-window.test.ts index 98adc7732..56d5b293f 100644 --- a/src/lib/insights/__tests__/comparison-snapshot-window.test.ts +++ b/src/lib/insights/__tests__/comparison-snapshot-window.test.ts @@ -23,6 +23,7 @@ import { summarize, type DataPoint } from "@/lib/analytics/trends"; const findUnique = vi.fn(); const measurementFindMany = vi.fn(); const measurementAggregate = vi.fn(); +const queryRaw = vi.fn(); vi.mock("@/lib/db", () => ({ prisma: { @@ -33,6 +34,7 @@ vi.mock("@/lib/db", () => ({ findMany: (...a: unknown[]) => measurementFindMany(...a), aggregate: (...a: unknown[]) => measurementAggregate(...a), }, + $queryRaw: (...a: unknown[]) => queryRaw(...a), }, })); // SLEEP_DURATION branch helpers — most tests never exercise the sleep @@ -84,6 +86,8 @@ beforeEach(() => { measurementFindMany.mockResolvedValue([]); // No measurements by default; individual tests override per call. measurementAggregate.mockResolvedValue({ _avg: { value: null } }); + // Pulse reads its day-weighted window mean through one raw query per window. + queryRaw.mockResolvedValue([]); }); describe("buildComparisonSnapshotForUser — aggregate windows", () => { @@ -131,9 +135,10 @@ describe("buildComparisonSnapshotForUser — aggregate windows", () => { expect(lteMs).toBeGreaterThanOrEqual(before - 30 * DAY); expect(lteMs).toBeLessThanOrEqual(after - 30 * DAY); } - // Six non-sleep types × 2 windows = 12 aggregate reads; no raw non-sleep read. - expect(currentCalls.length).toBe(6); - expect(baselineCalls.length).toBe(6); + // Five non-sleep, non-pulse types × 2 windows = 10 aggregate reads; no raw + // non-sleep read. Pulse reads its day-weighted mean instead (below). + expect(currentCalls.length).toBe(5); + expect(baselineCalls.length).toBe(5); }); it("reads SLEEP_DURATION through the raw ~400-day findMany, never aggregate", async () => { @@ -163,7 +168,7 @@ describe("buildComparisonSnapshotForUser — aggregate windows", () => { .measuredAt, ) .filter((m): m is { gt: Date; lte: Date } => m.lte != null); - expect(baselineCalls.length).toBe(6); + expect(baselineCalls.length).toBe(5); for (const w of baselineCalls) { const gtMs = w.gt.getTime(); const lteMs = w.lte.getTime(); @@ -175,6 +180,27 @@ describe("buildComparisonSnapshotForUser — aggregate windows", () => { }); }); +describe("buildComparisonSnapshotForUser — pulse", () => { + // A pulse window average is the mean of its days, each the mean of its + // hours' means, so a workout's dense readings do not decide the delta. A + // bare AVG cannot express it, so pulse never goes through the aggregate. + it("reads pulse through the day-weighted query, never the reading AVG", async () => { + queryRaw + .mockResolvedValueOnce([{ mean: 64 }]) + .mockResolvedValueOnce([{ mean: 60 }]); + const snapshot = await buildComparisonSnapshotForUser("u1"); + const pulseAggregates = measurementAggregate.mock.calls.filter( + (c) => (c[0] as { where: { type: string } }).where.type === "PULSE", + ); + expect(pulseAggregates).toHaveLength(0); + expect(queryRaw).toHaveBeenCalledTimes(2); + expect(JSON.stringify(queryRaw.mock.calls[0])).toContain("day_weight"); + const pulse = snapshot!.metrics.find((m) => m.type === "pulse"); + expect(pulse).toMatchObject({ unit: "bpm" }); + expect([pulse!.currentAvg, pulse!.baselineAvg].sort()).toEqual([60, 64]); + }); +}); + describe("buildComparisonSnapshotForUser — output-equivalence to the raw-average path", () => { it("emits identical metrics to findMany-then-summarize() for a fixture (lastMonth)", async () => { findUnique.mockResolvedValue({ diff --git a/src/lib/insights/comprehensive-generate.ts b/src/lib/insights/comprehensive-generate.ts index dd98110c5..5469072e9 100644 --- a/src/lib/insights/comprehensive-generate.ts +++ b/src/lib/insights/comprehensive-generate.ts @@ -78,6 +78,9 @@ import { type ComparisonBaseline, } from "@/lib/dashboard-layout"; import type { MeasurementType, Prisma } from "@/generated/prisma/client"; +import { Prisma as PrismaSql } from "@/generated/prisma/client"; +import { dayWeightedRowsSql, windowMeanSql } from "@/lib/measurements/day-mean"; +import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { insightResultSchema, type InsightResult } from "@/lib/ai/types"; import { resolveProvider, resolveProviderChain } from "@/lib/ai/provider"; import { AllProvidersFailedError } from "@/lib/ai/provider-runner"; @@ -321,6 +324,48 @@ function parseComprehensiveResult( : (parsed as Record); } +/** + * The window average of an hourly-mean type (pulse, see `day-mean.ts`) for + * the comparison snapshot: the mean of the user's local days in the window, + * each the mean of its local hours' means, so the current-vs-baseline delta + * compares the statistic the dashboard tile shows rather than a reading mean + * a workout tips. + */ +export async function hourlyMeanWindowAverage( + userId: string, + type: MeasurementType, + measuredAt: { gt: Date; lte?: Date }, +): Promise { + const tz = await resolveUserTimezone(userId); + const upper = + measuredAt.lte === undefined + ? PrismaSql.empty + : PrismaSql.sql`AND m."measured_at" <= ${measuredAt.lte}`; + const [row] = await prisma.$queryRaw>` + WITH src AS ( + SELECT * + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."type" = ${type}::measurement_type + AND m."deleted_at" IS NULL + AND m."measured_at" > ${measuredAt.gt} + ${upper} + ) + SELECT ${PrismaSql.raw( + windowMeanSql({ + typeColumn: 'm."type"', + value: 'm."value"', + weight: "m.day_weight", + }), + )}::double precision AS mean + FROM ${dayWeightedRowsSql("src", tz)} m + GROUP BY m."type" + `; + return row?.mean === null || row?.mean === undefined + ? null + : Number(row.mean); +} + /** * Comparison-snapshot builder — shared with the on-demand route. Returns * null when the user's comparison toggle is off (most users), so the @@ -406,11 +451,14 @@ export async function buildComparisonSnapshotForUser( type: MeasurementType, measuredAt: { gt: Date; lte?: Date }, ): Promise => { - const agg = await prisma.measurement.aggregate({ - where: { userId, type, deletedAt: null, measuredAt }, - _avg: { value: true }, - }); - const avg = agg._avg.value; + const avg = usesHourlyMeanDay(type) + ? await hourlyMeanWindowAverage(userId, type, measuredAt) + : ( + await prisma.measurement.aggregate({ + where: { userId, type, deletedAt: null, measuredAt }, + _avg: { value: true }, + }) + )._avg.value; return avg === null ? null : Math.round(avg * 100) / 100; }; diff --git a/tests/integration/pulse-comparison-availability.test.ts b/tests/integration/pulse-comparison-availability.test.ts new file mode 100644 index 000000000..06304bda5 --- /dev/null +++ b/tests/integration/pulse-comparison-availability.test.ts @@ -0,0 +1,91 @@ +/** + * The comparison snapshot's window average and the Coach's availability mean + * read pulse as the mean of its days, each the mean of its hours' means, on a + * real Postgres. + * + * Two days, three days back and two days back (inside the 30-day window): a + * workout day with one hour of 12 readings at 150 and three resting hours of + * one reading at 60 (day value 82.5), and a resting day of two hours at 70. + * Mean of the days: 76.25. Mean of the readings: (12 x 150 + 3 x 60 + 2 x 70) + * / 17 = 124.7. + */ +import { beforeAll, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { Prisma } from "@/generated/prisma/client"; +import { probeCoachAvailability } from "@/lib/ai/coach/tools/availability"; +import { hourlyMeanWindowAverage } from "@/lib/insights/comprehensive-generate"; +import { invalidateUserTimezone } from "@/lib/tz/resolver"; + +const prisma = getPrismaClient(); +const USER = "pulse-comparison-availability"; +const DAY = 86_400_000; +const now = new Date(); +const D1 = Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() - 3, +); +const D2 = D1 + DAY; +const DAY_MEAN = (82.5 + 70) / 2; +const READING_MEAN = (12 * 150 + 3 * 60 + 2 * 70) / 17; + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, timezone: "UTC" }, + }); + invalidateUserTimezone(USER); + const rows: Prisma.MeasurementCreateManyInput[] = []; + const add = (at: number, value: number) => + rows.push({ + id: `${USER}-${rows.length}`, + userId: USER, + type: "PULSE", + unit: "bpm", + source: "APPLE_HEALTH", + value, + measuredAt: new Date(at), + }); + for (let i = 0; i < 12; i += 1) add(D1 + 10 * 3_600_000 + i * 300_000, 150); + for (const h of [12, 14, 16]) add(D1 + h * 3_600_000, 60); + for (const h of [8, 9]) add(D2 + h * 3_600_000, 70); + await prisma.measurement.createMany({ data: rows }); +}, 120_000); + +describe("pulse window means on a real database", () => { + it("the fixture separates the two statistics", () => { + expect(READING_MEAN).toBeCloseTo(124.71, 2); + expect(DAY_MEAN).toBe(76.25); + }); + + it("the comparison snapshot's window average is the mean of the days", async () => { + const avg = await hourlyMeanWindowAverage(USER, "PULSE", { + gt: new Date(Date.now() - 30 * DAY), + }); + expect(avg).toBeCloseTo(DAY_MEAN, 9); + const empty = await hourlyMeanWindowAverage(USER, "PULSE", { + gt: new Date(Date.now() - 60 * DAY), + lte: new Date(Date.now() - 30 * DAY), + }); + expect(empty).toBeNull(); + }); + + it("the Coach availability mean is the mean of the days", async () => { + const probed = await probeCoachAvailability( + USER, + new Map([["pulse", { kind: "measurement", types: ["PULSE"] }]]), + { now }, + ); + const series = probed.get("pulse")?.series; + expect(series).toEqual([ + expect.objectContaining({ + series: "PULSE", + count: 17, + mean: Math.round(DAY_MEAN * 10) / 10, + min: 60, + max: 150, + }), + ]); + }); +}); From 06f210b56f2752fbb2b3b364147d7a5624a91c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 11:39:33 +0200 Subject: [PATCH 08/10] Guard every pulse mean reader behind the day-mean helper A structural test lists every file that takes a database or in-memory mean over measurement values and names pulse, and requires each to use the shared statistic, hand the type to a fold that does, or state why it never averages pulse readings. A new plain mean over pulse, or a reader that drops the type, fails it. --- .../pulse-day-mean-reader-guard.test.ts | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 src/__tests__/pulse-day-mean-reader-guard.test.ts diff --git a/src/__tests__/pulse-day-mean-reader-guard.test.ts b/src/__tests__/pulse-day-mean-reader-guard.test.ts new file mode 100644 index 000000000..6f76b206d --- /dev/null +++ b/src/__tests__/pulse-day-mean-reader-guard.test.ts @@ -0,0 +1,249 @@ +/** + * Structural guard: every reader that averages pulse goes through the + * day-mean helper. + * + * A watch samples pulse every few minutes at rest and every few seconds in a + * workout, so a plain mean over pulse readings lets one workout hour outweigh + * the rest of the day. A day of pulse is the mean of its hours' means and a + * window the mean of its days (`src/lib/measurements/day-statistic.ts`), and + * `src/lib/measurements/day-mean.ts` is the one place that computes it. The + * statistic first shipped in the chart and the rollup tier while some twenty + * other readers kept the plain mean, so the dashboard's year delta compared + * one statistic against another. + * + * Two matchers, both frozen both ways (a new unlisted match fails, a listed + * file that no longer matches fails): + * + * 1. Database means over measurement values: `AVG(value)`, the per-day + * folds `SUM(total) / SUM(cnt)` and `SUM(count * mean)`, and Prisma's + * `_avg: { value }`. A file that holds one imports the shared statistic + * (it averages pulse among other types and the helper's CASE keeps the + * others on the plain mean). + * + * 2. In-memory means in a file that names pulse: a left fold divided by a + * length, `avgInWindow(`, `summarize(`, `bucketWeekly(`, + * `buildDailyValueRows(` or `applyPayloadBudget(`. A file that holds one + * imports the shared statistic, hands the type to a fold that does, or + * is listed with the reason it never averages pulse readings. + * + * Comments are stripped before matching, the matchers are whitespace-tolerant, + * and the floors fail the guard if either matcher stops finding anything. The + * planted cases run each matcher over an offender; dropping the type from the + * mood correlation's fold, or a new file with a plain `AVG(value)` over pulse, + * fails the two list checks. It is a tripwire, not a proof: a mean written + * some other way slips it. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +/** The shared statistic: the helper, or the type set it is keyed on. */ +const HELPER_IMPORT = + /from\s+["'](?:@\/lib\/measurements\/|\.\/)day-(?:mean|statistic)["']/; + +/** Comments go first: a mean named in prose is not a mean computed. */ +function stripComments(text: string): string { + return text + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/^\s*\/\/.*$/gm, "") + .replace(/^\s*--.*$/gm, ""); +} + +/** + * Database means over a measurement value: SQL `AVG(value)`, the per-day folds + * `SUM(total) / SUM(cnt)` and `SUM(count * mean)`, and Prisma's + * `_avg: { value: ... }`. + */ +const SQL_MEAN = + /_avg\s*:\s*\{\s*value\s*:|\bAVG\s*\(\s*(?:[a-z_]+\s*\.\s*)?"?value"?\s*\)|\bSUM\s*\(\s*(?:[a-z_]+\s*\.\s*)?"?total"?\s*\)\s*\/\s*SUM\s*\(\s*(?:[a-z_]+\s*\.\s*)?"?cnt"?\s*\)|\bSUM\s*\(\s*"?count"?\s*\*\s*"?mean"?\s*\)/gi; + +/** In-memory means and the folds that make them, counted in files that name pulse. */ +const TS_MEAN = + /\.reduce\(\s*\(\s*\w+\s*,\s*\w+\s*\)\s*=>\s*\w+\s*\+\s*[\w.]+\s*,\s*0\s*\)\s*\/\s*[\w.]+\.length|\bavgInWindow\s*\(|\bsummarize\s*\(|\bbucketWeekly\s*\(|\bbuildDailyValueRows\s*\(|\bapplyPayloadBudget\s*\(/g; + +const NAMES_PULSE = /\bPULSE\b/; + +type Entry = + /** Averages pulse and imports the shared statistic. */ + | { helper: true } + /** Averages pulse through a fold that imports it, and hands that fold the type. */ + | { via: string; passes: RegExp } + /** Never averages pulse readings; the reason is the claim. */ + | { exempt: string }; + +const HELPER: Entry = { helper: true }; + +/** Files with a database mean over values. */ +const SQL_READERS: Record = { + "app/api/measurements/series/route.ts": HELPER, + "lib/ai/coach/tools/availability.ts": HELPER, + "lib/analytics/summaries-slice.ts": HELPER, + "lib/doctor-report/dense-buckets.ts": HELPER, + "lib/insights/comprehensive-aggregator.ts": HELPER, + "lib/insights/comprehensive-generate.ts": HELPER, + "lib/measurements/daily-series-read.ts": HELPER, + "lib/rollups/measurement-rollups.ts": HELPER, + "lib/rollups/tiered-context.ts": HELPER, +}; + +/** Files that name pulse and hold an in-memory mean or a mean-making fold. */ +const TS_READERS: Record = { + "app/api/measurements/series/route.ts": HELPER, + "lib/doctor-report/collect.ts": HELPER, + "lib/doctor-report-pdf/clinical-summary.ts": HELPER, + "lib/insights/comprehensive-generate.ts": HELPER, + "lib/insights/features.ts": HELPER, + "lib/insights/mood-status.ts": { + via: "lib/insights/bucket-series.ts", + passes: /\{\s*now\s*,\s*tz\s*,\s*type\s*\}/, + }, + "lib/insights/pulse-status.ts": { + via: "lib/insights/bucket-series.ts", + passes: + /applyPayloadBudget\(\s*pulsePoints\s*,\s*\{[^}]*type\s*:\s*"PULSE"/, + }, + "lib/insights/signals-of-day.ts": { + via: "lib/insights/features.ts", + passes: /byType\(\s*"PULSE"\s*\)[\s\S]{0,80}type\s*:\s*"PULSE"/, + }, + "lib/ai/coach/snapshot-blocks/core-metrics-block.ts": { + via: "lib/ai/coach/snapshot-series.ts", + passes: /buildDailyValueRows\([^)]*"PULSE"/, + }, + "components/charts/health-chart.tsx": { + exempt: + "a rolling average over the points the chart draws: display smoothing, not a stated figure", + }, + "lib/analytics/correlations-fast-path.ts": { + exempt: "mood mean; pulse enters as the resting series, never a day mean", + }, + "lib/dashboard/snapshot.ts": { + exempt: "summarises water-intake days and glucose contexts", + }, + "lib/gamification/expansion-metrics.ts": { + exempt: "means of mood day means; pulse is only counted", + }, + "lib/illness/correlation.ts": { exempt: "mean of episode durations" }, + "lib/insights/mood-aggregates.ts": { exempt: "the mood series only" }, + "lib/insights/narrative/period-narrative.ts": { + exempt: "means of day-keyed series, already one value per day", + }, + "lib/mcp/rich-reads.ts": { + exempt: + "changepoints over rollup bucket means, which weigh pulse days once (readCanonicalRollupBuckets)", + }, + "lib/medications/efficacy/build-efficacy.ts": { + exempt: "means of the day-mean series (readDayMeanSeries)", + }, + "lib/targets/build-response.ts": { + exempt: + "pulse is excluded from the 30-day averages; the resting proxy stands in", + }, + "lib/targets/vitals-builder.ts": { + exempt: + "trend halves over the other vitals and the resting proxy's day points", + }, + "app/api/insights/cards/route.ts": { + exempt: "pulse is summarised from day means (readDayMeanSeries)", + }, + "app/api/insights/comprehensive/route.ts": { + exempt: "summarises the mood series only", + }, +}; + +function scan(): { sql: Map; ts: Map } { + const files = walkSourceFiles(SRC, { floor: 1500 }).filter( + (f) => + !f.startsWith("generated/") && + !f.includes("__tests__/") && + !/\.test\.tsx?$/.test(f) && + f !== "lib/measurements/day-mean.ts", + ); + const sql = new Map(); + const ts = new Map(); + for (const f of files) { + const text = stripComments(readFileSync(join(SRC, f), "utf8")); + const s = text.match(SQL_MEAN)?.length ?? 0; + if (s > 0) sql.set(f, s); + if (NAMES_PULSE.test(text)) { + const t = text.match(TS_MEAN)?.length ?? 0; + if (t > 0) ts.set(f, t); + } + } + return { sql, ts }; +} + +const { sql, ts } = scan(); +const source = (f: string) => readFileSync(join(SRC, f), "utf8"); + +function check(found: Map, listed: Record) { + const unlisted = [...found.keys()].filter((f) => !(f in listed)).sort(); + const stale = Object.keys(listed) + .filter((f) => !found.has(f)) + .sort(); + const broken: string[] = []; + for (const [f, entry] of Object.entries(listed)) { + if (!found.has(f)) continue; + if ("helper" in entry && !HELPER_IMPORT.test(source(f))) { + broken.push(`${f}: does not import the day-mean helper`); + } + if ("via" in entry) { + if (!HELPER_IMPORT.test(source(entry.via))) { + broken.push(`${f}: ${entry.via} does not import the day-mean helper`); + } + if (!entry.passes.test(source(f))) { + broken.push(`${f}: no longer hands the type to ${entry.via}`); + } + } + } + return { unlisted, stale, broken }; +} + +describe("pulse day-mean reader guard", () => { + it("finds the SQL means it is meant to find", () => { + expect(sql.size).toBeGreaterThanOrEqual(9); + expect([...sql.values()].reduce((a, b) => a + b, 0)).toBeGreaterThanOrEqual( + 15, + ); + }); + + it("finds the in-memory pulse means it is meant to find", () => { + expect(ts.size).toBeGreaterThanOrEqual(20); + }); + + it("every SQL mean over values is listed, and every listed pulse reader uses the helper", () => { + expect(check(sql, SQL_READERS)).toEqual({ + unlisted: [], + stale: [], + broken: [], + }); + }); + + it("every in-memory pulse mean is listed, and every listed reader uses the helper", () => { + expect(check(ts, TS_READERS)).toEqual({ + unlisted: [], + stale: [], + broken: [], + }); + }); + + it("catches a planted plain AVG and a planted fold", () => { + expect( + `SELECT AVG( m."value" ) FROM measurements m`.match(SQL_MEAN), + ).toHaveLength(1); + expect(`(SUM(c.total) / SUM(c.cnt))`.match(SQL_MEAN)).toHaveLength(1); + expect(`SUM("count" * "mean")`.match(SQL_MEAN)).toHaveLength(1); + expect(`aggregate({ _avg: { value: true } })`.match(SQL_MEAN)).toHaveLength( + 1, + ); + expect(stripComments("// AVG(value) in prose").match(SQL_MEAN)).toBeNull(); + const planted = `const pulse = byType("PULSE"); + const avg = pulse.reduce((s, r) => s + r.value, 0) / pulse.length;`; + expect(NAMES_PULSE.test(planted)).toBe(true); + expect(planted.match(TS_MEAN)).toHaveLength(1); + }); +}); From 57c7cf8be9f14d3dd06f2e6ba29a5caf298e41a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 14:55:00 +0200 Subject: [PATCH 09/10] chore(db): number the pulse mean migration after 0371 --- .../migration.sql | 0 tests/integration/pulse-hourly-mean-migration.test.ts | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename prisma/migrations/{0373_pulse_hourly_mean_day => 0372_pulse_hourly_mean_day}/migration.sql (100%) diff --git a/prisma/migrations/0373_pulse_hourly_mean_day/migration.sql b/prisma/migrations/0372_pulse_hourly_mean_day/migration.sql similarity index 100% rename from prisma/migrations/0373_pulse_hourly_mean_day/migration.sql rename to prisma/migrations/0372_pulse_hourly_mean_day/migration.sql diff --git a/tests/integration/pulse-hourly-mean-migration.test.ts b/tests/integration/pulse-hourly-mean-migration.test.ts index 040aa53fa..053c22c77 100644 --- a/tests/integration/pulse-hourly-mean-migration.test.ts +++ b/tests/integration/pulse-hourly-mean-migration.test.ts @@ -1,5 +1,5 @@ /** - * Migration 0373 rewrites the `mean` of stored PULSE rollup rows to the + * Migration 0372 rewrites the `mean` of stored PULSE rollup rows to the * hourly-mean statistic, and touches nothing else. * * The rows are first built by the writer, then their `mean` is set to a @@ -26,7 +26,7 @@ const MIGRATION = readFileSync( process.cwd(), "prisma", "migrations", - "0373_pulse_hourly_mean_day", + "0372_pulse_hourly_mean_day", "migration.sql", ), "utf8", @@ -131,7 +131,7 @@ const meanOf = (all: Row[], type: string, granularity: string, at?: number) => (at === undefined || r.bucket_start.getTime() === at), )?.mean; -describe("migration 0373", () => { +describe("migration 0372", () => { it("covers all four granularities of both types", () => { for (const type of ["PULSE", "HEART_RATE_VARIABILITY"]) { for (const g of ["DAY", "WEEK", "MONTH", "YEAR"]) { From 28046f05f7a58f14883bd463859dcb6cd875729e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 4 Oct 2026 16:14:10 +0200 Subject: [PATCH 10/10] test(measurements): group the hourly-mean fake once instead of per day The in-memory day-aggregate fake re-filtered every row for every day, converting each timestamp to the profile timezone again, so a 1000-day series took about five seconds and the graded pulse-status test hit its timeout. It now groups by day and segment in one pass (4.8 s to 85 ms). --- .../__tests__/fake-day-aggregates.ts | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/lib/measurements/__tests__/fake-day-aggregates.ts b/src/lib/measurements/__tests__/fake-day-aggregates.ts index dd17f37ef..9e2503dd0 100644 --- a/src/lib/measurements/__tests__/fake-day-aggregates.ts +++ b/src/lib/measurements/__tests__/fake-day-aggregates.ts @@ -81,22 +81,31 @@ export async function fakeReadDayAggregates( // Mirror the SQL's hourly-mean day: the mean of the (day, segment)'s local // hours' means. const starts = opts.segmentStarts ?? []; + // Group once by (day, segment) with the fold's own filters, so a long + // series stays linear instead of re-filtering every row per day. + const membersByKey = new Map< + string, + Array<{ measuredAt: Date; value: number }> + >(); + for (const r of rows) { + const t = r.measuredAt.getTime(); + if (t < opts.since.getTime()) continue; + if (opts.until && t > opts.until.getTime()) continue; + if ( + opts.valueRange && + (r.value < opts.valueRange.min || r.value > opts.valueRange.max) + ) { + continue; + } + const key = `${userDayKey(r.measuredAt, opts.timeZone)}|${ + starts.filter((st) => t < st.getTime()).length + }`; + const list = membersByKey.get(key); + if (list) list.push(r); + else membersByKey.set(key, [r]); + } return days.map((d) => { - const members = rows.filter((r) => { - const t = r.measuredAt.getTime(); - if (t < opts.since.getTime()) return false; - if (opts.until && t > opts.until.getTime()) return false; - if ( - opts.valueRange && - (r.value < opts.valueRange.min || r.value > opts.valueRange.max) - ) { - return false; - } - return ( - userDayKey(r.measuredAt, opts.timeZone) === d.day && - starts.filter((s) => t < s.getTime()).length === d.segment - ); - }); + const members = membersByKey.get(`${d.day}|${d.segment}`) ?? []; const dayMean = dayValue(opts.type, members, opts.timeZone); return dayMean === null ? d : { ...d, dayMean }; });