diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 1ba6ce29e..c4dca333d 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -40748,7 +40748,9 @@ components: type: string format: date-time pattern: ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d+)?)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ - description: Day-bucket start (ISO-8601). + description: "Day-bucket start (ISO-8601): the user's local midnight. A window longer than the daily cap is served from + a coarser tier (a UTC day, week or month); its rows are stamped at noon UTC of the bucket's first + day, which falls inside the bucket's own week or month in every zone." count: description: Underlying raw-row count for the bucket. type: integer diff --git a/messages/de.json b/messages/de.json index 92cf76a1b..a9dd2e313 100644 --- a/messages/de.json +++ b/messages/de.json @@ -9572,6 +9572,7 @@ "systolic": "Systolisch", "diastolic": "Diastolisch", "mean": "Durchschnitt", + "total": "Summe", "min": "Niedrigster", "max": "Höchster", "readings": "Messwerte", diff --git a/messages/en.json b/messages/en.json index 840bb19bb..5c8867cc1 100644 --- a/messages/en.json +++ b/messages/en.json @@ -9572,6 +9572,7 @@ "systolic": "Systolic", "diastolic": "Diastolic", "mean": "Average", + "total": "Total", "min": "Lowest", "max": "Highest", "readings": "Readings", diff --git a/messages/es.json b/messages/es.json index 34d461a2c..934eed626 100644 --- a/messages/es.json +++ b/messages/es.json @@ -9572,6 +9572,7 @@ "systolic": "Sistólica", "diastolic": "Diastólica", "mean": "Promedio", + "total": "Total", "min": "Mínimo", "max": "Máximo", "readings": "Mediciones", diff --git a/messages/fr.json b/messages/fr.json index ee56afde8..fe3e5f3f3 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -9572,6 +9572,7 @@ "systolic": "Systolique", "diastolic": "Diastolique", "mean": "Moyenne", + "total": "Total", "min": "Minimum", "max": "Maximum", "readings": "Mesures", diff --git a/messages/it.json b/messages/it.json index dccfb26e6..607881026 100644 --- a/messages/it.json +++ b/messages/it.json @@ -9572,6 +9572,7 @@ "systolic": "Sistolica", "diastolic": "Diastolica", "mean": "Media", + "total": "Totale", "min": "Minimo", "max": "Massimo", "readings": "Misurazioni", diff --git a/messages/ko.json b/messages/ko.json index e6f1f6ec1..1de539767 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -9572,6 +9572,7 @@ "systolic": "수축기", "diastolic": "이완기", "mean": "평균", + "total": "합계", "min": "최저", "max": "최고", "readings": "측정 횟수", diff --git a/messages/pl.json b/messages/pl.json index dac0acbe9..fc35699df 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -9572,6 +9572,7 @@ "systolic": "Skurczowe", "diastolic": "Rozkurczowe", "mean": "Średnia", + "total": "Suma", "min": "Najniższa", "max": "Najwyższa", "readings": "Pomiary", diff --git a/src/lib/ai/coach/dialog-keys.ts b/src/lib/ai/coach/dialog-keys.ts index 6c4860441..54b01c05d 100644 --- a/src/lib/ai/coach/dialog-keys.ts +++ b/src/lib/ai/coach/dialog-keys.ts @@ -137,6 +137,7 @@ export const COACH_RESULT_COLUMN_KEYS = { systolic: "coach.result.column.systolic", diastolic: "coach.result.column.diastolic", mean: "coach.result.column.mean", + total: "coach.result.column.total", min: "coach.result.column.min", max: "coach.result.column.max", readings: "coach.result.column.readings", diff --git a/src/lib/ai/coach/method.ts b/src/lib/ai/coach/method.ts index f2ef3d41b..1404a2f39 100644 --- a/src/lib/ai/coach/method.ts +++ b/src/lib/ai/coach/method.ts @@ -31,6 +31,7 @@ import { methodReadingsKey, methodTotalsKey, } from "@/lib/ai/coach/dialog-keys"; +import { aggregationKind } from "@/lib/ai/coach/results/chart-spec"; /** More than this and the line stops being a line. */ export const METHOD_MAX_ENTRIES = 6; @@ -38,18 +39,6 @@ export const METHOD_MAX_ENTRIES = 6; type Aggregation = NonNullable; type Absent = NonNullable; -/** - * Metrics whose buckets are totals (a week of steps is summed, not - * averaged). Every other time series is a level, and its buckets are means. - */ -const TOTAL_METRICS: ReadonlySet = new Set([ - "steps", - "active_energy", - "flights", - "distance", - "daylight", -]); - const ABSENT_REASONS: ReadonlySet = new Set([ "no_data", "outside_window", @@ -70,7 +59,7 @@ function aggregationFor(result: CoachResultMeta): Aggregation | undefined { if (domain === "sleep" && result.source.granularity === "day") { return undefined; } - return TOTAL_METRICS.has(domain) ? "sum" : "mean"; + return aggregationKind(domain) === "total" ? "sum" : "mean"; } } } diff --git a/src/lib/ai/coach/results/__tests__/metric-table-tool.test.ts b/src/lib/ai/coach/results/__tests__/metric-table-tool.test.ts index 22910f8b6..3441728d8 100644 --- a/src/lib/ai/coach/results/__tests__/metric-table-tool.test.ts +++ b/src/lib/ai/coach/results/__tests__/metric-table-tool.test.ts @@ -6,6 +6,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const readDailySeries = vi.fn(); +const readLiveBuckets = vi.fn(); +const readCanonicalRollupBuckets = vi.fn(); const moodFindMany = vi.fn(); const measurementFindMany = vi.fn(); @@ -17,9 +19,12 @@ vi.mock("@/lib/db", () => ({ })); vi.mock("@/lib/measurements/daily-series-read", () => ({ readDailySeries: (...a: unknown[]) => readDailySeries(...a), + readLiveBuckets: (...a: unknown[]) => readLiveBuckets(...a), })); vi.mock("@/lib/rollups/measurement-read", () => ({ loadUserSourcePriority: vi.fn(async () => null), + readCanonicalRollupBuckets: (...a: unknown[]) => + readCanonicalRollupBuckets(...a), })); import { bucketTimeSeries } from "@/lib/charts/bucket-time-series"; @@ -54,6 +59,8 @@ function dayRow(type: string, key: string, value: number, count = 1) { beforeEach(() => { readDailySeries.mockReset(); + readLiveBuckets.mockReset(); + readCanonicalRollupBuckets.mockReset(); moodFindMany.mockReset(); measurementFindMany.mockReset(); }); @@ -353,3 +360,249 @@ describe("summariseTable", () => { expect(JSON.stringify(summary)).not.toContain("ignore all previous"); }); }); + +describe("totals and levels", () => { + const WEST = "America/Los_Angeles"; + // 10:00 on 20 September in Los Angeles. + const WEST_NOW = new Date("2026-09-20T17:00:00Z"); + + /** A MONTH bucket of the rollup fold, starting at the UTC month. */ + function monthBucket(month: string, sumValue: number, count: number) { + return { + bucketStart: new Date(`${month}-01T00:00:00.000Z`), + count, + mean: sumValue / count, + sumValue, + minValue: 0, + maxValue: 0, + }; + } + + function allTime(metric: "steps" | "weight" | "audio_event", tz = WEST) { + return readMetricTable({ + userId: "u1", + metric, + window: "allTime", + period: "current", + granularity: undefined, + timeZone: tz, + locale: "en", + ref: "r1", + now: WEST_NOW, + }); + } + + it("sums a total's days into its weeks and labels the column a total", async () => { + const days = Array.from({ length: 30 }, (_, i) => + shiftDateKey("2026-08-29", i), + ); + readDailySeries.mockResolvedValue( + days.map((key, i) => dayRow("ACTIVITY_STEPS", key, 1_000 + i, 50)), + ); + const table = await readMetricTable({ + userId: "u1", + metric: "steps", + window: "last30days", + period: "current", + granularity: "week", + timeZone: TZ, + locale: "en", + ref: "r1", + now: NOW, + }); + expect(table!.columns[1]).toMatchObject({ + key: "value", + labelKey: "coach.result.column.total", + label: "Total", + }); + // Monday 31 August to Sunday 6 September: seven whole days. + const week = table!.rows.find((row) => row[0] === "2026-08-31")!; + const expected = days + .map((key, i) => ({ key, value: 1_000 + i })) + .filter(({ key }) => key >= "2026-08-31" && key <= "2026-09-06") + .reduce((sum, { value }) => sum + value, 0); + expect(week[1]).toBe(expected); + expect(week[2]).toBe(7 * 50); + }); + + it("keeps a level's weeks as means", async () => { + readDailySeries.mockResolvedValue([ + dayRow("WEIGHT", "2026-09-21", 80), + dayRow("WEIGHT", "2026-09-22", 82), + ]); + const table = await readMetricTable({ + userId: "u1", + metric: "weight", + window: "last30days", + period: "current", + granularity: "week", + timeZone: TZ, + locale: "en", + ref: "r1", + now: NOW, + }); + expect(table!.columns[1].labelKey).toBe("coach.result.column.mean"); + expect(table!.rows.find((row) => row[0] === "2026-09-21")![1]).toBe(81); + }); + + it("reads all time as UTC months, named for their own month west of UTC", async () => { + readCanonicalRollupBuckets.mockResolvedValue([ + monthBucket("2026-01", 240_000, 31), + monthBucket("2026-02", 200_000, 28), + monthBucket("2026-03", 310_000, 31), + ]); + const table = (await allTime("steps"))!; + const call = readCanonicalRollupBuckets.mock.calls[0][0]; + expect(call.granularity).toBe("MONTH"); + // The first whole UTC month of the range: the tier drops a bucket that + // starts before `from`. + expect(call.from.toISOString().slice(8)).toBe("01T00:00:00.000Z"); + const byMonth = new Map(table.rows.map((row) => [row[0], row])); + // 1 January 00:00 UTC is 31 December in Los Angeles: re-reading it in + // the user's zone put January's total under December. + expect(table.rows[0]).toEqual(["2026-01", 240_000, 31]); + expect(byMonth.has("2025-12")).toBe(false); + expect(byMonth.get("2026-03")).toEqual(["2026-03", 310_000, 31]); + expect(table.columns[1].labelKey).toBe("coach.result.column.total"); + expect(readLiveBuckets).not.toHaveBeenCalled(); + }); + + it("gives the same all-time months when the live table serves them", async () => { + readCanonicalRollupBuckets.mockResolvedValue([ + monthBucket("2026-01", 240_000, 31), + monthBucket("2026-02", 200_000, 28), + ]); + const fromTier = (await allTime("steps"))!; + + readCanonicalRollupBuckets.mockResolvedValue([]); + readLiveBuckets.mockResolvedValue([ + { + type: "ACTIVITY_STEPS", + value: 240_000, + measuredAt: "2026-01-01T00:00:00.000Z", + count: 31, + }, + { + type: "ACTIVITY_STEPS", + value: 200_000, + measuredAt: "2026-02-01T00:00:00.000Z", + count: 28, + }, + ]); + const fromLive = (await allTime("steps"))!; + expect(fromLive.rows).toEqual(fromTier.rows); + const liveCall = readLiveBuckets.mock.calls[0][0]; + expect(liveCall).toMatchObject({ grain: "monthly", timeZone: "UTC" }); + }); + + it("gives the same month whatever the history length, a year or ten", async () => { + // A short history and a long one hold the same January: the month reads + // the same total from both, because all time always reads months. + readCanonicalRollupBuckets.mockResolvedValue([ + monthBucket("2026-01", 240_000, 31), + ]); + const short = (await allTime("steps"))!; + readCanonicalRollupBuckets.mockResolvedValue([ + monthBucket("2017-03", 150_000, 31), + monthBucket("2026-01", 240_000, 31), + ]); + const long = (await allTime("steps"))!; + const jan = (t: typeof short) => t.rows.find((row) => row[0] === "2026-01"); + expect(jan(long)).toEqual(jan(short)); + expect(jan(short)).toEqual(["2026-01", 240_000, 31]); + }); + + it("averages a level's all-time months", async () => { + readCanonicalRollupBuckets.mockResolvedValue([ + { ...monthBucket("2026-01", 2_400, 30), mean: 80 }, + ]); + const table = (await allTime("weight"))!; + expect(table.rows[0]).toEqual(["2026-01", 80, 30]); + expect(table.columns[1].labelKey).toBe("coach.result.column.mean"); + }); + + it("counts loud-sound events as a total, in whole events", async () => { + // Every event row is 1; the daily reader averages the type, so a day of + // four events reads 1 with a count of 4. + readDailySeries.mockResolvedValue([ + dayRow("AUDIO_EXPOSURE_EVENT", "2026-09-25", 1, 4), + ]); + const table = await readMetricTable({ + userId: "u1", + metric: "audio_event", + window: "last7days", + period: "current", + granularity: undefined, + timeZone: TZ, + locale: "en", + ref: "r1", + now: NOW, + }); + const day = table!.rows.find((row) => row[0] === "2026-09-25")!; + expect(day[1]).toBe(4); + expect(table!.columns[1].decimals).toBe(0); + }); + + it("shows a step length to the centimetre", async () => { + readDailySeries.mockResolvedValue([ + dayRow("WALKING_STEP_LENGTH", "2026-09-25", 0.724), + ]); + const table = await readMetricTable({ + userId: "u1", + metric: "walking_step_length", + window: "last7days", + period: "current", + granularity: undefined, + timeZone: TZ, + locale: "en", + ref: "r1", + now: NOW, + }); + expect(table!.columns[1].decimals).toBe(2); + const summary = summariseTable(table!); + expect((summary.rows as unknown[][])[0][1]).toBe(0.72); + }); + + it("leads HRV with the estimator the person mostly has and counts a night once", async () => { + readDailySeries.mockImplementation(async ({ type }: { type: string }) => + type === "HRV_RMSSD" + ? [ + dayRow("HRV_RMSSD", "2026-09-24", 40), + dayRow("HRV_RMSSD", "2026-09-25", 42), + ] + : [dayRow("HEART_RATE_VARIABILITY", "2026-09-25", 55)], + ); + const table = await readMetricTable({ + userId: "u1", + metric: "hrv", + window: "last7days", + period: "current", + granularity: undefined, + timeZone: TZ, + locale: "en", + ref: "r1", + now: NOW, + }); + expect(table!.columns.map((c) => c.key)).toEqual([ + "day", + "rmssd", + "sdnn", + "readings", + ]); + const byDay = new Map(table!.rows.map((row) => [row[0], row])); + // Both estimators read on the 25th: one night, one reading. + expect(byDay.get("2026-09-25")).toEqual(["2026-09-25", 42, 55, 1]); + }); + + it("tells the model a total's periods are totals and what they add up to", async () => { + readCanonicalRollupBuckets.mockResolvedValue([ + monthBucket("2026-01", 240_000, 31), + monthBucket("2026-02", 200_000, 28), + ]); + const summary = summariseTable((await allTime("steps"))!); + expect(summary.valuesAre).toEqual({ value: "total per month" }); + expect(summary.stats).toMatchObject({ + value: { total: 440_000, mean: 220_000 }, + }); + }); +}); diff --git a/src/lib/ai/coach/results/chart-spec.ts b/src/lib/ai/coach/results/chart-spec.ts index 02151bf10..841908029 100644 --- a/src/lib/ai/coach/results/chart-spec.ts +++ b/src/lib/ai/coach/results/chart-spec.ts @@ -86,7 +86,12 @@ export const METRIC_AGGREGATION_KIND: Readonly< workouts: "total", }; -function aggregationKind(domain: CoachStepDomain): "level" | "total" { +/** + * Whether a metric's periods are totals or levels. The table tool folds a + * week or month with it (a sum or a mean), the method line names it, and the + * chart draws it: one answer for all three. + */ +export function aggregationKind(domain: CoachStepDomain): "level" | "total" { return ( (METRIC_AGGREGATION_KIND as Partial>)[ domain @@ -124,6 +129,14 @@ function columnIndex(table: ChartInput, key: string): number { return table.columns.findIndex((column) => column.key === key); } +/** + * Metrics whose value columns are alternative estimators of one quantity + * (HRV as SDNN and as RMSSD), not two sides of one reading: the chart draws + * the first, which the table tool orders as the one with more readings. + */ +const SINGLE_SERIES_DOMAINS: ReadonlySet = + new Set(["hrv"]); + /** * The series a time series is drawn with: its value columns, and only the * first one plus one partner in the same unit — two axes with different @@ -137,6 +150,7 @@ function timeSeriesColumns(table: ChartInput): CoachResultColumn[] { : table.columns.filter((column) => column.kind === "count"); const [first, ...rest] = candidates; if (!first) return []; + if (SINGLE_SERIES_DOMAINS.has(table.source.domain)) return [first]; const partner = rest.find((column) => column.unit === first.unit); return partner ? [first, partner] : [first]; } diff --git a/src/lib/ai/coach/results/metric-table-tool.ts b/src/lib/ai/coach/results/metric-table-tool.ts index d85774a65..73fa54545 100644 --- a/src/lib/ai/coach/results/metric-table-tool.ts +++ b/src/lib/ai/coach/results/metric-table-tool.ts @@ -4,11 +4,20 @@ * * The numbers are the app's chart numbers, read the way the charts read * them: `readDailySeries` cuts the user's own local days and collapses - * overlapping sources to the ladder-canonical one, and weeks and months are - * folded from those days by `bucketTimeSeries` in the user's zone. Mood is - * the mean score of each local day (the key its entries were written under); - * sleep is time asleep per night, the wake-day key `reconstructSleepNights` - * gives every other sleep surface. + * overlapping sources to the ladder-canonical one. Mood is the mean score of + * each local day (the key its entries were written under); sleep is time + * asleep per night, the wake-day key `reconstructSleepNights` gives every + * other sleep surface. + * + * A week or month means one thing per metric, whatever serves it. A total + * (steps, energy, distance, daylight, loud-sound events: `aggregationKind`) + * is the sum of its days, labelled as a total; a level is the mean of its + * days, folded by `bucketTimeSeries` in the user's zone as the chart folds + * it. All time is read as calendar months straight from the MONTH fold of + * the rollup tier (or the same fold of the live table when the tier has no + * rows), keyed by the UTC month the bucket starts in: the tier's buckets are + * UTC months, and re-reading their start in a zone west of UTC named the + * month before. * * Every period of the range gets a row. A period without a reading carries * `null`, never a zero, so absence stays explicit in the table and in the @@ -26,8 +35,16 @@ import type { Locale } from "@/lib/i18n/config"; import { getServerTranslator } from "@/lib/i18n/server-translator"; import { shiftDateKey, userDayKey } from "@/lib/tz/format"; import { startOfLocalDayKey } from "@/lib/tz/local-day"; -import { readDailySeries } from "@/lib/measurements/daily-series-read"; -import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { + readDailySeries, + readLiveBuckets, +} from "@/lib/measurements/daily-series-read"; +import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; +import { + loadUserSourcePriority, + readCanonicalRollupBuckets, +} from "@/lib/rollups/measurement-read"; +import { annotate } from "@/lib/logging/context"; import { bucketTimeSeries } from "@/lib/charts/bucket-time-series"; import { reconstructSleepNights } from "@/lib/analytics/sleep-night"; import { getUnitForType } from "@/lib/validations/measurement"; @@ -47,6 +64,7 @@ import type { CoachScopeWindow, } from "@/lib/ai/coach/types"; +import { aggregationKind } from "./chart-spec"; import { RESULT_TABLE_MAX_ROWS } from "./project"; /** @@ -197,6 +215,8 @@ interface DaySeries { column: Omit; /** Local day key → the day's value. */ byDay: Map; + /** How days fold into a week or month: a total sums, a level averages. */ + fold: "sum" | "mean"; } /** Per-day values and reading counts for a metric, before bucketing. */ @@ -211,6 +231,22 @@ const UNIT_TOKENS: Readonly> = { celsius: "°C", }; +/** + * Decimals for the types whose unit alone decides wrongly: a step length of + * 0.72 m is not 1 m, a walking speed of 1.35 m/s is not 1.4, and a count of + * loud-sound events has no fraction. + */ +const DECIMALS_BY_TYPE: Readonly>> = { + WALKING_STEP_LENGTH: 2, + WALKING_SPEED: 2, + AUDIO_EXPOSURE_EVENT: 0, +}; + +/** Decimals a value is shown with: the type's own, else by unit. */ +function decimalsForType(type: MeasurementType, unit: string): number { + return DECIMALS_BY_TYPE[type] ?? decimalsFor(unit); +} + /** Decimals a value is shown with, by unit. */ function decimalsFor(unit: string): number { switch (unit) { @@ -243,11 +279,31 @@ const SERIES_SUFFIX: Readonly>> = { HRV_RMSSD: "RMSSD", }; +type MeasurementColumn = DaySeries["column"] & { + type: MeasurementType; + suffix?: string; +}; + +/** + * A type's value for one day (or month) from a series row. The daily reader + * sums the cumulative types and averages every other one; a total the reader + * averages (loud-sound events, each row one event) is its mean times its + * readings. + */ +function totalOrLevel( + type: MeasurementType, + total: boolean, + value: number, + count: number, +): number { + return total && !CUMULATIVE_HK_TYPES.has(type) ? value * count : value; +} + function measurementColumns( metric: CoachScopeSource, types: readonly MeasurementType[], granularity: CoachResultGranularity, -): Array { +): MeasurementColumn[] { if (metric === "bp") { return [ { @@ -271,7 +327,9 @@ function measurementColumns( const labelKey = granularity === "day" ? COACH_RESULT_COLUMN_KEYS.value - : COACH_RESULT_COLUMN_KEYS.mean; + : aggregationKind(metric) === "total" + ? COACH_RESULT_COLUMN_KEYS.total + : COACH_RESULT_COLUMN_KEYS.mean; return types.map((type) => { const unit = unitTokenFor(type); const suffix = types.length > 1 ? (SERIES_SUFFIX[type] ?? type) : undefined; @@ -280,13 +338,37 @@ function measurementColumns( kind: "number" as const, labelKey, ...(unit ? { unit } : {}), - decimals: decimalsFor(unit ?? ""), + decimals: decimalsForType(type, unit ?? ""), type, ...(suffix ? { suffix } : {}), }; }); } +/** + * Put the series with the most readings first and count each period's + * readings from the first series that has it. Blood pressure's two sides are + * one reading and keep their order (systolic first); HRV's two estimators + * are alternatives, so the one the person mostly has leads the table and + * the chart, and a night measured by both is one night, not two. + */ +function orderAndCount( + metric: CoachScopeSource, + series: Array }>, +): { ordered: DaySeries[]; counts: Map } { + const total = (s: { counts: Map }) => + [...s.counts.values()].reduce((sum, v) => sum + v, 0); + const ordered = + metric === "bp" ? series : [...series].sort((a, b) => total(b) - total(a)); + const counts = new Map(); + for (const s of ordered) { + for (const [key, count] of s.counts) { + if (!counts.has(key)) counts.set(key, count); + } + } + return { ordered: ordered.map(({ counts: _c, ...rest }) => rest), counts }; +} + async function readMeasurementDays(args: { userId: string; metric: CoachScopeSource; @@ -310,23 +392,148 @@ async function readMeasurementDays(args: { }), ), ); - const counts = new Map(); + const total = aggregationKind(metric) === "total"; const suffixes = new Map(); - const series: DaySeries[] = columns.map((column, index) => { + const read = columns.map((column, index) => { const byDay = new Map(); + const counts = new Map(); for (const row of rowsByType[index]) { const day = userDayKey(new Date(row.measuredAt), timeZone); - byDay.set(day, row.value); - // Paired readings count once: blood pressure counts its systolic side. - if (metric !== "bp" || index === 0) { - counts.set(day, (counts.get(day) ?? 0) + (row.count ?? 1)); - } + const count = row.count ?? 1; + byDay.set(day, totalOrLevel(column.type, total, row.value, count)); + counts.set(day, (counts.get(day) ?? 0) + count); } if (column.suffix) suffixes.set(column.key, column.suffix); const { type: _type, suffix: _suffix, ...rest } = column; - return { column: rest, byDay }; + return { + column: rest, + byDay, + fold: total ? ("sum" as const) : ("mean" as const), + counts, + }; }); - return { series, counts, suffixes }; + const { ordered, counts } = orderAndCount(metric, read); + return { series: ordered, counts, suffixes }; +} + +/** `YYYY-MM-DD` of an instant in UTC. */ +function utcDayKey(date: Date): string { + return date.toISOString().slice(0, 10); +} + +/** Folded periods: period key → one value per series, and readings. */ +interface PeriodValues { + values: Map>; + counts: Map; +} + +/** + * All time for a measurement metric: one row per UTC calendar month, a total + * summed and a level averaged over every reading of the month's canonical + * days. The rollup tier's MONTH fold serves it; when the tier holds nothing + * for the type (a backfill not caught up, a read that failed), the live table + * is folded into the same UTC months the same way. Either path gives the + * same number for the same month, so the table no longer depends on how + * long the history is or which tier the chart would pick for it. + */ +async function readMeasurementMonths(args: { + userId: string; + metric: CoachScopeSource; + range: TableRange; +}): Promise<{ + series: DaySeries[]; + periods: PeriodValues; + suffixes: Map; +}> { + const { userId, metric, range } = args; + const types = COACH_SOURCE_MEASUREMENT_TYPES[metric]; + const columns = measurementColumns(metric, types, "month"); + const total = aggregationKind(metric) === "total"; + const priorityJson = await loadUserSourcePriority(userId); + // The first whole UTC month of the range: a bucket that starts before + // `from` is left out of the tier's fold. + const from = new Date(`${utcDayKey(range.from).slice(0, 7)}-01T00:00:00Z`); + const byType = await Promise.all( + columns.map(async (column) => { + const months = new Map(); + try { + const buckets = await readCanonicalRollupBuckets({ + userId, + type: column.type, + granularity: "MONTH", + from, + to: range.to, + toInclusive: true, + userPriorityJson: priorityJson, + }); + for (const b of buckets) { + // The fold's sum is every reading's value added up, which is the + // total for a cumulative type and for an event count alike. + months.set(utcDayKey(b.bucketStart).slice(0, 7), { + value: total ? (b.sumValue ?? b.mean * b.count) : b.mean, + count: b.count, + }); + } + } catch (err) { + annotate({ + meta: { + coach_table_rollup_read_threw: true, + type: column.type, + error: err instanceof Error ? err.message : String(err), + }, + }); + } + if (months.size === 0) { + const rows = await readLiveBuckets({ + userId, + type: column.type, + from, + to: range.to, + cap: 1_000, + priorityJson, + grain: "monthly", + timeZone: "UTC", + }); + for (const row of rows) { + const count = row.count ?? 1; + months.set(row.measuredAt.slice(0, 7), { + value: totalOrLevel(column.type, total, row.value, count), + count, + }); + } + } + return months; + }), + ); + + const suffixes = new Map(); + const read = columns.map((column, index) => { + if (column.suffix) suffixes.set(column.key, column.suffix); + const { type: _type, suffix: _suffix, ...rest } = column; + const byDay = new Map(); + const counts = new Map(); + for (const [month, { value, count }] of byType[index]) { + byDay.set(month, value); + counts.set(month, count); + } + return { + column: rest, + byDay, + fold: total ? ("sum" as const) : ("mean" as const), + counts, + }; + }); + const { ordered, counts } = orderAndCount(metric, read); + const values = new Map>(); + for (const s of ordered) { + for (const month of s.byDay.keys()) { + values.set( + month, + ordered.map((x) => x.byDay.get(month)), + ); + } + } + return { series: ordered, periods: { values, counts }, suffixes }; } async function readMoodDays(args: { @@ -362,6 +569,7 @@ async function readMoodDays(args: { decimals: 1, }, byDay, + fold: "mean", }, ], counts, @@ -415,6 +623,7 @@ async function readSleepDays(args: { decimals: 0, }, byDay, + fold: "mean", }, ], counts, @@ -424,18 +633,16 @@ async function readSleepDays(args: { // ── Bucketing ─────────────────────────────────────────────────────────── /** - * Fold per-day values into the table's periods. Days stay as they are; - * weeks and months go through `bucketTimeSeries` in the user's zone — the - * chart's own fold, so a weekly mean here is the chart's weekly point. + * Fold per-day values into the table's periods. Days stay as they are. A + * level's weeks and months go through `bucketTimeSeries` in the user's zone + * — the chart's own fold, so a weekly mean here is the chart's weekly point; + * a total's are the sum of its local days. */ function foldIntoPeriods(args: { days: MetricDays; granularity: CoachResultGranularity; timeZone: string; -}): { - values: Map>; - counts: Map; -} { +}): PeriodValues { const { days, granularity, timeZone } = args; const values = new Map>(); const counts = new Map(); @@ -455,8 +662,18 @@ function foldIntoPeriods(args: { } return { values, counts }; } + const sums = new Map>(); const points = new Map>(); days.series.forEach((s, index) => { + if (s.fold === "sum") { + for (const [day, value] of s.byDay) { + const key = periodKeyOf(day, granularity); + const slot = sums.get(key) ?? new Map(); + slot.set(index, (slot.get(index) ?? 0) + value); + sums.set(key, slot); + } + return; + } for (const [day, value] of s.byDay) { const point = points.get(day) ?? {}; point[String(index)] = value; @@ -470,12 +687,22 @@ function foldIntoPeriods(args: { })), { bucket: granularity, timeZone }, ); + const levels = new Map>(); for (const point of bucketed.points) { const dayKey = dayKeyOfUtc(point.timestamp); - const key = granularity === "month" ? monthKeyOf(dayKey) : dayKey; + levels.set( + granularity === "month" ? monthKeyOf(dayKey) : dayKey, + point.values, + ); + } + for (const key of new Set([...levels.keys(), ...sums.keys()])) { values.set( key, - days.series.map((_, index) => point.values[String(index)]), + days.series.map((s, index) => + s.fold === "sum" + ? sums.get(key)?.get(index) + : levels.get(key)?.[String(index)], + ), ); } return { values, counts }; @@ -522,11 +749,24 @@ export async function readMetricTable(args: { const range = resolveTableRange({ window, period, timeZone, now }); let suffixes = new Map(); - let days: MetricDays; - if (metric === "mood") { - days = await readMoodDays({ userId, range }); - } else if (metric === "sleep") { - days = await readSleepDays({ userId, range, timeZone }); + let series: DaySeries[]; + let folded: PeriodValues; + // All time of a measurement metric is keyed by UTC month (the tier's own + // buckets); every other table by the user's local days. + let keyRange: [string, string] = [range.fromKey, range.toKey]; + if (metric === "mood" || metric === "sleep") { + const days = + metric === "mood" + ? await readMoodDays({ userId, range }) + : await readSleepDays({ userId, range, timeZone }); + series = days.series; + folded = foldIntoPeriods({ days, granularity, timeZone }); + } else if (window === "allTime") { + const read = await readMeasurementMonths({ userId, metric, range }); + series = read.series; + folded = read.periods; + suffixes = read.suffixes; + keyRange = [utcDayKey(range.from), utcDayKey(range.to)]; } else { const read = await readMeasurementDays({ userId, @@ -535,16 +775,17 @@ export async function readMetricTable(args: { range, timeZone, }); - days = read; + series = read.series; + folded = foldIntoPeriods({ days: read, granularity, timeZone }); suffixes = read.suffixes; } - const { values, counts } = foldIntoPeriods({ days, granularity, timeZone }); + const { values, counts } = folded; if (values.size === 0) return null; // All time starts at the first period that holds a reading, not ten // years of empty months before it. - const keys = periodKeys(range.fromKey, range.toKey, granularity); + const keys = periodKeys(keyRange[0], keyRange[1], granularity); const firstWithData = window === "allTime" ? keys.findIndex((key) => values.has(key)) : 0; const periods = keys.slice(Math.max(0, firstWithData)); @@ -566,7 +807,7 @@ export async function readMetricTable(args: { : PERIOD_COLUMN_KEYS[granularity], ), }, - ...days.series.map((s) => { + ...series.map((s) => { const base = t(s.column.labelKey); const suffix = suffixes.get(s.column.key); return { ...s.column, label: suffix ? `${base} (${suffix})` : base }; @@ -582,7 +823,7 @@ export async function readMetricTable(args: { const cells = values.get(key); return [ key, - ...days.series.map((_, index) => { + ...series.map((_, index) => { const value = cells?.[index]; return value === undefined || !Number.isFinite(value) ? null : value; }), @@ -659,6 +900,19 @@ export function summariseTable( : valueIndexes.some((index) => typeof row[index] === "number"), ); + // What a week's or month's value is: a total of its days or their mean. + // Without it a monthly figure of 240 000 steps reads as a daily count. + const valuesAre: Record = {}; + for (const { column } of numeric) { + if (column.kind !== "number") continue; + const per = table.source.granularity ?? "day"; + if (column.labelKey === COACH_RESULT_COLUMN_KEYS.total) { + valuesAre[column.key] = `total per ${per}`; + } else if (column.labelKey === COACH_RESULT_COLUMN_KEYS.mean) { + valuesAre[column.key] = `average per ${per}`; + } + } + const stats: Record = {}; for (const { column, index } of numeric) { const cells = table.rows @@ -670,11 +924,16 @@ export function summariseTable( stats[column.key] = { total: cells.reduce((s, v) => s + v, 0) }; continue; } + const sum = cells.reduce((s, v) => s + v, 0); stats[column.key] = { n: cells.length, - mean: round(cells.reduce((s, v) => s + v, 0) / cells.length, decimals), + mean: round(sum / cells.length, decimals), min: round(Math.min(...cells), decimals), max: round(Math.max(...cells), decimals), + // A total's periods add up to the range's total; a level's do not. + ...(column.labelKey === COACH_RESULT_COLUMN_KEYS.total + ? { total: round(sum, decimals) } + : {}), }; } @@ -684,6 +943,7 @@ export function summariseTable( source: table.source, shape: table.shape, columns: table.columns.map(columnName), + ...(Object.keys(valuesAre).length > 0 ? { valuesAre } : {}), periods: table.rowCount, periodsWithReadings: withValues.length, ...(withValues.length > 0 diff --git a/src/lib/insights/__tests__/coach-result-clipboard.test.ts b/src/lib/insights/__tests__/coach-result-clipboard.test.ts index cf5d99d81..c9ab965c7 100644 --- a/src/lib/insights/__tests__/coach-result-clipboard.test.ts +++ b/src/lib/insights/__tests__/coach-result-clipboard.test.ts @@ -39,13 +39,27 @@ describe("sanitiseCell", () => { expect(sanitiseCell(input)).toBe(expected); }); - it("turns a leading tab into a space, which no spreadsheet reads as a formula", () => { - expect(sanitiseCell("\t=1")).toBe(" =1"); - }); + it.each([ + ["\t=1", "' =1"], + [" =1+1", "' =1+1"], + ["\u00a0@SUM(A1)", "'\u00a0@SUM(A1)"], + ["\uff1d1+1", "'\uff1d1+1"], + ["\uff0b1", "'\uff0b1"], + ["\uff0d1+1", "'\uff0d1+1"], + ["\uff20SUM(A1)", "'\uff20SUM(A1)"], + ["|cmd", "'|cmd"], + [" |cmd", "' |cmd"], + ])( + "neutralises a formula start behind whitespace or in fullwidth: %j", + (input, expected) => { + expect(sanitiseCell(input)).toBe(expected); + }, + ); it("leaves plain negatives and ordinary text alone", () => { expect(sanitiseCell("-4.5")).toBe("-4.5"); expect(sanitiseCell("-1 234,5")).toBe("-1 234,5"); + expect(sanitiseCell(" -4.5")).toBe(" -4.5"); expect(sanitiseCell("LDL")).toBe("LDL"); expect(sanitiseCell(null)).toBe(""); }); diff --git a/src/lib/insights/coach-result-clipboard.ts b/src/lib/insights/coach-result-clipboard.ts index 1a86f2b9d..ccfa8b564 100644 --- a/src/lib/insights/coach-result-clipboard.ts +++ b/src/lib/insights/coach-result-clipboard.ts @@ -6,8 +6,9 @@ * The cells arrive already formatted the way the table shows them. Two * things are done to them on the way out: * - * - Formula injection. A cell that starts with `=`, `+`, `-` or `@` (or a - * tab / carriage return) is read by a spreadsheet as a formula, and a + * - Formula injection. A cell that starts with `=`, `+`, `-`, `@` or `|` + * (their fullwidth forms too, and behind leading whitespace, which some + * spreadsheets skip) is read as a formula or a command, and a * table can carry text the person did not type — a lab analyte read off * an uploaded report. Such a cell gets a leading `'`, which spreadsheets * treat as "this is text". A plain negative number is left alone. @@ -28,8 +29,8 @@ export interface ClipboardGrid { alignRight?: ReadonlyArray; } -const FORMULA_START = /^[=+\-@\t\r]/; -const NEGATIVE_NUMBER = /^-\s?\d[\d\s.,  ]*$/; +const FORMULA_START = /^\s*[=+\-@|=+-@]/; +const NEGATIVE_NUMBER = /^\s*-\s?\d[\d\s.,  ]*$/; /** A cell with its separators flattened and a formula start neutralised. */ export function sanitiseCell(value: string | null): string { diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index d07c7cfec..103bea081 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -763,7 +763,9 @@ const seriesBatchRowSchema = z.object({ value: z.number().describe("Daily mean (spot metrics) or SUM (cumulative)."), measuredAt: z.iso .datetime({ offset: true }) - .describe("Day-bucket start (ISO-8601)."), + .describe( + "Day-bucket start (ISO-8601): the user's local midnight. A window longer than the daily cap is served from a coarser tier (a UTC day, week or month); its rows are stamped at noon UTC of the bucket's first day, which falls inside the bucket's own week or month in every zone.", + ), count: z .number() .int() diff --git a/src/lib/rollups/__tests__/measurement-read-wmy.test.ts b/src/lib/rollups/__tests__/measurement-read-wmy.test.ts index bfca576c4..d436b6408 100644 --- a/src/lib/rollups/__tests__/measurement-read-wmy.test.ts +++ b/src/lib/rollups/__tests__/measurement-read-wmy.test.ts @@ -53,7 +53,7 @@ import { readTieredRollupSeries, type RollupBucketRow, } from "../measurement-read-wmy"; -import { pickBucket } from "@/lib/charts/bucket-time-series"; +import { bucketTimeSeries, pickBucket } from "@/lib/charts/bucket-time-series"; const { findMany, aggregate, queryRaw } = mocks; const DAY_MS = 86_400_000; @@ -459,7 +459,7 @@ describe("readTieredRollupSeries", () => { expect(result?.granularity).toBe("MONTH"); expect(sqlOf(queryRaw.mock.calls[0])).toContain("'month'"); - expect(result?.rows[0].measuredAt).toBe("2017-01-01T00:00:00.000Z"); + expect(result?.rows[0].measuredAt).toBe("2017-01-01T12:00:00.000Z"); expect(result?.rows).toHaveLength(2); }); @@ -516,7 +516,7 @@ describe("readTieredRollupSeries", () => { }); expect(result?.granularity).toBe("DAY"); expect(queryRaw).not.toHaveBeenCalled(); - expect(result?.rows[0].measuredAt).toBe("2026-02-03T00:00:00.000Z"); + expect(result?.rows[0].measuredAt).toBe("2026-02-03T12:00:00.000Z"); }); it("serves WEEK buckets for a span between one and two years", async () => { @@ -535,6 +535,74 @@ describe("readTieredRollupSeries", () => { expect(sqlOf(queryRaw.mock.calls[0])).toContain("'week'"); }); }); + + // The chart names a row by the date its `measuredAt` falls on in the + // person's zone and folds it into that week or month. A bucket stamped at + // UTC midnight read as the day before west of UTC, so every month of the + // "All" range sat one month early there. + describe("row stamp", () => { + const ZONES = [ + "Pacific/Pago_Pago", + "America/Los_Angeles", + "UTC", + "Europe/Berlin", + "Pacific/Auckland", + "Pacific/Kiritimati", + ]; + + it("puts each month in its own month in every zone", async () => { + aggregate.mockResolvedValueOnce( + spanOf("2023-01-01T00:00:00.000Z", "2026-03-01T00:00:00.000Z"), + ); + queryRaw.mockResolvedValueOnce([ + folded("2026-01-01T00:00:00.000Z", [["2026-01-01", 80]]), + folded("2026-02-01T00:00:00.000Z", [["2026-02-01", 81]]), + folded("2026-03-01T00:00:00.000Z", [["2026-03-01", 82]]), + ]); + const result = await readTieredRollupSeries({ + userId: "u", + type: "WEIGHT", + ...win(3650), + }); + expect(result?.granularity).toBe("MONTH"); + for (const timeZone of ZONES) { + const months = bucketTimeSeries( + result!.rows.map((row) => ({ + timestamp: new Date(row.measuredAt), + values: { v: row.value }, + })), + { bucket: "month", timeZone }, + ).points.map((p) => new Date(p.timestamp).toISOString().slice(0, 7)); + expect(months, timeZone).toEqual(["2026-01", "2026-02", "2026-03"]); + } + }); + + it("puts each week in its own ISO week in every zone", async () => { + aggregate.mockResolvedValueOnce( + spanOf("2025-01-01T00:00:00.000Z", "2026-05-01T00:00:00.000Z"), + ); + queryRaw.mockResolvedValueOnce([ + folded("2025-12-29T00:00:00.000Z", [["2025-12-29", 80]]), + folded("2026-01-05T00:00:00.000Z", [["2026-01-05", 81]]), + ]); + const result = await readTieredRollupSeries({ + userId: "u", + type: "WEIGHT", + ...win(3650), + }); + expect(result?.granularity).toBe("WEEK"); + for (const timeZone of ZONES) { + const weeks = bucketTimeSeries( + result!.rows.map((row) => ({ + timestamp: new Date(row.measuredAt), + values: { v: row.value }, + })), + { bucket: "week", timeZone }, + ).points.map((p) => new Date(p.timestamp).toISOString().slice(0, 10)); + expect(weeks, timeZone).toEqual(["2025-12-29", "2026-01-05"]); + } + }); + }); }); // v1.37.29 — the server tier ladder and the client caption ladder diff --git a/src/lib/rollups/measurement-read-wmy.ts b/src/lib/rollups/measurement-read-wmy.ts index d40941945..8d353ad06 100644 --- a/src/lib/rollups/measurement-read-wmy.ts +++ b/src/lib/rollups/measurement-read-wmy.ts @@ -285,6 +285,17 @@ export function pickRollupGranularityForWindow( return "DAY"; } +/** + * A tier bucket starts at UTC midnight, and a client names a row by the + * date its `measuredAt` falls on in the person's zone. West of UTC that + * midnight is still the evening before, so a March bucket read as February + * and every month of the chart's "All" range sat one month early. A row is + * therefore stamped at noon UTC of the bucket's first day: that instant falls + * inside the bucket's own week, month and year in every zone, and on the + * bucket's own day from UTC−12 to UTC+11. + */ +const TIER_ROW_ANCHOR_MS = 12 * 3_600_000; + /** DAY → WEEK → MONTH → YEAR, finest first. */ const TIER_ORDER: RollupGranularity[] = ["DAY", "WEEK", "MONTH", "YEAR"]; @@ -399,7 +410,9 @@ export async function readTieredRollupSeries(opts: { rows: rows.map((r) => ({ type, value: useSum ? (r.sumValue ?? r.mean * r.count) : r.mean, - measuredAt: r.bucketStart.toISOString(), + measuredAt: new Date( + r.bucketStart.getTime() + TIER_ROW_ANCHOR_MS, + ).toISOString(), count: r.count, minValue: useSum ? undefined : r.minValue, maxValue: useSum ? undefined : r.maxValue,