diff --git a/src/lib/measurements/__tests__/daily-series-read-tiered.test.ts b/src/lib/measurements/__tests__/daily-series-read-tiered.test.ts new file mode 100644 index 000000000..dca7b0525 --- /dev/null +++ b/src/lib/measurements/__tests__/daily-series-read-tiered.test.ts @@ -0,0 +1,157 @@ +/** + * v1.19.2 W-CHARTS — `readDailySeries` long-range tier step-up. + * + * Pins that a window WIDER than the DAY bucket cap routes through the + * tiered rollup reader (whole-history coverage, downsampled by tier) + * instead of the DAY path that would `LIMIT`/`slice` to the cap and + * silently drop the older history. Short / normal windows must NOT touch + * the tiered reader, so the common case stays byte-identical with the + * pre-v1.19.2 daily path (no perf regression). + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + findMany: vi.fn(), + queryRaw: vi.fn(), + readTieredRollupSeries: vi.fn(), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { + measurementRollup: { findMany: mocks.findMany }, + $queryRaw: mocks.queryRaw, + }, +})); + +vi.mock("@/lib/rollups/measurement-read-wmy", () => ({ + readTieredRollupSeries: mocks.readTieredRollupSeries, +})); + +vi.mock("@/lib/rollups/measurement-rollups", () => ({ + recomputeUserRollups: vi.fn(), +})); + +import { readDailySeries } from "../daily-series-read"; + +const DAY_MS = 86_400_000; + +beforeEach(() => { + mocks.findMany.mockReset(); + mocks.queryRaw.mockReset(); + mocks.readTieredRollupSeries.mockReset(); + // Default DAY-rollup read returns enough rows (≥ SUSPICIOUS_ROW_FLOOR) + // so the short-window path resolves without the coverage probe / live + // query firing. + mocks.findMany.mockResolvedValue([ + { + type: "WEIGHT", + source: "MANUAL", + bucketStart: new Date("2026-06-01T00:00:00.000Z"), + mean: 81, + count: 1, + sumValue: null, + minValue: 81, + maxValue: 81, + }, + { + type: "WEIGHT", + source: "MANUAL", + bucketStart: new Date("2026-06-02T00:00:00.000Z"), + mean: 82, + count: 1, + sumValue: null, + minValue: 82, + maxValue: 82, + }, + { + type: "WEIGHT", + source: "MANUAL", + bucketStart: new Date("2026-06-03T00:00:00.000Z"), + mean: 80, + count: 1, + sumValue: null, + minValue: 80, + maxValue: 80, + }, + ]); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("readDailySeries — long-range tier step-up", () => { + it("routes a multi-year window through the tiered reader and returns its whole-history rows", async () => { + const to = new Date("2026-06-21T00:00:00.000Z"); + const from = new Date(to.getTime() - 3650 * DAY_MS); + const tieredRows = [ + { + type: "WEIGHT", + value: 80, + measuredAt: "2017-01-01T00:00:00.000Z", + count: 12, + }, + { + type: "WEIGHT", + value: 79, + measuredAt: "2026-01-01T00:00:00.000Z", + count: 12, + }, + ]; + mocks.readTieredRollupSeries.mockResolvedValueOnce({ + granularity: "MONTH", + rows: tieredRows, + }); + + const result = await readDailySeries({ + userId: "u", + type: "WEIGHT", + from, + to, + priorityJson: null, + }); + + expect(mocks.readTieredRollupSeries).toHaveBeenCalledTimes(1); + expect(mocks.readTieredRollupSeries.mock.calls[0][0].windowDays).toBe(3650); + // Whole-history coverage: the earliest 2017 bucket survives. + expect(result[0].measuredAt).toBe("2017-01-01T00:00:00.000Z"); + expect(result).toHaveLength(2); + // The DAY-rollup path was NOT consulted for the long window. + expect(mocks.findMany).not.toHaveBeenCalled(); + }); + + it("falls through to the daily path on a tiered coverage miss (no silent empty)", async () => { + const to = new Date("2026-06-21T00:00:00.000Z"); + const from = new Date(to.getTime() - 3650 * DAY_MS); + mocks.readTieredRollupSeries.mockResolvedValueOnce(null); + + const result = await readDailySeries({ + userId: "u", + type: "WEIGHT", + from, + to, + priorityJson: null, + }); + + expect(mocks.readTieredRollupSeries).toHaveBeenCalledTimes(1); + // Daily DAY-rollup read still ran as the fallback. + expect(mocks.findMany).toHaveBeenCalled(); + expect(result).toHaveLength(3); + }); + + it("does NOT touch the tiered reader for a normal 90-day window", async () => { + const to = new Date("2026-06-21T00:00:00.000Z"); + const from = new Date(to.getTime() - 90 * DAY_MS); + + await readDailySeries({ + userId: "u", + type: "WEIGHT", + from, + to, + priorityJson: null, + }); + + expect(mocks.readTieredRollupSeries).not.toHaveBeenCalled(); + expect(mocks.findMany).toHaveBeenCalled(); + }); +}); diff --git a/src/lib/measurements/daily-series-read.ts b/src/lib/measurements/daily-series-read.ts index 969c8f140..8759c4e72 100644 --- a/src/lib/measurements/daily-series-read.ts +++ b/src/lib/measurements/daily-series-read.ts @@ -20,6 +20,7 @@ import { prisma } from "@/lib/db"; import { BUCKET_CAP } from "@/lib/measurements/range-aggregation"; import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; import { collapseRollupRowsBySource } from "@/lib/rollups/measurement-read"; +import { readTieredRollupSeries } from "@/lib/rollups/measurement-read-wmy"; import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; import { annotate } from "@/lib/logging/context"; @@ -59,6 +60,45 @@ export async function readDailySeries(opts: { const { userId, type, from, to, priorityJson } = opts; const cap = Math.min(opts.limit ?? BUCKET_CAP.daily, BUCKET_CAP.daily); + // v1.19.2 — whole-history step-up for very long ranges. When the + // requested window holds more days than the DAY bucket cap can carry, + // the daily path below would `LIMIT`/`slice` to `cap` (365) buckets and + // silently drop the older history — a multi-year "Alle" range collapsed + // to roughly the most recent year. Step UP the bucket tier + // (DAY → WEEK → MONTH → YEAR) so the series spans the whole window + // inside a sane point budget instead. The chart's own `bucketTimeSeries` + // downsampler already renders week / month points for long ranges, so a + // coarser server tier is the resolution it would have collapsed to + // anyway — minus the truncation. Short / normal ranges (≤ cap days) + // never enter this branch and stay byte-identical with the prior daily + // path, so there is no perf regression for the common case. + const windowDaysFull = Math.ceil( + (to.getTime() - from.getTime()) / 86_400_000, + ); + if (windowDaysFull > cap) { + const tiered = await readTieredRollupSeries({ + userId, + type, + windowDays: windowDaysFull, + priorityJson, + }); + if (tiered && tiered.rows.length > 0) { + return tiered.rows; + } + // Coverage miss at every tier — fall through to the daily path, which + // probes the live table and folds an under-converged window. The + // daily fallback still caps at `cap`; the annotate below records that + // the long-range read could not step up so a truncated daily slice is + // an explicit, logged outcome rather than a silent one. + annotate({ + meta: { + tiered_series_coverage_miss: true, + window_days: windowDaysFull, + type, + }, + }); + } + const readRollup = async () => collapseRollupRowsBySource( await prisma.measurementRollup.findMany({ diff --git a/src/lib/rollups/__tests__/measurement-read-wmy.test.ts b/src/lib/rollups/__tests__/measurement-read-wmy.test.ts index f47c6dffa..733937bb6 100644 --- a/src/lib/rollups/__tests__/measurement-read-wmy.test.ts +++ b/src/lib/rollups/__tests__/measurement-read-wmy.test.ts @@ -44,6 +44,7 @@ vi.mock("@/lib/db", () => ({ import { aggregateWmyBuckets, readBestGranularityRollups, + readTieredRollupSeries, type RollupBucketRow, } from "../measurement-read-wmy"; @@ -286,3 +287,126 @@ describe("readBestGranularityRollups — cross-consumer routing parity", () => { expect(monthAgg.mean).toBeCloseTo(dayAgg.mean ?? Number.NaN, 5); }); }); + +/** + * v1.19.2 W-CHARTS — whole-history series reader. Pins that a very long + * "Alle" window reads the display-matching coarse tier (WEEK for 1–2 y, + * MONTH beyond) and returns coverage spanning the FULL span rather than a + * truncated recent slice; that the finer fallback rescues a coverage + * miss; and that the wire shape mirrors the daily reader. + */ +describe("readTieredRollupSeries", () => { + it("returns null on a non-positive window without touching the db", async () => { + expect( + await readTieredRollupSeries({ + userId: "u", + type: "WEIGHT", + windowDays: 0, + }), + ).toBeNull(); + expect(findMany).not.toHaveBeenCalled(); + }); + + it("reads the MONTH tier for a multi-year window and spans the whole history", async () => { + // 10-year window → MONTH tier. The fixture carries buckets from the + // OLDEST month through the most recent — the result must keep the + // oldest bucket, proving no recent-slice truncation. + const months: RollupBucketRow[] = []; + for (let y = 2017; y <= 2026; y++) { + months.push(bucket(`${y}-01-01T00:00:00.000Z`, { count: 12, mean: 80 })); + } + findMany.mockResolvedValueOnce(months); + + const result = await readTieredRollupSeries({ + userId: "u", + type: "WEIGHT", + windowDays: 3650, + }); + + expect(result).not.toBeNull(); + expect(result?.granularity).toBe("MONTH"); + expect(findMany.mock.calls[0][0].where.granularity).toBe("MONTH"); + // Whole-history coverage: the earliest 2017 bucket survives. + expect(result?.rows[0].measuredAt).toBe("2017-01-01T00:00:00.000Z"); + expect(result?.rows.at(-1)?.measuredAt).toBe("2026-01-01T00:00:00.000Z"); + expect(result?.rows.length).toBe(10); + }); + + it("reads the WEEK tier for a 1–2 year window", async () => { + findMany.mockResolvedValueOnce([bucket("2025-08-04T00:00:00.000Z")]); + const result = await readTieredRollupSeries({ + userId: "u", + type: "WEIGHT", + windowDays: 540, + }); + expect(result?.granularity).toBe("WEEK"); + expect(findMany.mock.calls[0][0].where.granularity).toBe("WEEK"); + }); + + it("falls FINER (MONTH → WEEK → DAY) when the target tier has no coverage", async () => { + findMany + .mockResolvedValueOnce([]) // MONTH miss + .mockResolvedValueOnce([]) // WEEK miss + .mockResolvedValueOnce([bucket("2024-03-01T00:00:00.000Z")]); // DAY hit + const result = await readTieredRollupSeries({ + userId: "u", + type: "WEIGHT", + windowDays: 3650, + }); + expect(result?.granularity).toBe("DAY"); + expect(findMany.mock.calls[0][0].where.granularity).toBe("MONTH"); + expect(findMany.mock.calls[1][0].where.granularity).toBe("WEEK"); + expect(findMany.mock.calls[2][0].where.granularity).toBe("DAY"); + }); + + it("returns null when no tier at or below the target carries coverage", async () => { + findMany.mockResolvedValue([]); + const result = await readTieredRollupSeries({ + userId: "u", + type: "WEIGHT", + windowDays: 3650, + }); + expect(result).toBeNull(); + // MONTH, WEEK, DAY all probed (never coarser than the target). + expect(findMany).toHaveBeenCalledTimes(3); + }); + + it("surfaces the cumulative summed total for SUM metrics and drops the spread", async () => { + findMany.mockResolvedValueOnce([ + bucket("2025-01-01T00:00:00.000Z", { + count: 30, + mean: 8000, + sumValue: 240_000, + minValue: 0, + maxValue: 20_000, + }), + ]); + const result = await readTieredRollupSeries({ + userId: "u", + type: "ACTIVITY_STEPS", + windowDays: 3650, + }); + expect(result?.rows[0].value).toBe(240_000); + expect(result?.rows[0].minValue).toBeUndefined(); + expect(result?.rows[0].maxValue).toBeUndefined(); + }); + + it("carries the count-weighted mean + spread for spot metrics", async () => { + findMany.mockResolvedValueOnce([ + bucket("2025-01-01T00:00:00.000Z", { + count: 12, + mean: 81.5, + minValue: 78, + maxValue: 85, + }), + ]); + const result = await readTieredRollupSeries({ + userId: "u", + type: "WEIGHT", + windowDays: 3650, + }); + expect(result?.rows[0].value).toBe(81.5); + expect(result?.rows[0].minValue).toBe(78); + expect(result?.rows[0].maxValue).toBe(85); + }); +}); diff --git a/src/lib/rollups/measurement-read-wmy.ts b/src/lib/rollups/measurement-read-wmy.ts index 7d9222ec7..46eb1235d 100644 --- a/src/lib/rollups/measurement-read-wmy.ts +++ b/src/lib/rollups/measurement-read-wmy.ts @@ -61,6 +61,8 @@ import type { } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; +import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping"; +import { annotate } from "@/lib/logging/context"; import { collapseRollupRowsBySource, loadUserSourcePriority, @@ -272,3 +274,135 @@ async function readGranularity( maxValue: r.maxValue, })); } + +/** + * Wire-row shape the chart-data client consumes — mirrors + * `DailySeriesRow` in `daily-series-read.ts` so a tier-stepped series + * interleaves with the daily path without the caller branching on the + * source granularity. `measuredAt` is the bucket-start ISO string at the + * resolved tier (one row per DAY / WEEK / MONTH / YEAR bucket). + */ +export interface TieredSeriesRow { + type: string; + value: number; + measuredAt: string; + count: number; + minValue?: number | null; + maxValue?: number | null; +} + +/** + * Pick the rollup granularity that matches the chart's own client-side + * `bucketTimeSeries` downsampler (`src/lib/charts/bucket-time-series.ts`): + * + * - > 730 days → MONTH (the chart renders month points) + * - 366–730 → WEEK (the chart renders week points) + * - ≤ 365 → DAY (daily resolution; the DAY cap covers it) + * + * Mirroring the display tier means the server returns exactly the + * resolution the chart will paint — no finer (which would truncate at the + * DAY cap) and no coarser (which would drop visible detail). The finer + * fallback in `readTieredRollupSeries` still rescues a tenant whose + * coarse buckets the worker has not minted yet. + */ +function pickRollupGranularityForWindow(windowDays: number): RollupGranularity { + if (windowDays > 730) return "MONTH"; + if (windowDays > 365) return "WEEK"; + return "DAY"; +} + +/** DAY → WEEK → MONTH → YEAR ordering for the finer-fallback walk. */ +const TIER_ORDER: RollupGranularity[] = ["DAY", "WEEK", "MONTH", "YEAR"]; + +/** + * v1.19.2 — whole-history series reader for very long chart ranges. + * + * The DAY-only `readDailySeries` reader caps its result at + * `BUCKET_CAP.daily` (365) buckets. A multi-year "Alle" range therefore + * silently truncated to roughly the most recent year — the older history + * never reached the client even though the chart's own + * `bucketTimeSeries` downsampler would have rendered it as week / month + * points. This reader closes that gap by reading the bucket tier that + * matches the chart's display granularity (WEEK for 1–2 years, MONTH + * beyond), so the returned series spans the WHOLE requested window inside + * a sane point budget instead of being chopped to a recent slice. + * + * The tier is purely a downsampling choice: `count` and the + * count-weighted `mean` compose identically across any granularity (see + * the compositional contract above), so a 5-year window rendered as ~60 + * MONTH points carries the same trend as the ~1 800 DAY points would have + * — minus the truncation. `minValue` / `maxValue` ride through for the + * range band on spot metrics; cumulative metrics (steps, energy, + * distance) surface the bucket's summed total and drop the spread. + * + * Coverage handling: the target tier is read first; on a coverage miss + * (the boot backfill / worker has not minted the coarse buckets yet) the + * reader walks FINER (MONTH → WEEK → DAY) so a tenant still gets a usable + * series rather than an empty chart. Returns `null` only when no tier at + * or below the target carries any buckets for the window — the caller + * then falls through to its live-SQL path. + * + * The result is NOT capped — the tier selection bounds the row count + * (≤ ~104 weeks for the WEEK tier, ≤ ~12 months/year for MONTH). If a + * future tier ever risks an unbounded count it must step coarser rather + * than truncate; no silent cap lives on this path. + */ +export async function readTieredRollupSeries(opts: { + userId: string; + type: MeasurementType; + windowDays: number; + priorityJson?: unknown; +}): Promise<{ + granularity: RollupGranularity; + rows: TieredSeriesRow[]; +} | null> { + const { userId, type, windowDays, priorityJson } = opts; + if (!Number.isFinite(windowDays) || windowDays <= 0) return null; + + const since = new Date(Date.now() - windowDays * 24 * 60 * 60 * 1000); + const priority = + priorityJson !== undefined + ? priorityJson + : await loadUserSourcePriority(userId); + + const target = pickRollupGranularityForWindow(windowDays); + // Walk the target tier first, then finer (never coarser — coarser would + // drop detail the chart can render). Stop at the first tier with coverage. + const targetIdx = TIER_ORDER.indexOf(target); + for (let i = targetIdx; i >= 0; i--) { + const granularity = TIER_ORDER[i]; + const rows = await readGranularity( + userId, + type, + granularity, + since, + priority, + ); + if (rows && rows.length > 0) { + const useSum = CUMULATIVE_HK_TYPES.has(type); + annotate({ + action: { name: "measurement.list" }, + meta: { + total: rows.length, + type, + aggregate: "tiered", + granularity, + target_granularity: target, + source: "rollup", + }, + }); + return { + granularity, + rows: rows.map((r) => ({ + type, + value: useSum ? (r.sumValue ?? r.mean * r.count) : r.mean, + measuredAt: r.bucketStart.toISOString(), + count: r.count, + minValue: useSum ? undefined : r.minValue, + maxValue: useSum ? undefined : r.maxValue, + })), + }; + } + } + return null; +}