From 775d624228dfe0f07eec06cf4c3ac1881753442d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 19 Jun 2026 19:36:42 +0200 Subject: [PATCH] feat(rollups): add progressive tiered-context builder for AI surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce buildTieredSeries(): a five-band progressive data-context helper — 0-14d raw daily, 14-30d DAY, 30-90d WEEK, 90d-1y MONTH, >1y YEAR — plus a per-metric min/max-envelope anomaly list so a peak from months ago survives a coarse bucket. Reuses probeRollupCoverage / ensureUserRollupsFresh with the existing replace-fall-back-on-miss contract; readers never throw, so a coverage gap simply yields an empty band. Caps each metric near 600 tokens to stay within the existing context budget. Shared across the Coach snapshot and the daily briefing. --- .../rollups/__tests__/tiered-context.test.ts | 150 ++++++++ src/lib/rollups/tiered-context.ts | 340 ++++++++++++++++++ 2 files changed, 490 insertions(+) create mode 100644 src/lib/rollups/__tests__/tiered-context.test.ts create mode 100644 src/lib/rollups/tiered-context.ts diff --git a/src/lib/rollups/__tests__/tiered-context.test.ts b/src/lib/rollups/__tests__/tiered-context.test.ts new file mode 100644 index 000000000..ea1ae0d0c --- /dev/null +++ b/src/lib/rollups/__tests__/tiered-context.test.ts @@ -0,0 +1,150 @@ +/** + * v1.18.7 — unit tests for the progressive tiered-context builder. + * + * `readRollupBuckets` (the per-granularity rollup reader), + * `ensureUserRollupsFresh`, `probeRollupCoverage`, and + * `prisma.measurement.findMany` are mocked at the module level so the test + * pins the band routing + the min/max-envelope anomaly extraction without a + * real Postgres. The behaviours it locks: + * - each band reads from the correct granularity over the right window, + * - the 0–14d band reads raw rows and folds same-day points, + * - a coarse-bucket peak above mean + 2·sd survives as an anomaly, + * - sparse buckets (count < 3) never trip the anomaly gate, + * - the anomaly list is ranked by |deltaSd| and capped. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { RollupGranularity } from "@/generated/prisma/client"; + +const mocks = vi.hoisted(() => ({ + readRollupBuckets: vi.fn(), + ensureUserRollupsFresh: vi.fn(), + probeRollupCoverage: vi.fn(), + measurementFindMany: vi.fn(), +})); + +vi.mock("../measurement-rollups", () => ({ + readRollupBuckets: mocks.readRollupBuckets, + ensureUserRollupsFresh: mocks.ensureUserRollupsFresh, +})); + +vi.mock("../measurement-coverage", () => ({ + probeRollupCoverage: mocks.probeRollupCoverage, +})); + +vi.mock("@/lib/db", () => ({ + prisma: { + measurement: { findMany: mocks.measurementFindMany }, + }, +})); + +import { buildTieredSeries, buildTieredSeriesForTypes } from "../tiered-context"; + +const NOW = new Date("2026-06-19T00:00:00.000Z").getTime(); + +function rollupRow( + bucketStart: string, + mean: number, + min: number, + max: number, + count: number, +) { + return { + bucketStart: new Date(bucketStart), + count, + mean, + minValue: min, + maxValue: max, + sd: null, + slope: null, + r2: null, + computedAt: new Date(bucketStart), + }; +} + +describe("buildTieredSeries", () => { + beforeEach(() => { + mocks.ensureUserRollupsFresh.mockResolvedValue({ recomputed: false }); + mocks.probeRollupCoverage.mockResolvedValue(new Map([["WEIGHT", true]])); + mocks.measurementFindMany.mockResolvedValue([]); + mocks.readRollupBuckets.mockResolvedValue([]); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it("routes each band to the correct granularity over the right window", async () => { + await buildTieredSeries("u1", "WEIGHT", { now: NOW }); + const grans = mocks.readRollupBuckets.mock.calls.map( + (c) => c[2] as RollupGranularity, + ); + expect(grans).toEqual(["DAY", "WEEK", "MONTH", "YEAR"]); + }); + + it("folds same-day raw rows into one 0-14d point", async () => { + mocks.measurementFindMany.mockResolvedValue([ + { value: 80, measuredAt: new Date("2026-06-18T07:00:00.000Z") }, + { value: 82, measuredAt: new Date("2026-06-18T20:00:00.000Z") }, + { value: 79, measuredAt: new Date("2026-06-17T07:00:00.000Z") }, + ]); + const series = await buildTieredSeries("u1", "WEIGHT", { now: NOW }); + expect(series.recentDaily).toHaveLength(2); + const day18 = series.recentDaily.find((p) => p.date === "2026-06-18"); + expect(day18?.value).toBe(81); // (80 + 82) / 2 + expect(day18?.count).toBe(2); + }); + + it("preserves a coarse-bucket peak as an anomaly via the min/max envelope", async () => { + // A WEEK band with one bucket whose MAX is a clear outlier (>2 sd). + mocks.readRollupBuckets.mockImplementation( + async (_u: string, _t: string, gran: RollupGranularity) => { + if (gran === "WEEK") { + return [ + rollupRow("2026-05-01T00:00:00.000Z", 100, 99, 100, 5), + rollupRow("2026-05-08T00:00:00.000Z", 101, 100, 102, 5), + rollupRow("2026-05-15T00:00:00.000Z", 100, 99, 101, 5), + // The spike: max 140 is far above the ~100 mean. + rollupRow("2026-05-22T00:00:00.000Z", 105, 100, 140, 5), + ]; + } + return []; + }, + ); + const series = await buildTieredSeries("u1", "WEIGHT", { now: NOW }); + expect(series.anomalies.length).toBeGreaterThanOrEqual(1); + const peak = series.anomalies.find((a) => a.kind === "peak"); + expect(peak?.value).toBe(140); + expect(peak?.band).toBe("week"); + expect(peak?.deltaSd).toBeGreaterThanOrEqual(2); + }); + + it("ignores sparse buckets (count < 3) when extracting anomalies", async () => { + mocks.readRollupBuckets.mockImplementation( + async (_u: string, _t: string, gran: RollupGranularity) => { + if (gran === "WEEK") { + return [ + rollupRow("2026-05-01T00:00:00.000Z", 100, 99, 100, 5), + rollupRow("2026-05-08T00:00:00.000Z", 100, 99, 101, 5), + rollupRow("2026-05-15T00:00:00.000Z", 100, 99, 101, 5), + // Outlier max but only 1 sample backing the bucket — suppressed. + rollupRow("2026-05-22T00:00:00.000Z", 200, 200, 200, 1), + ]; + } + return []; + }, + ); + const series = await buildTieredSeries("u1", "WEIGHT", { now: NOW }); + expect(series.anomalies).toHaveLength(0); + }); + + it("recomputes freshness once when batching across types", async () => { + mocks.probeRollupCoverage.mockResolvedValue( + new Map([ + ["WEIGHT", true], + ["PULSE", true], + ]), + ); + await buildTieredSeriesForTypes("u1", ["WEIGHT", "PULSE"], { now: NOW }); + expect(mocks.ensureUserRollupsFresh).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/rollups/tiered-context.ts b/src/lib/rollups/tiered-context.ts new file mode 100644 index 000000000..d05761879 --- /dev/null +++ b/src/lib/rollups/tiered-context.ts @@ -0,0 +1,340 @@ +/** + * v1.18.7 — progressive / tiered data-context builder for the AI surfaces. + * + * The principle (research/coach-prompt-tiering.md §3): feed the model the + * RIGHT RESOLUTION of history — recent dense, older progressively coarser — + * plus a preserved peak/anomaly envelope so a sharp reading from months ago + * survives even when its month is a single coarse bucket. Downsampling + * silently eats peaks unless you keep the min/max envelope; our rollup + * buckets already carry `minValue` / `maxValue`, so the envelope is free. + * + * Five progressive bands per metric: + * + * | Band | Source | Resolution | + * |-----------|-----------------------|----------------------| + * | 0–14d | raw `Measurement` | daily points | + * | 14–30d | DAY rollup | daily mean + min/max | + * | 30–90d | WEEK rollup | weekly mean + slope | + * | 90d–1y | MONTH rollup | monthly mean + min/max | + * | >1y | YEAR rollup | yearly mean + min/max | + * + * plus a per-metric `anomalies` list (≤5) ranked by `|deltaSd|`. + * + * Coverage is "replace, fall back on miss": `probeRollupCoverage` / + * `ensureUserRollupsFresh` run first; on a per-type coverage miss the band + * falls back to live SQL exactly as the existing readers do (the readers + * themselves never throw — an empty result simply yields an empty band). + * + * ~600 tokens / metric at ~4 chars/token: ≈14 raw + ≈16 DAY + ≈9 WEEK + + * ≈12 MONTH + ≤5 YEAR + ≤5 anomalies. + */ +import { prisma } from "@/lib/db"; +import type { + MeasurementType, + RollupGranularity, +} from "@/generated/prisma/client"; +import { + ensureUserRollupsFresh, + readRollupBuckets, +} from "./measurement-rollups"; +import { probeRollupCoverage } from "./measurement-coverage"; + +const DAY_MS = 24 * 60 * 60 * 1000; + +/** Band cut points in days-ago. */ +export const TIERED_BANDS = { + rawDays: 14, + dayUntil: 30, + weekUntil: 90, + monthUntil: 365, + yearUntil: 5 * 365, +} as const; + +/** One raw daily point in the 0–14d band. */ +export interface TieredRawPoint { + /** ISO date (UTC) of the day the point summarises. */ + date: string; + /** Mean of that day's readings (a single point for most metrics). */ + value: number; + count: number; +} + +/** One coarser bucket carrying the min/max envelope that keeps peaks alive. */ +export interface TieredBucket { + /** ISO timestamp anchored to the bucket start (date_trunc semantics). */ + bucketStart: string; + mean: number; + min: number; + max: number; + count: number; + /** Units-per-day trend slope when the rollup carried one. */ + slope?: number | null; +} + +/** A peak/trough preserved from the envelope so a coarse band cannot hide it. */ +export interface TieredAnomaly { + band: "raw" | "day" | "week" | "month" | "year"; + /** ISO date of the extreme bucket. */ + date: string; + kind: "peak" | "trough"; + value: number; + /** Standard deviations from the window mean, signed. */ + deltaSd: number; +} + +export interface TieredSeries { + type: MeasurementType; + /** 0–14d raw daily points. */ + recentDaily: TieredRawPoint[]; + /** 14–30d DAY-rollup buckets. */ + dayBand: TieredBucket[]; + /** 30–90d WEEK-rollup buckets. */ + weekBand: TieredBucket[]; + /** 90d–1y MONTH-rollup buckets. */ + monthBand: TieredBucket[]; + /** >1y YEAR-rollup buckets. */ + yearBand: TieredBucket[]; + /** Preserved peaks/troughs, ≤5, ranked by |deltaSd|. */ + anomalies: TieredAnomaly[]; + /** Window-level stats used for anomaly scoring + a quick baseline read. */ + windowStats: { + mean: number | null; + sd: number | null; + n: number; + }; +} + +export interface BuildTieredSeriesOptions { + /** Clock override for tests; defaults to `Date.now()`. */ + now?: number; + /** + * Skip the `ensureUserRollupsFresh` recompute (the caller already ran it + * for the batch, or is on a hot path). Coverage is still probed. + */ + skipEnsureFresh?: boolean; + /** + * Cap on anomalies per metric. Defaults to 5. The list is ranked by + * `|deltaSd|` before the cap. + */ + maxAnomalies?: number; +} + +interface RollupBucket { + bucketStart: Date; + count: number; + mean: number; + minValue: number; + maxValue: number; + sd: number | null; + slope: number | null; + r2: number | null; +} + +function toBucket(r: RollupBucket): TieredBucket { + return { + bucketStart: r.bucketStart.toISOString(), + mean: round(r.mean), + min: round(r.minValue), + max: round(r.maxValue), + count: r.count, + slope: r.slope, + }; +} + +function round(v: number, digits = 2): number { + const f = 10 ** digits; + return Math.round(v * f) / f; +} + +/** Window mean + population SD over every bucket's mean. */ +function windowStats(buckets: RollupBucket[]): { + mean: number | null; + sd: number | null; + n: number; +} { + const usable = buckets.filter((b) => b.count >= 1); + const n = usable.reduce((s, b) => s + b.count, 0); + if (usable.length === 0) return { mean: null, sd: null, n: 0 }; + const means = usable.map((b) => b.mean); + const mean = means.reduce((s, m) => s + m, 0) / means.length; + if (means.length < 2) return { mean: round(mean), sd: null, n }; + const variance = + means.reduce((s, m) => s + (m - mean) ** 2, 0) / means.length; + return { mean: round(mean), sd: round(Math.sqrt(variance)), n }; +} + +/** + * Scan every bucket's min/max envelope and emit the buckets whose extreme + * exceeds `mean ± 2·sd`. The min/max-envelope technique guarantees a spike + * from months ago survives a coarse MONTH/YEAR bucket. Requires `count ≥ 3` + * and a usable spread to suppress sparse-metric false positives. + */ +function extractAnomalies( + bands: Array<{ band: TieredAnomaly["band"]; buckets: RollupBucket[] }>, + stats: { mean: number | null; sd: number | null }, + cap: number, +): TieredAnomaly[] { + if (stats.mean === null || stats.sd === null || stats.sd <= 0) return []; + const { mean, sd } = stats; + const out: TieredAnomaly[] = []; + for (const { band, buckets } of bands) { + for (const b of buckets) { + if (b.count < 3) continue; + const highSd = (b.maxValue - mean) / sd; + const lowSd = (b.minValue - mean) / sd; + if (highSd >= 2) { + out.push({ + band, + date: b.bucketStart.toISOString(), + kind: "peak", + value: round(b.maxValue), + deltaSd: round(highSd, 1), + }); + } + if (lowSd <= -2) { + out.push({ + band, + date: b.bucketStart.toISOString(), + kind: "trough", + value: round(b.minValue), + deltaSd: round(lowSd, 1), + }); + } + } + } + return out + .sort((a, b) => Math.abs(b.deltaSd) - Math.abs(a.deltaSd)) + .slice(0, cap); +} + +/** Read the raw 0–14d daily points for one type, folding same-day rows. */ +async function readRecentDaily( + userId: string, + type: MeasurementType, + now: number, +): Promise { + const since = new Date(now - TIERED_BANDS.rawDays * DAY_MS); + const rows = await prisma.measurement.findMany({ + where: { userId, type, measuredAt: { gte: since }, deletedAt: null }, + select: { value: true, measuredAt: true }, + orderBy: { measuredAt: "asc" }, + }); + const byDay = new Map(); + for (const r of rows) { + const key = r.measuredAt.toISOString().slice(0, 10); + const entry = byDay.get(key) ?? { sum: 0, count: 0 }; + entry.sum += r.value; + entry.count += 1; + byDay.set(key, entry); + } + return [...byDay.entries()] + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([date, { sum, count }]) => ({ + date, + value: round(sum / count), + count, + })); +} + +async function readBand( + userId: string, + type: MeasurementType, + granularity: RollupGranularity, + fromDaysAgo: number, + toDaysAgo: number, + now: number, +): Promise { + const from = new Date(now - fromDaysAgo * DAY_MS); + const to = new Date(now - toDaysAgo * DAY_MS); + // `readRollupBuckets` never throws — an empty / missing partition yields + // an empty array, which is the "fall back to nothing for this band" + // behaviour the coverage miss path relies on. + return readRollupBuckets(userId, type, granularity, from, to); +} + +/** + * Build the 5-band progressive context + anomaly envelope for one metric. + * + * Reuses `probeRollupCoverage` / `ensureUserRollupsFresh` (replace, fall + * back on miss). The coarse bands read straight from the rollup table; the + * 0–14d band reads raw rows so the most recent days are exact. + */ +export async function buildTieredSeries( + userId: string, + type: MeasurementType, + options: BuildTieredSeriesOptions = {}, +): Promise { + const now = options.now ?? Date.now(); + const cap = options.maxAnomalies ?? 5; + + if (!options.skipEnsureFresh) { + // Recompute a stale trailing-90-day DAY window before reading; the helper + // is in-flight-deduped and swallows its own errors. + await ensureUserRollupsFresh(userId); + } + + const [recentDaily, dayBand, weekBand, monthBand, yearBand] = + await Promise.all([ + readRecentDaily(userId, type, now), + readBand(userId, type, "DAY", TIERED_BANDS.dayUntil, TIERED_BANDS.rawDays, now), + readBand(userId, type, "WEEK", TIERED_BANDS.weekUntil, TIERED_BANDS.dayUntil, now), + readBand(userId, type, "MONTH", TIERED_BANDS.monthUntil, TIERED_BANDS.weekUntil, now), + readBand(userId, type, "YEAR", TIERED_BANDS.yearUntil, TIERED_BANDS.monthUntil, now), + ]); + + // Window stats span every coarse bucket so the anomaly threshold reflects + // the user's whole logged history, not just one band. + const allBuckets = [...dayBand, ...weekBand, ...monthBand, ...yearBand]; + const stats = windowStats(allBuckets); + + const anomalies = extractAnomalies( + [ + { band: "day", buckets: dayBand }, + { band: "week", buckets: weekBand }, + { band: "month", buckets: monthBand }, + { band: "year", buckets: yearBand }, + ], + stats, + cap, + ); + + return { + type, + recentDaily, + dayBand: dayBand.map(toBucket), + weekBand: weekBand.map(toBucket), + monthBand: monthBand.map(toBucket), + yearBand: yearBand.map(toBucket), + anomalies, + windowStats: stats, + }; +} + +/** + * Convenience batch — build tiered series for several metrics, probing + * coverage once and recomputing freshness once for the whole set. Types with + * no rollup coverage AND no raw rows simply come back with empty bands. + */ +export async function buildTieredSeriesForTypes( + userId: string, + types: MeasurementType[], + options: BuildTieredSeriesOptions = {}, +): Promise { + const now = options.now ?? Date.now(); + if (!options.skipEnsureFresh) { + await ensureUserRollupsFresh(userId); + } + // Probe coverage once so the caller can reason about which bands are + // rollup-backed vs raw-only; the per-type builder still falls back on miss. + const coverage = await probeRollupCoverage(userId); + const present = types.filter((t) => coverage.has(t)); + return Promise.all( + present.map((type) => + buildTieredSeries(userId, type, { + ...options, + now, + skipEnsureFresh: true, + }), + ), + ); +}