From 9c51dd0d304ad17bfbf08244fb82d4018cbc392b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 03:35:03 +0200 Subject: [PATCH 01/11] perf(analytics): read the latest value per type with one probe per type The latest canonical reading per type ran as `DISTINCT ON (type)` over every live row of the account in five places (summaries slice warm and cold path, comprehensive aggregator warm and cold path, targets). With no skip scan in Postgres 16 that sorted the whole account to keep one row per type. `readLatestCanonical` drives the read from the type list instead: one index probe for each type's latest instant, then a closed one-day window ranked by the source ladder. The closed window also keeps the plan under `jit_above_cost`. The targets read reuses `readLatestEver` with a floor. The summaries slice was built inside three cache cells (the tile-strip route, the thick analytics body, the dashboard snapshot). It now has one cell, read through `readSummariesSlice`; composing callers read it fresh-only so a refresh after a write cannot cache the pre-write slice again, and the snapshot folds the water tile into a copy instead of the cached object. --- src/__tests__/api/analytics/since-cap.test.ts | 10 +- .../latest-per-type-read-guard.test.ts | 82 +++++++++++++ src/app/api/analytics/route.ts | 15 +-- .../insights/targets/__tests__/route.test.ts | 49 ++++---- .../__tests__/summaries-slice.test.ts | 37 +++++- ...onfig-change-is-not-a-health-event.test.ts | 6 +- src/lib/analytics/summaries-slice.ts | 82 ++++++------- src/lib/dashboard/__tests__/snapshot.test.ts | 38 ++++-- src/lib/dashboard/snapshot.ts | 41 ++++--- src/lib/dashboard/summary-reads.ts | 10 +- src/lib/insights/comprehensive-aggregator.ts | 44 ++----- src/lib/measurements/latest-canonical.ts | 66 ++++++++++ src/lib/targets/build-response.ts | 27 +---- .../integration/latest-canonical-read.test.ts | 114 ++++++++++++++++++ 14 files changed, 436 insertions(+), 185 deletions(-) create mode 100644 src/__tests__/latest-per-type-read-guard.test.ts create mode 100644 src/lib/measurements/latest-canonical.ts create mode 100644 tests/integration/latest-canonical-read.test.ts diff --git a/src/__tests__/api/analytics/since-cap.test.ts b/src/__tests__/api/analytics/since-cap.test.ts index 221f1342e..1b3cfbf16 100644 --- a/src/__tests__/api/analytics/since-cap.test.ts +++ b/src/__tests__/api/analytics/since-cap.test.ts @@ -118,7 +118,7 @@ vi.mock("@/lib/analytics/correlations-fast-path", () => ({ })); vi.mock("@/lib/analytics/summaries-slice", () => ({ - computeSummariesSlice: vi.fn().mockResolvedValue({ + readSummariesSlice: vi.fn().mockResolvedValue({ summaries: {}, bmi: null, bpInTargetPct: null, @@ -176,7 +176,7 @@ beforeEach(async () => { await import("@/lib/analytics/score/reader"); const { computeCorrelationHypothesesFastPath } = await import("@/lib/analytics/correlations-fast-path"); - const { computeSummariesSlice } = + const { readSummariesSlice } = await import("@/lib/analytics/summaries-slice"); (ensureUserRollupsFresh as ReturnType).mockResolvedValue( @@ -196,7 +196,7 @@ beforeEach(async () => { ( computeCorrelationHypothesesFastPath as ReturnType ).mockResolvedValue({ hypotheses: [] }); - (computeSummariesSlice as ReturnType).mockResolvedValue({ + (readSummariesSlice as ReturnType).mockResolvedValue({ summaries: {}, bmi: null, lastSeenByType: {}, @@ -206,7 +206,7 @@ beforeEach(async () => { describe("GET /api/analytics — per-type live walk retired (v1.4.49.1)", () => { // v1.4.49.1 — the W-SINCE 425-day `since` cap was a defense-in-depth // bound on the now-deleted 15-way per-type live walk. The default - // slice routes through `computeSummariesSlice` exclusively, which + // slice routes through `readSummariesSlice` exclusively, which // reads `measurement_rollups` DAY buckets + a 90-day narrow // `$queryRaw` directly — no chunked findMany against `measurements` // at all on the default critical path. The two assertions that pinned @@ -222,7 +222,7 @@ describe("GET /api/analytics — per-type live walk retired (v1.4.49.1)", () => ); expect(response.status).toBe(200); - // The slim slice resolves through `computeSummariesSlice` only; + // The slim slice resolves through `readSummariesSlice` only; // the per-type findMany loop is skipped entirely so the rollup- // path semantics are unaffected by the `since` cap. const findMany = prisma.measurement.findMany as ReturnType; diff --git a/src/__tests__/latest-per-type-read-guard.test.ts b/src/__tests__/latest-per-type-read-guard.test.ts new file mode 100644 index 000000000..a0e7eb796 --- /dev/null +++ b/src/__tests__/latest-per-type-read-guard.test.ts @@ -0,0 +1,82 @@ +/** + * Structural guard: the latest reading per measurement type has two readers, + * and the summaries slice one cache cell. + * + * Postgres 16 has no skip scan, so `DISTINCT ON (type)` over `measurements` + * sorts every row it can see to keep one per type. Five copies of that shape + * sat in the dashboard, analytics, insights and targets reads; each one now + * goes through `readLatestCanonical` (the ladder-canonical reading of the + * latest day) or `readLatestEver` (the latest raw reading), both driven from + * the type list with one index probe per type. + * + * The summaries slice was computed inside three cells (the tile-strip route, + * the thick analytics body, the dashboard snapshot). `readSummariesSlice` is + * the one cell; `computeSummariesSlice` is its builder and is called nowhere + * else in production code. + * + * Comments are stripped before matching, and the planted cases prove each + * matcher finds what it claims to. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +function stripComments(text: string): string { + return text + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/^\s*\/\/.*$/gm, "") + .replace(/^\s*--.*$/gm, ""); +} + +/** `DISTINCT ON (type)` or `DISTINCT ON (m."type")`, the type alone. */ +const LATEST_PER_TYPE = /DISTINCT\s+ON\s*\(\s*(?:\w+\s*\.\s*)?"?type"?\s*\)/i; + +const SLICE_BUILDER_CALL = /\bcomputeSummariesSlice\s*\(/; + +function productionFiles(): string[] { + return walkSourceFiles(SRC, { floor: 3000 }).filter( + (file) => + !file.startsWith("generated/") && + !file.includes("__tests__/") && + !/\.test\.tsx?$/.test(file), + ); +} + +describe("latest-per-type reads", () => { + it("the matchers find their shapes", () => { + expect( + LATEST_PER_TYPE.test(`SELECT DISTINCT ON (m."type") m."value"`), + ).toBe(true); + expect(LATEST_PER_TYPE.test(`SELECT DISTINCT ON ("type") value`)).toBe( + true, + ); + expect( + LATEST_PER_TYPE.test(`SELECT DISTINCT ON ("type", "bucket_start")`), + ).toBe(false); + expect( + SLICE_BUILDER_CALL.test(`await computeSummariesSlice(user.id)`), + ).toBe(true); + }); + + it("no production read keeps one row per type with DISTINCT ON", () => { + const offenders = productionFiles().filter((file) => + LATEST_PER_TYPE.test( + stripComments(readFileSync(join(SRC, file), "utf8")), + ), + ); + expect(offenders).toEqual([]); + }); + + it("the summaries slice is built only behind its cache cell", () => { + const callers = productionFiles().filter((file) => + SLICE_BUILDER_CALL.test( + stripComments(readFileSync(join(SRC, file), "utf8")), + ), + ); + expect(callers).toEqual(["lib/analytics/summaries-slice.ts"]); + }); +}); diff --git a/src/app/api/analytics/route.ts b/src/app/api/analytics/route.ts index de8e909bd..6c5f611de 100644 --- a/src/app/api/analytics/route.ts +++ b/src/app/api/analytics/route.ts @@ -6,7 +6,7 @@ import { analyticsQuerySchema } from "@/lib/validations/analytics"; import { NO_STORE_BUT_BFCACHE } from "@/lib/http/cache-headers"; import { cachedSwr, caches, type ServerCache } from "@/lib/cache/server-cache"; import { summarize, type DataPoint } from "@/lib/analytics/trends"; -import { computeSummariesSlice } from "@/lib/analytics/summaries-slice"; +import { readSummariesSlice } from "@/lib/analytics/summaries-slice"; import { resolveEffectiveBpTargets } from "@/lib/analytics/effective-range"; import { bpTargetsEqual, getBpTargets } from "@/lib/analytics/bp-targets"; import { DEFAULT_TIMEZONE } from "@/lib/tz/resolver"; @@ -103,14 +103,7 @@ export const GET = apiHandler(async (request?: Request) => { // tile strip's next mount should repaint from the prior summaries // instantly while one background recompute refreshes them, instead // of paying the 2-SQL-pass rebuild inline. - const slim = await cachedSwr( - caches.analytics as ServerCache< - Awaited> - >, - `${user.id}|summaries`, - () => computeSummariesSlice(user.id), - annotate, - ); + const slim = await readSummariesSlice(user.id, { serveStale: true }); // Drop the types whose module the user turned off, from the SAME map // the snapshot builder gates on (`SUMMARY_TYPE_MODULE`, via // `gateSummariesByModules`). With `NEXT_PUBLIC_DASHBOARD_SNAPSHOT=false` @@ -282,7 +275,7 @@ async function buildAnalyticsResponse(user: AuthedUser, locale: Locale) { // release; previously only the live walk produced it) // - `avg30LastYear` populated for any type whose WMY tier carries // the year-ago window - // - `lastSeenByType` from the `DISTINCT ON (type)` latest read + // - `lastSeenByType` from the per-type latest read // // `anomalyCount` and `slope90` left this wire in v1.37.19: the slim // path could never compute the former (it always reported 0) and no @@ -292,7 +285,7 @@ async function buildAnalyticsResponse(user: AuthedUser, locale: Locale) { // `avg30LastYear` on types with no WMY coverage, which matches the // pre-fix behaviour for tenants whose year-ago window happened to // fall outside the 425-day floor. - const slim = await computeSummariesSlice(user.id); + const slim = await readSummariesSlice(user.id); // Same module gate as the slim slice and the snapshot builder — the // default slice feeds the dashboard too (it is the thick half of the // legacy two-cell path), so it cannot be the one feed that leaks a diff --git a/src/app/api/insights/targets/__tests__/route.test.ts b/src/app/api/insights/targets/__tests__/route.test.ts index 9338f928d..d83596dc3 100644 --- a/src/app/api/insights/targets/__tests__/route.test.ts +++ b/src/app/api/insights/targets/__tests__/route.test.ts @@ -170,9 +170,9 @@ beforeEach(() => { ( prisma.moodEntryRollup.findMany as ReturnType ).mockResolvedValue([]); - // v1.28.25 — the latest-ever-per-type read is a raw `DISTINCT ON` - // (Prisma's `distinct` dedups client-side after pulling every row). (prisma.$queryRawUnsafe as ReturnType).mockResolvedValue([]); + // The latest-ever-per-type read (`readLatestEver`). + (prisma.$queryRaw as ReturnType).mockResolvedValue([]); vi.mocked(readGlucoseTargetSummaries).mockResolvedValue([]); vi.mocked(readRestingPulseProxy).mockResolvedValue([]); }); @@ -321,35 +321,30 @@ describe("GET /api/insights/targets — mood-rollup tier swap", () => { expect(moodTarget).toBeUndefined(); }); - it("reads latest-ever-per-type via raw DISTINCT ON with the one-year floor", async () => { - // Audit High finding 3 established the 365-day floor; v1.28.25 moves - // the read onto a real Postgres `DISTINCT ON` (Prisma's `distinct` - // dedups in the driver AFTER pulling every row in the window, which - // on a dense-type tenant shipped a year of raw rows to Node just to - // keep seven). Pin the raw query: DISTINCT ON, per-type descending - // walk, user id + floored date as bound parameters. + it("reads latest-ever-per-type one index probe per type, floored at one year", async () => { + // Audit High finding 3 established the 365-day floor. The read is the + // shared `readLatestEver`: driven from the type list, one LIMIT 1 probe + // per type, so a dense type never sorts a year of raw rows to keep one. await callGet(makeReq()); - const rawCalls = (prisma.$queryRawUnsafe as ReturnType).mock - .calls; - const distinctCall = rawCalls.find( - (c) => typeof c[0] === "string" && c[0].includes("DISTINCT ON"), + const rawCalls = (prisma.$queryRaw as ReturnType).mock.calls; + const latestCall = rawCalls.find( + (c) => + Array.isArray(c[0]) && + (c[0] as string[]).join("?").includes("CROSS JOIN LATERAL"), ); - expect(distinctCall).toBeDefined(); - const [sql, boundUserId, flooredAt] = distinctCall as [ - string, - string, - Date, - ]; - expect(sql).toContain(`DISTINCT ON (m."type")`); - expect(sql).toContain(`ORDER BY m."type" ASC, m."measured_at" DESC`); - // Parameter-bound, never spliced: user id + date ride $1/$2. - expect(sql).toContain("$1"); - expect(sql).toContain("$2"); - expect(boundUserId).toBe("user-targets-1"); + expect(latestCall).toBeDefined(); + const [strings, ...values] = latestCall as [string[], ...unknown[]]; + expect(strings.join("?")).not.toContain("DISTINCT ON"); + expect(values).toContain("user-targets-1"); + const floor = values.find( + (v): v is { values: unknown[] } => + typeof v === "object" && v !== null && !Array.isArray(v), + ); + const flooredAt = floor?.values[0]; expect(flooredAt).toBeInstanceOf(Date); const oneYearMs = 365 * 24 * 60 * 60 * 1000; - const ageMs = Date.now() - flooredAt.getTime(); + const ageMs = Date.now() - (flooredAt as Date).getTime(); // Floor is somewhere around one year ago — give a ±1-hour window // for clock drift between the route's `Date.now()` and the test // assertion. @@ -454,7 +449,7 @@ describe("GET /api/insights/targets — mood-rollup tier swap", () => { timeZone, ), ); - (prisma.$queryRawUnsafe as ReturnType).mockResolvedValue([ + (prisma.$queryRaw as ReturnType).mockResolvedValue([ { type: "WEIGHT", value: 70 }, { type: "BLOOD_PRESSURE_SYS", value: 125 }, { type: "BLOOD_PRESSURE_DIA", value: 75 }, diff --git a/src/lib/analytics/__tests__/summaries-slice.test.ts b/src/lib/analytics/__tests__/summaries-slice.test.ts index a503f58e0..de194fba8 100644 --- a/src/lib/analytics/__tests__/summaries-slice.test.ts +++ b/src/lib/analytics/__tests__/summaries-slice.test.ts @@ -66,7 +66,8 @@ vi.mock("@/lib/analytics/source-rank-sql", () => { import { prisma } from "@/lib/db"; import { canonicalMeasurementsCte } from "@/lib/analytics/source-rank-sql"; import { startOfUtcDay } from "@/lib/tz/start-of-utc-day"; -import { computeSummariesSlice } from "../summaries-slice"; +import { caches, __resetAllCachesForTests } from "@/lib/cache/server-cache"; +import { computeSummariesSlice, readSummariesSlice } from "../summaries-slice"; const RAW = prisma.$queryRaw as unknown as ReturnType; const UNSAFE = prisma.$queryRawUnsafe as unknown as ReturnType; @@ -686,3 +687,37 @@ describe("computeSummariesSlice", () => { }); }); }); + +describe("readSummariesSlice — the one cache cell", () => { + beforeEach(() => { + __resetAllCachesForTests(); + UNSAFE.mockResolvedValue([]); + }); + + it("builds the slice once for every consumer inside the fresh TTL", async () => { + const first = await readSummariesSlice("user-cell", { serveStale: true }); + const builds = RAW.mock.calls.length; + expect(builds).toBeGreaterThan(0); + + const composed = await readSummariesSlice("user-cell"); + const snapshot = await readSummariesSlice("user-cell", { + coverage: new Map(), + }); + + expect(composed).toBe(first); + expect(snapshot).toBe(first); + expect(RAW.mock.calls.length).toBe(builds); + }); + + it("serves stale only to the edge reader; a composing reader rebuilds", async () => { + const before = await readSummariesSlice("user-stale"); + caches.analytics.markStaleByPrefix("user-stale|"); + + const edge = await readSummariesSlice("user-stale", { serveStale: true }); + expect(edge).toBe(before); + + caches.analytics.markStaleByPrefix("user-stale|"); + const composed = await readSummariesSlice("user-stale"); + expect(composed).not.toBe(before); + }); +}); diff --git a/src/lib/analytics/score/__tests__/config-change-is-not-a-health-event.test.ts b/src/lib/analytics/score/__tests__/config-change-is-not-a-health-event.test.ts index 2d46fa5c1..8e1511cb5 100644 --- a/src/lib/analytics/score/__tests__/config-change-is-not-a-health-event.test.ts +++ b/src/lib/analytics/score/__tests__/config-change-is-not-a-health-event.test.ts @@ -189,7 +189,7 @@ const fakeDb = vi.hoisted(() => { vi.mock("@/lib/db", () => ({ prisma: fakeDb })); // ── Everything around the score, faked at its own seam ────────────── -const computeSummariesSlice = vi.fn(); +const readSummariesSlice = vi.fn(); const probeRollupCoverage = vi.fn(); const isFullyCovered = vi.fn(); const readMoodDayRollups = vi.fn(); @@ -205,7 +205,7 @@ const loadIntradayPulse = vi.fn(); const readDayMeanSeries = vi.fn(); vi.mock("@/lib/analytics/summaries-slice", () => ({ - computeSummariesSlice: (...a: unknown[]) => computeSummariesSlice(...a), + readSummariesSlice: (...a: unknown[]) => readSummariesSlice(...a), })); vi.mock("@/lib/rollups/measurement-coverage", () => ({ probeRollupCoverage: (...a: unknown[]) => probeRollupCoverage(...a), @@ -381,7 +381,7 @@ beforeEach(() => { vi.clearAllMocks(); __resetAllCachesForTests(); dismissedKeys.clear(); - computeSummariesSlice.mockResolvedValue({ + readSummariesSlice.mockResolvedValue({ summaries: {}, lastSeenByType: {}, bmi: null, diff --git a/src/lib/analytics/summaries-slice.ts b/src/lib/analytics/summaries-slice.ts index b1a5519a2..4bc962cfc 100644 --- a/src/lib/analytics/summaries-slice.ts +++ b/src/lib/analytics/summaries-slice.ts @@ -30,7 +30,7 @@ * windowed columns (`avg7`, `avg30`, and the slope tuples). The * heavy COUNT/MIN/MAX/AVG columns are NOT projected. * - * 3. **`DISTINCT ON (type)` latest read** — most-recent `(type, + * 3. **Latest read** (`readLatestCanonical`) — most-recent `(type, * value, measured_at)` triplet per type for the `latest` field * and the `lastSeenByType` freshness map. * @@ -77,6 +77,7 @@ import { import type { DataSummary } from "@/lib/analytics/trends"; import { measurementTypeEnum } from "@/lib/validations/measurement"; import { annotate } from "@/lib/logging/context"; +import { caches, type ServerCache } from "@/lib/cache/server-cache"; import { ensureUserRollupsFresh, readRollupBuckets, @@ -95,6 +96,7 @@ import { buildSourceRankCase, canonicalMeasurementsCte, } from "@/lib/analytics/source-rank-sql"; +import { readLatestCanonical } from "@/lib/measurements/latest-canonical"; import { isFullyCovered, probeRollupCoverage, @@ -223,19 +225,6 @@ interface WindowedAggregateRow extends NarrowAggregateRow { r2_30: number | null; } -interface LatestRow { - type: string; - value: number; - /** - * v1.4.34 IW-B — per-type freshness timestamp. The DISTINCT ON pass - * already orders by `measured_at DESC` to pick the latest row's - * value; surfacing `measured_at` from the same row costs zero extra - * round-trips and feeds the dashboard tile-strip's stale-data - * caption (driven via ``). - */ - measured_at: Date; -} - /** Same threshold/rounding contract as `trendSlope()` in `trends.ts`. */ function buildSlope( slope: number | null, @@ -373,6 +362,35 @@ export async function computeSummariesSlice( ); } +/** + * The one cache cell of the summaries slice. The tile-strip route, the thick + * analytics body and the dashboard snapshot all read the slice; each used to + * compute it inside its own cell, so one dashboard mount built it up to three + * times over the same rows. + * + * `serveStale` is for the caller that answers the client with the slice + * itself. The composing callers are stale-while-revalidate cells of their + * own and read fresh-only: a background refresh after a write must not + * compose the pre-write slice and cache it again as fresh. + */ +export async function readSummariesSlice( + userId: string, + options: { coverage?: RollupCoverageMap; serveStale?: boolean } = {}, +): Promise { + const cache = caches.analytics as ServerCache; + const key = summariesSliceCacheKey(userId); + const build = () => computeSummariesSlice(userId, options.coverage); + const { value, outcome } = options.serveStale + ? await cache.wrapSwr(key, build) + : await cache.wrap(key, build); + annotate({ meta: { "cache.summaries_slice.outcome": outcome } }); + return value; +} + +export function summariesSliceCacheKey(userId: string): string { + return `${userId}|summaries`; +} + /** * `latest` of an hourly-mean type (pulse) is the value of its latest local * day, not its latest single reading. @@ -559,15 +577,13 @@ async function computeFromRollups(userId: string): Promise { // v1.11.1 — resolve the user's source-priority ladders once, then build the // rank CASE expressions used to collapse overlapping sources to the canonical // reading per (type, day) inside each query. `"type"`/`"source"` variants for - // the unqualified rollup + inner-subquery columns, `m."…"` for the latest - // read's outer alias. + // the unqualified rollup + inner-subquery columns. const priorityJson = await loadUserSourcePriority(userId); const rankUnqualified = buildSourceRankCase( priorityJson, '"type"', '"source"', ); - const rankM = buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'); const rankPerSource = buildSourceRankCase( priorityJson, 'p."type"', @@ -618,22 +634,7 @@ async function computeFromRollups(userId: string): Promise { `, userId, ), - // v1.11.1 — the latest tile reflects the canonical source for the latest - // day, matching the chart: order by latest day first, then the ladder rank - // (canonical source wins), then the latest reading of that source. - prisma.$queryRawUnsafe( - ` - SELECT DISTINCT ON (m."type") - m."type"::text AS type, - m."value"::double precision AS value, - m."measured_at" AS measured_at - FROM measurements m - WHERE m."user_id" = $1 - AND m."deleted_at" IS NULL - ORDER BY m."type", date_trunc('day', m."measured_at") DESC, (${rankM}), m."measured_at" DESC - `, - userId, - ), + readLatestCanonical(userId, priorityJson), // v1.4.37.2 hotfix — the v1.4.35 implementation read EVERY DAY // rollup bucket for the user (`findMany` without a `bucketStart` // window) and then composed `count / min / max / mean` in JS. @@ -1055,7 +1056,6 @@ async function computeFromLiveAggregate( '"type"', '"source"', ); - const rankM = buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'); const [allTime, windowed, latests] = await Promise.all([ prisma.$queryRawUnsafe( @@ -1129,19 +1129,7 @@ async function computeFromLiveAggregate( `, userId, ), - prisma.$queryRawUnsafe( - ` - SELECT DISTINCT ON (m."type") - m."type"::text AS type, - m."value"::double precision AS value, - m."measured_at" AS measured_at - FROM measurements m - WHERE m."user_id" = $1 - AND m."deleted_at" IS NULL - ORDER BY m."type", date_trunc('day', m."measured_at") DESC, (${rankM}), m."measured_at" DESC - `, - userId, - ), + readLatestCanonical(userId, priorityJson), ]); const latestByType = new Map(); diff --git a/src/lib/dashboard/__tests__/snapshot.test.ts b/src/lib/dashboard/__tests__/snapshot.test.ts index 03c7e4fab..53daaec70 100644 --- a/src/lib/dashboard/__tests__/snapshot.test.ts +++ b/src/lib/dashboard/__tests__/snapshot.test.ts @@ -13,7 +13,7 @@ */ import { describe, it, expect, vi, beforeEach } from "vitest"; -const computeSummariesSlice = vi.fn(); +const readSummariesSlice = vi.fn(); const probeRollupCoverage = vi.fn(); const isFullyCovered = vi.fn(); const readMoodDayRollups = vi.fn(); @@ -24,7 +24,7 @@ const buildMedsTodayBlock = vi.fn(); const computeUserHealthScoreMock = vi.fn(); vi.mock("@/lib/analytics/summaries-slice", () => ({ - computeSummariesSlice: (...a: unknown[]) => computeSummariesSlice(...a), + readSummariesSlice: (...a: unknown[]) => readSummariesSlice(...a), })); vi.mock("@/lib/rollups/measurement-coverage", () => ({ probeRollupCoverage: (...a: unknown[]) => probeRollupCoverage(...a), @@ -210,7 +210,7 @@ function baseUser( beforeEach(() => { vi.clearAllMocks(); - computeSummariesSlice.mockResolvedValue({ + readSummariesSlice.mockResolvedValue({ summaries: { WEIGHT: { ...emptySummary, count: 3, latest: 80 } }, lastSeenByType: { WEIGHT: { lastSeenAt: new Date().toISOString() }, @@ -301,7 +301,31 @@ describe("buildDashboardSnapshot — envelope shape", () => { expect(probeRollupCoverage).toHaveBeenCalledTimes(1); // The already-probed map is passed as the slice's second argument so // it reuses it instead of probing again. - expect(computeSummariesSlice).toHaveBeenCalledWith("user-1", coverageMap); + expect(readSummariesSlice).toHaveBeenCalledWith("user-1", { + coverage: coverageMap, + }); + }); +}); + +describe("buildDashboardSnapshot — the shared summaries cell", () => { + it("folds the water tile into a copy, never into the cached slice", async () => { + probeRollupCoverage.mockResolvedValue(new Map([["WEIGHT", false]])); + isFullyCovered.mockReturnValue(false); + const cached = { + summaries: { WEIGHT: { ...emptySummary, count: 3, latest: 80 } }, + lastSeenByType: { WEIGHT: { lastSeenAt: new Date().toISOString() } }, + bmi: null, + }; + readSummariesSlice.mockResolvedValue(cached); + fakePrisma.nutrientIntakeDay.findMany.mockResolvedValue([ + { day: new Date().toISOString().slice(0, 10), amount: 1500 }, + ]); + + const snap = await buildDashboardSnapshot(fakePrisma, baseUser()); + + expect(snap.tiles.summaries.NUTRIENT_WATER).toBeDefined(); + expect(Object.keys(cached.summaries)).toEqual(["WEIGHT"]); + expect(Object.keys(cached.lastSeenByType)).toEqual(["WEIGHT"]); }); }); @@ -891,7 +915,7 @@ describe("buildDashboardSnapshot — metricStates (iOS cold-launch seed)", () => it("keys each entry by the iOS MetricKind raw value (incl. the non-obvious raws)", async () => { const now = new Date().toISOString(); - computeSummariesSlice.mockResolvedValue({ + readSummariesSlice.mockResolvedValue({ summaries: { WEIGHT: { ...emptySummary, count: 1, latest: 80 }, OXYGEN_SATURATION: { ...emptySummary, count: 1, latest: 98 }, @@ -948,7 +972,7 @@ describe("buildDashboardSnapshot — metricStates (iOS cold-launch seed)", () => }); it("omits metrics with no latest value or no timestamp", async () => { - computeSummariesSlice.mockResolvedValue({ + readSummariesSlice.mockResolvedValue({ summaries: { WEIGHT: { ...emptySummary, count: 0, latest: null }, PULSE: { ...emptySummary, count: 1, latest: 60 }, @@ -1162,7 +1186,7 @@ describe("buildDashboardSnapshot — module gating (v1.18.0)", () => { priorMonth: { pct: 60 }, priorYear: { pct: 50 }, }); - computeSummariesSlice.mockResolvedValue({ + readSummariesSlice.mockResolvedValue({ summaries: { WEIGHT: { ...emptySummary, count: 3, latest: 80 }, SLEEP_DURATION: { ...emptySummary, count: 5, latest: 420 }, diff --git a/src/lib/dashboard/snapshot.ts b/src/lib/dashboard/snapshot.ts index 4d404906e..bcd5054c5 100644 --- a/src/lib/dashboard/snapshot.ts +++ b/src/lib/dashboard/snapshot.ts @@ -35,7 +35,7 @@ * The builder NEVER POSTs `/api/insights/generate`. */ import type { PrismaClient, MeasurementType } from "@/generated/prisma/client"; -import { computeSummariesSlice } from "@/lib/analytics/summaries-slice"; +import { readSummariesSlice } from "@/lib/analytics/summaries-slice"; import type { SleepSourceDiscrepancy } from "@/lib/analytics/sleep-night"; import { getUnitForType } from "@/lib/validations/measurement"; import { @@ -1013,7 +1013,7 @@ async function liftBriefing( /** * v1.7.0 — derive the per-metric latest-reading block from the slim * summaries slice already fetched for `tiles.summaries`. ZERO extra DB - * round-trips: the slim slice's `DISTINCT ON (type)` read already + * round-trips: the slim slice's per-type latest read already * carries the latest `value` (`summaries[type].latest`) and the * matching timestamp (`lastSeenByType[type].lastSeenAt`); the unit is * the static canonical-unit lookup. This keeps the snapshot pool-safe @@ -1329,7 +1329,7 @@ export async function buildDashboardSnapshot( ] = await Promise.all([ // A5 — reuse the coverage map already probed above so the slice // doesn't re-run the identical `probeRollupCoverage` query. - time("summaries", () => computeSummariesSlice(user.id, coverage)), + time("summaries", () => readSummariesSlice(user.id, { coverage })), time("mood", () => buildMoodBlock(prisma, user.id)), // v1.34 — the thick slice awaits the shared `modulesPromise` (the same // pattern the score-ring task uses) so the hero ring's health score can @@ -1372,29 +1372,30 @@ export async function buildDashboardSnapshot( // `NUTRIENT_WATER` key BEFORE the module gate below strips it (like // every other summary type) when `nutrients` is off. Not a real // `MeasurementType` — see the widened `SUMMARY_TYPE_MODULE` note. - if (nutrientWater) { - const lastMs = new Date(nutrientWater.lastSeenAt).getTime(); - const daysAgo = Math.floor((nowMs - lastMs) / (24 * 60 * 60 * 1000)); - slimRaw.summaries = { - ...slimRaw.summaries, - NUTRIENT_WATER: nutrientWater.summary, - }; - slimRaw.lastSeenByType = { - ...slimRaw.lastSeenByType, - NUTRIENT_WATER: { lastSeenAt: nutrientWater.lastSeenAt, daysAgo }, - }; - } + // The slice is the shared cache cell's value: fold into copies, never into + // the cached object itself. + const slimSummaries = nutrientWater + ? { ...slimRaw.summaries, NUTRIENT_WATER: nutrientWater.summary } + : slimRaw.summaries; + const slimLastSeen = nutrientWater + ? { + ...slimRaw.lastSeenByType, + NUTRIENT_WATER: { + lastSeenAt: nutrientWater.lastSeenAt, + daysAgo: Math.floor( + (nowMs - new Date(nutrientWater.lastSeenAt).getTime()) / + (24 * 60 * 60 * 1000), + ), + }, + } + : slimRaw.lastSeenByType; // v1.18.0 — strip disabled-module data before it leaves the server. // Mood is blanked when the mood module is off; sleep / glucose summary // types are dropped from the slim slice; the glucose clinical panel + // per-context block are cleared when glucose is off; the layout + // catalogue hide every disabled-module widget. - const slim = gateSummariesByModules( - slimRaw.summaries, - slimRaw.lastSeenByType, - modules, - ); + const slim = gateSummariesByModules(slimSummaries, slimLastSeen, modules); const mood = modules.mood === false ? { summary: null, entries: [] as DashboardSnapshotMoodEntry[] } diff --git a/src/lib/dashboard/summary-reads.ts b/src/lib/dashboard/summary-reads.ts index 15fc47786..a5206307e 100644 --- a/src/lib/dashboard/summary-reads.ts +++ b/src/lib/dashboard/summary-reads.ts @@ -15,7 +15,7 @@ * an equality on the leading type column, which turns a full pass over * the account into one bounded lookup per type. */ -import type { PrismaClient } from "@/generated/prisma/client"; +import { Prisma, type PrismaClient } from "@/generated/prisma/client"; import type { MeasurementType } from "@/generated/prisma/client"; /** v1.4.38 W-F — the single most recent reading per measurement @@ -29,7 +29,8 @@ import type { MeasurementType } from "@/generated/prisma/client"; * was still in the database. The all-time aggregate (`groupBy` on * `measurements`) already proves the row exists; the row count is * still bounded by `|measurementTypes|`, now because the query is - * driven FROM that list (v1.38.5) rather than grouped down to it. */ + * driven FROM that list (v1.38.5) rather than grouped down to it. + * `since` drops a type whose latest reading is older than it. */ export interface LatestEverRow { type: MeasurementType; value: number; @@ -90,7 +91,11 @@ export function readLatestEver( db: PrismaClient, userId: string, measurementTypes: readonly MeasurementType[], + since: Date | null = null, ): Promise { + const floor = since + ? Prisma.sql`AND m."measured_at" >= ${since}` + : Prisma.empty; return db.$queryRaw` SELECT mt."type" AS type, @@ -105,6 +110,7 @@ export function readLatestEver( WHERE m."user_id" = ${userId} AND m."type" = mt."type" AND m."deleted_at" IS NULL + ${floor} ORDER BY m."measured_at" DESC LIMIT 1 ) l diff --git a/src/lib/insights/comprehensive-aggregator.ts b/src/lib/insights/comprehensive-aggregator.ts index 63f22ec82..d6b27a377 100644 --- a/src/lib/insights/comprehensive-aggregator.ts +++ b/src/lib/insights/comprehensive-aggregator.ts @@ -114,6 +114,7 @@ import { type AccumulatorBucketRow, } from "@/lib/rollups/measurement-read"; import { windowWeighting } from "@/lib/measurements/day-statistic"; +import { readLatestCanonical } from "@/lib/measurements/latest-canonical"; import { SESSION_DAY_FRAME, dayWeightedRows, @@ -198,11 +199,9 @@ interface NarrowAggregateRow { // they no longer scan the raw measurements partition on the warm path. } -/** Latest raw value per type (DISTINCT ON over the 90-day window). */ -interface LatestRow { - type: string; - value: number; - measured_at: Date; +/** The latest canonical reading per type is read only inside the 90-day window. */ +function latestFloor(): Date { + return new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); } /** @@ -426,8 +425,7 @@ async function buildFromRollups( // (`type: { in: [...] }`) plus per-sub-query wall-clock timings. // v1.11.1 — collapse overlapping sources to the ladder-canonical reading per // (type, day) before aggregating. `rankUnqualified` ranks the inner - // canonical-source subquery; `rankM` ranks the latest read's `m.`-aliased - // columns. The window_stats CTE + the main narrow both read from the + // canonical-source subquery. The window_stats CTE + the main narrow both read from the // canonical-source-filtered set so a dual-source vital isn't blended. const priorityJson = await loadUserSourcePriority(userId); const rankUnqualified = buildSourceRankCase( @@ -435,7 +433,6 @@ async function buildFromRollups( '"type"', '"source"', ); - const rankM = buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'); const [narrows, latests, dayBuckets, bpRawRows, firstRows] = await Promise.all([ @@ -484,20 +481,7 @@ async function buildFromRollups( ), ), timeSubquery(timings, "latests", () => - prisma.$queryRawUnsafe( - ` - SELECT DISTINCT ON (m."type") - m."type"::text AS type, - m."value"::double precision AS value, - m."measured_at" AS measured_at - FROM measurements m - WHERE m."user_id" = $1 - AND m."measured_at" >= (NOW() AT TIME ZONE 'UTC') - INTERVAL '90 days' - AND m."deleted_at" IS NULL - ORDER BY m."type", date_trunc('day', m."measured_at") DESC, (${rankM}), m."measured_at" DESC - `, - userId, - ), + readLatestCanonical(userId, priorityJson, latestFloor()), ), // v1.4.35 — DAY rollup buckets for the trailing 90 days, every // type. Drives both the per-type `count / min / max / mean` @@ -676,7 +660,6 @@ async function buildFromLiveAggregate( '"type"', '"source"', ); - const rankM = buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'); const [aggregates, latests, dayBuckets, bpRawRows] = await Promise.all([ timeSubquery(timings, "heavy", () => @@ -760,20 +743,7 @@ async function buildFromLiveAggregate( ), ), timeSubquery(timings, "latests", () => - prisma.$queryRawUnsafe( - ` - SELECT DISTINCT ON (m."type") - m."type"::text AS type, - m."value"::double precision AS value, - m."measured_at" AS measured_at - FROM measurements m - WHERE m."user_id" = $1 - AND m."measured_at" >= (NOW() AT TIME ZONE 'UTC') - INTERVAL '90 days' - AND m."deleted_at" IS NULL - ORDER BY m."type", date_trunc('day', m."measured_at") DESC, (${rankM}), m."measured_at" DESC - `, - userId, - ), + readLatestCanonical(userId, priorityJson, latestFloor()), ), // Buckets may exist for some types even when the COUNT probe came // back zero (race window between the probe and a sibling diff --git a/src/lib/measurements/latest-canonical.ts b/src/lib/measurements/latest-canonical.ts new file mode 100644 index 000000000..a8e7f3ec9 --- /dev/null +++ b/src/lib/measurements/latest-canonical.ts @@ -0,0 +1,66 @@ +import { prisma } from "@/lib/db"; +import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; +import { measurementTypeEnum } from "@/lib/validations/measurement"; + +export interface LatestCanonicalRow { + type: string; + value: number; + measured_at: Date; +} + +/** + * The latest canonical reading per measurement type: the latest UTC day + * first, then the source ladder (the canonical source wins), then the latest + * reading of that source, so a tile's headline matches its chart. `since` + * drops a type whose latest reading is older than it. + * + * Driven from the type list rather than a `DISTINCT ON (type)` over the + * account: Postgres 16 has no skip scan, so that shape sorted every live row + * of the account to keep one per type. Here each type costs one probe of + * `measurements_live_covering_idx` for its latest instant and one read of that + * single day. The closed day window matters to the planner as much as to the + * result: with only the lower bound it estimates a third of the type's rows + * per probe, and the summed estimate crosses `jit_above_cost`, whose + * compilation then costs more than the query itself. + */ +export function readLatestCanonical( + userId: string, + priorityJson: unknown, + since: Date | null = null, +): Promise { + const rank = buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'); + return prisma.$queryRawUnsafe( + ` + SELECT + mt."type"::text AS type, + l.value AS value, + l.measured_at AS measured_at + FROM unnest($2::"measurement_type"[]) AS mt("type") + CROSS JOIN LATERAL ( + SELECT date_trunc('day', d."measured_at") AS day + FROM measurements d + WHERE d."user_id" = $1 + AND d."type" = mt."type" + AND d."deleted_at" IS NULL + ${since ? `AND d."measured_at" >= $3` : ""} + ORDER BY d."measured_at" DESC + LIMIT 1 + ) last_day + CROSS JOIN LATERAL ( + SELECT + m."value"::double precision AS value, + m."measured_at" AS measured_at + FROM measurements m + WHERE m."user_id" = $1 + AND m."type" = mt."type" + AND m."deleted_at" IS NULL + AND m."measured_at" >= last_day.day + AND m."measured_at" < last_day.day + INTERVAL '1 day' + ORDER BY (${rank}), m."measured_at" DESC, m."id" DESC + LIMIT 1 + ) l + ORDER BY mt."type" + `, + ...[userId, measurementTypeEnum.options, ...(since ? [since] : [])], + ); +} diff --git a/src/lib/targets/build-response.ts b/src/lib/targets/build-response.ts index 7b0ddc569..ac8ad62ce 100644 --- a/src/lib/targets/build-response.ts +++ b/src/lib/targets/build-response.ts @@ -32,6 +32,7 @@ import { } from "@/lib/rollups/mood-rollups"; import { readRestingPulseProxy } from "@/lib/analytics/resting-pulse-read"; import { readSevenDayMeans } from "@/lib/analytics/summaries-slice"; +import { readLatestEver } from "@/lib/dashboard/summary-reads"; import { summarizeSleepNights, type SleepStageRow, @@ -145,31 +146,7 @@ export async function buildTargetsResponse(user: AuthedUser) { select: { type: true, value: true, measuredAt: true }, }), ), - limit(() => { - for (const type of types) { - if (!/^[A-Z0-9_]+$/.test(type)) { - throw new Error(`invalid measurement type: ${type}`); - } - } - const typeList = types - .map((type) => `'${type}'::"measurement_type"`) - .join(","); - return prisma.$queryRawUnsafe< - Array<{ type: MeasurementType; value: number }> - >( - `SELECT DISTINCT ON (m."type") - m."type"::text AS type, - m."value" AS value - FROM measurements m - WHERE m."user_id" = $1 - AND m."type" IN (${typeList}) - AND m."measured_at" >= $2 - AND m."deleted_at" IS NULL - ORDER BY m."type" ASC, m."measured_at" DESC`, - userId, - oneYearAgo, - ); - }), + limit(() => readLatestEver(prisma, userId, types, oneYearAgo)), limit(() => prisma.measurement.findMany({ where: { diff --git a/tests/integration/latest-canonical-read.test.ts b/tests/integration/latest-canonical-read.test.ts new file mode 100644 index 000000000..88ffbe801 --- /dev/null +++ b/tests/integration/latest-canonical-read.test.ts @@ -0,0 +1,114 @@ +/** + * The latest canonical reading per type, against a real Postgres. + * + * `readLatestCanonical` replaced five `DISTINCT ON (type)` reads that sorted + * every row of the account. The contract it keeps: the latest UTC day decides, + * inside it the source ladder decides (Withings outranks Apple Health for + * weight), inside the winning source the latest reading; a type with no + * reading drops out, and `since` drops a type whose latest reading is older. + * The day window is closed: an older day of the ladder winner never beats a + * newer day of a lower source. + */ +import { beforeAll, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { readLatestCanonical } from "@/lib/measurements/latest-canonical"; + +const prisma = getPrismaClient(); +const USER = "latest-canonical"; +const now = new Date(); +/** UTC midnight two days back. */ +const D = Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() - 2, +); +const at = (day: number, hour: number) => + new Date(D + day * 86_400_000 + hour * 3_600_000); + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, email: `${USER}@example.test` }, + }); + const rows: Array<[string, string, number, Date, string]> = [ + // WEIGHT: the latest day holds Apple at 20:00 and Withings at 07:00; + // Withings ranks first, so 82 wins over the later 81. + ["WEIGHT", "WITHINGS", 80, at(-1, 8), "kg"], + ["WEIGHT", "WITHINGS", 82, at(0, 7), "kg"], + ["WEIGHT", "APPLE_HEALTH", 81, at(0, 20), "kg"], + // BODY_FAT: Withings is the ladder winner but only on an older day; the + // newer Apple day wins. + ["BODY_FAT", "WITHINGS", 20, at(-1, 8), "%"], + ["BODY_FAT", "APPLE_HEALTH", 21, at(0, 9), "%"], + // PULSE: one source, the latest reading of the latest day. + ["PULSE", "APPLE_HEALTH", 60, at(0, 6), "bpm"], + ["PULSE", "APPLE_HEALTH", 64, at(0, 22), "bpm"], + // BLOOD_GLUCOSE: only a reading far in the past. + ["BLOOD_GLUCOSE", "MANUAL", 95, new Date(D - 200 * 86_400_000), "mg/dL"], + ]; + await prisma.measurement.createMany({ + data: rows.map(([type, source, value, measuredAt, unit], i) => ({ + id: `${USER}-${i}`, + userId: USER, + type: type as "WEIGHT", + source: source as "WITHINGS", + value, + unit, + measuredAt, + })), + }); + // A deleted later reading never counts. + await prisma.measurement.create({ + data: { + id: `${USER}-deleted`, + userId: USER, + type: "PULSE", + source: "APPLE_HEALTH", + value: 99, + unit: "bpm", + measuredAt: at(0, 23), + deletedAt: new Date(), + }, + }); +}); + +describe("readLatestCanonical", () => { + it("picks the latest day, then the ladder, then the latest reading", async () => { + const rows = await readLatestCanonical(USER, null); + const byType = new Map(rows.map((r) => [r.type, r])); + expect([...byType.keys()].sort()).toEqual([ + "BLOOD_GLUCOSE", + "BODY_FAT", + "PULSE", + "WEIGHT", + ]); + expect(byType.get("WEIGHT")?.value).toBe(82); + expect(byType.get("WEIGHT")?.measured_at.toISOString()).toBe( + at(0, 7).toISOString(), + ); + expect(byType.get("BODY_FAT")?.value).toBe(21); + expect(byType.get("PULSE")?.value).toBe(64); + expect(byType.get("BLOOD_GLUCOSE")?.value).toBe(95); + }); + + it("drops a type whose latest reading is older than `since`", async () => { + const rows = await readLatestCanonical( + USER, + null, + new Date(Date.now() - 90 * 86_400_000), + ); + expect(rows.map((r) => r.type).sort()).toEqual([ + "BODY_FAT", + "PULSE", + "WEIGHT", + ]); + }); + + it("honours a stored source order", async () => { + const rows = await readLatestCanonical(USER, { + weight: ["APPLE_HEALTH", "WITHINGS"], + }); + expect(rows.find((r) => r.type === "WEIGHT")?.value).toBe(81); + }); +}); From 6026a28fe42713fe9bf9a9a8a9b88b47462c58e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 03:35:19 +0200 Subject: [PATCH 02/11] fix(dashboard): key streak and compliance days in the person's own zone `measured_at` and `scheduled_for` are naive UTC timestamps. The dashboard streak and the medication-compliance backfill keyed them with a single `AT TIME ZONE tz`, which reads the value as local time and shifts it the wrong way: in Berlin a reading at 23:30 UTC counted for the day before instead of the day after, so a streak could break or stretch near midnight and the backfill recomputed the wrong day. The streak's measurement days now come from `readActivityDayKeys`: one existence probe per local day instead of a DISTINCT over a year of readings. The DAY rollups cannot answer this because their buckets are UTC days. The two compliance-rollup readers had no caller left and are removed with the coverage probe that carried the same shift. A guard fails on any single shift of a naive column into a bound zone. --- .../dashboard-summary-read-plan-guard.test.ts | 7 +- .../naive-timestamp-zone-shift-guard.test.ts | 67 +++++++ src/app/api/dashboard/summary/route.ts | 43 +---- src/lib/dashboard/summary-reads.ts | 43 +++++ .../medication-compliance-rollups.test.ts | 103 ----------- .../rollups/medication-compliance-rollups.ts | 163 ++---------------- tests/integration/local-day-keys.test.ts | 124 +++++++++++++ 7 files changed, 258 insertions(+), 292 deletions(-) create mode 100644 src/__tests__/naive-timestamp-zone-shift-guard.test.ts create mode 100644 tests/integration/local-day-keys.test.ts diff --git a/src/__tests__/dashboard-summary-read-plan-guard.test.ts b/src/__tests__/dashboard-summary-read-plan-guard.test.ts index 44c9707d8..b1b9a14e1 100644 --- a/src/__tests__/dashboard-summary-read-plan-guard.test.ts +++ b/src/__tests__/dashboard-summary-read-plan-guard.test.ts @@ -97,10 +97,9 @@ describe("dashboard summary reads — the route keeps using them", () => { expect(route).toMatch( /readSparkBuckets\(\s*prisma,\s*userId,\s*measurementTypes,/, ); - // No second copy of the rollup read can drift out of the guarded - // module. (`FROM measurements` still appears in the route's own - // streak-day aggregate, which is a bounded 365-row read and not part - // of this finding.) + expect(route).toMatch(/readActivityDayKeys\(\s*prisma,\s*userId,/); + // No second copy of a read can drift out of the guarded module. expect(route.includes("FROM measurement_rollups")).toBe(false); + expect(route.includes("FROM measurements")).toBe(false); }); }); diff --git a/src/__tests__/naive-timestamp-zone-shift-guard.test.ts b/src/__tests__/naive-timestamp-zone-shift-guard.test.ts new file mode 100644 index 000000000..d0e55846c --- /dev/null +++ b/src/__tests__/naive-timestamp-zone-shift-guard.test.ts @@ -0,0 +1,67 @@ +/** + * Structural guard: a naive UTC timestamp is read as UTC before it is + * shifted into a person's zone. + * + * `measured_at`, `scheduled_for`, `taken_at` and the other `*_at` instants + * are `timestamp` columns holding UTC wall-clock time. On that type, + * `col AT TIME ZONE tz` means "this is local time in tz" and returns the + * matching instant, so the value moves by the offset in the wrong direction: + * in Berlin a 23:30 UTC reading lands on the day before instead of the day + * after. The right form is `(col AT TIME ZONE 'UTC') AT TIME ZONE tz`. The + * dashboard streak and the medication-compliance backfill both keyed days + * with the single form, while their comments called it identical to + * `userDayKey`. + * + * The matcher flags a quoted `*_at` or `scheduled_for` column shifted + * straight into a bound zone (`${...}` or `$N`). A `timestamptz` column + * (`bucket_start`) is out of its reach by name. Comments are stripped first; + * the planted cases prove it fires. It is a tripwire, not a proof. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +const SINGLE_SHIFT = + /"(?:[a-z_]+_at|scheduled_for)"\s*\)?\s*AT\s+TIME\s+ZONE\s+(?:\$\{|\$\d)/i; + +function stripComments(text: string): string { + return text + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/^\s*\/\/.*$/gm, "") + .replace(/^\s*--.*$/gm, ""); +} + +describe("naive timestamp zone shifts", () => { + it("the matcher fires on the single shift and spares the double one", () => { + expect( + SINGLE_SHIFT.test(`to_char(m."measured_at" AT TIME ZONE \${tz}, 'D')`), + ).toBe(true); + expect(SINGLE_SHIFT.test(`("scheduled_for" AT TIME ZONE $3)`)).toBe(true); + expect( + SINGLE_SHIFT.test( + `("scheduled_for" AT TIME ZONE 'UTC') AT TIME ZONE \${tz}`, + ), + ).toBe(false); + expect(SINGLE_SHIFT.test(`r."bucket_start" AT TIME ZONE \${tz}`)).toBe( + false, + ); + }); + + it("no production read shifts a naive column once", () => { + const offenders = walkSourceFiles(SRC, { floor: 3000 }) + .filter( + (file) => + !file.startsWith("generated/") && + !file.includes("__tests__/") && + !/\.test\.tsx?$/.test(file), + ) + .filter((file) => + SINGLE_SHIFT.test(stripComments(readFileSync(join(SRC, file), "utf8"))), + ); + expect(offenders).toEqual([]); + }); +}); diff --git a/src/app/api/dashboard/summary/route.ts b/src/app/api/dashboard/summary/route.ts index 55a3d77c3..5fe712a16 100644 --- a/src/app/api/dashboard/summary/route.ts +++ b/src/app/api/dashboard/summary/route.ts @@ -78,6 +78,7 @@ import { cachedSwr, caches, type ServerCache } from "@/lib/cache/server-cache"; import { buildMedsTodayBlock } from "@/lib/dashboard/meds-today"; import { computeStreak } from "@/lib/dashboard/streak"; import { + readActivityDayKeys, readLatestEver, readSparkBuckets, } from "@/lib/dashboard/summary-reads"; @@ -329,15 +330,6 @@ function trendOf(values: number[]): MetricCard["trend"] { return delta > 0 ? "up" : "down"; } -/** v1.4.38 W-F — distinct activity day-keys from the streak window. - * The route only needs YYYY-MM-DD strings, so the aggregate runs - * `date_trunc('day', m.measured_at AT TIME ZONE $userTz)` and returns - * the day-keys directly. At most 365 rows; vs. the legacy `findMany` - * which could return tens of thousands. */ -interface ActivityDayRow { - day_key: string; -} - export const GET = apiHandler(async () => { // v1.37.0 — MANAGE-level read: computed over the whole record, with no // provider anywhere on the path. @@ -532,27 +524,8 @@ async function buildDashboardSummary( select: { takenAt: true, scheduledFor: true }, }), ), - // v1.4.38 W-F — replaces the legacy 365-day `measurement.findMany` - // that pulled every raw row in the window only to collapse them to - // day-keys in JS. The SQL `date_trunc` returns at most 365 rows - // even on accounts with millions of step samples. The `userTz` - // conversion is folded server-side so the day boundary matches - // `userDayKey`'s output (it formats the timestamp in the user's - // tz before slicing YYYY-MM-DD). - time( - "streakDays", - () => - prisma.$queryRaw` - SELECT DISTINCT - to_char( - (m."measured_at" AT TIME ZONE ${userTz}), - 'YYYY-MM-DD' - ) AS day_key - FROM measurements m - WHERE m."user_id" = ${userId} - AND m."measured_at" >= ${streakWindowStart} - AND m."deleted_at" IS NULL - `, + time("streakDays", () => + readActivityDayKeys(prisma, userId, userTz, streakWindowStart, now), ), // v1.11.4 — raw per-stage SLEEP_DURATION rows for the night-total // tile. `SLEEP_DURATION` is stored one row per STAGE per night @@ -635,13 +608,9 @@ async function buildDashboardSummary( } // v1.4.38 W-F — activity-day set assembled from the bounded streak - // queries. The measurement side already arrives as YYYY-MM-DD keys - // from the SQL `date_trunc + AT TIME ZONE` aggregate, byte-identical - // to what `userDayKey(measuredAt, userTz)` would produce in JS. - const activityDays = new Set(); - for (const row of measurementStreakDays) { - if (row.day_key) activityDays.add(row.day_key); - } + // queries. The measurement side arrives as local YYYY-MM-DD keys, the + // same keys `userDayKey(measuredAt, userTz)` produces for the intakes. + const activityDays = new Set(measurementStreakDays); for (const e of streakActivity) { activityDays.add(userDayKey(e.takenAt ?? e.scheduledFor, userTz)); } diff --git a/src/lib/dashboard/summary-reads.ts b/src/lib/dashboard/summary-reads.ts index a5206307e..2ca4df790 100644 --- a/src/lib/dashboard/summary-reads.ts +++ b/src/lib/dashboard/summary-reads.ts @@ -165,3 +165,46 @@ export function readSparkBuckets( ORDER BY type, bucket_start ASC `; } + +/** + * The local days in `[since, now]` that hold at least one live reading, as + * `YYYY-MM-DD` keys in `timeZone`: the measurement half of the streak. + * + * One existence probe per local day against `(user_id, measured_at)`, each + * stopping at its first row, instead of a `DISTINCT` over a year of readings + * (tens of thousands of rows on an account with a watch). The DAY rollups + * cannot answer this: their buckets are UTC days, and the streak counts the + * person's own days. + * + * `measured_at` is a naive UTC timestamp, so a local day's bounds are its + * local midnights read in `timeZone` and taken back to UTC. The read this + * replaced shifted the column with a single `AT TIME ZONE`, which treats the + * UTC value as local time and moves it the wrong way: in Berlin a reading at + * 23:30 UTC landed on the day before instead of the day after. + */ +export async function readActivityDayKeys( + db: PrismaClient, + userId: string, + timeZone: string, + since: Date, + now: Date, +): Promise { + const rows = await db.$queryRaw>` + SELECT to_char(d, 'YYYY-MM-DD') AS day_key + FROM generate_series( + (${since}::timestamptz AT TIME ZONE ${timeZone})::date, + (${now}::timestamptz AT TIME ZONE ${timeZone})::date, + INTERVAL '1 day' + ) AS d + WHERE EXISTS ( + SELECT 1 + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."measured_at" >= (d::timestamp AT TIME ZONE ${timeZone}) AT TIME ZONE 'UTC' + AND m."measured_at" < ((d + INTERVAL '1 day')::timestamp AT TIME ZONE ${timeZone}) AT TIME ZONE 'UTC' + ) + ORDER BY d + `; + return rows.map((row) => row.day_key); +} diff --git a/src/lib/rollups/__tests__/medication-compliance-rollups.test.ts b/src/lib/rollups/__tests__/medication-compliance-rollups.test.ts index 848cce605..13c7d970a 100644 --- a/src/lib/rollups/__tests__/medication-compliance-rollups.test.ts +++ b/src/lib/rollups/__tests__/medication-compliance-rollups.test.ts @@ -50,8 +50,6 @@ import { dayKeyForScheduledFor, recomputeMedicationComplianceForDay, recomputeMedicationComplianceForEvent, - readMedicationCompliance, - hasMedicationComplianceCoverage, recomputeUserMedicationCompliance, enqueueBootTimeMedicationComplianceBackfill, enqueueUserMedicationComplianceBackfill, @@ -317,107 +315,6 @@ describe("recomputeMedicationComplianceForEvent", () => { }); }); -describe("readMedicationCompliance", () => { - it("returns a trailing-window zero-filled bucket array", async () => { - rollupFindMany.mockResolvedValue([ - { day: "2026-05-18", scheduled: 2, taken: 1, skipped: 1 }, - ]); - - const now = new Date("2026-05-18T12:00:00.000Z"); - const buckets = await readMedicationCompliance( - "user-1", - 3, - "Europe/Berlin", - now, - ); - - expect(buckets).toHaveLength(3); - expect(buckets[buckets.length - 1]).toEqual({ - date: "2026-05-18", - scheduled: 2, - taken: 1, - }); - expect(buckets[0].scheduled).toBe(0); - expect(buckets[0].taken).toBe(0); - expect(buckets.map((b) => b.date)).toEqual([ - "2026-05-16", - "2026-05-17", - "2026-05-18", - ]); - }); - - it("folds per-medication rows into one per-day total", async () => { - rollupFindMany.mockResolvedValue([ - { day: "2026-05-18", scheduled: 1, taken: 1, skipped: 0 }, - { day: "2026-05-18", scheduled: 2, taken: 0, skipped: 1 }, - ]); - const now = new Date("2026-05-18T12:00:00.000Z"); - const buckets = await readMedicationCompliance( - "user-1", - 1, - "Europe/Berlin", - now, - ); - expect(buckets[0].scheduled).toBe(3); - expect(buckets[0].taken).toBe(1); - }); -}); - -describe("hasMedicationComplianceCoverage", () => { - it("returns true when rolled-day count meets the event-day count in window", async () => { - queryRaw.mockResolvedValue([ - { rolled_days: BigInt(7), event_days: BigInt(7) }, - ]); - await expect( - hasMedicationComplianceCoverage( - "user-1", - 7, - "Europe/Berlin", - new Date("2026-05-18T12:00:00.000Z"), - ), - ).resolves.toBe(true); - }); - - it("returns true when the user has zero events in window (trivially covered)", async () => { - // Zero events → the read path returns a trailing-window zero-fill - // from an empty rollup table; covered semantically. - queryRaw.mockResolvedValue([ - { rolled_days: BigInt(0), event_days: BigInt(0) }, - ]); - await expect( - hasMedicationComplianceCoverage("user-1", 7, "Europe/Berlin"), - ).resolves.toBe(true); - }); - - it("returns false on partial coverage (boot backfill mid-fold)", async () => { - // QA F-H-01 (v1.4.39): the legacy "any row exists" probe would - // flip true here and serve zero-filled buckets for days N..days-1 - // that the backfill hasn't reached yet. With the tightened probe - // partial coverage forces the route into the legacy aggregator - // until the fold completes. - queryRaw.mockResolvedValue([ - { rolled_days: BigInt(2), event_days: BigInt(7) }, - ]); - await expect( - hasMedicationComplianceCoverage( - "user-1", - 7, - "Europe/Berlin", - new Date("2026-05-18T12:00:00.000Z"), - ), - ).resolves.toBe(false); - }); - - it("returns false on zero rollups when events exist (legacy account cold start)", async () => { - queryRaw.mockResolvedValue([ - { rolled_days: BigInt(0), event_days: BigInt(7) }, - ]); - await expect( - hasMedicationComplianceCoverage("user-1", 7, "Europe/Berlin"), - ).resolves.toBe(false); - }); -}); - describe("recomputeUserMedicationCompliance", () => { it("upserts one rollup per (medication, day) pair returned by discovery", async () => { queryRaw.mockResolvedValue([ diff --git a/src/lib/rollups/medication-compliance-rollups.ts b/src/lib/rollups/medication-compliance-rollups.ts index 46f9e2e29..8e6163ac3 100644 --- a/src/lib/rollups/medication-compliance-rollups.ts +++ b/src/lib/rollups/medication-compliance-rollups.ts @@ -9,10 +9,11 @@ * Write hooks fire from every `MedicationIntakeEvent` mutation site * (create / update / delete / bulk import) plus the reminder-worker's * mint path that creates `takenAt:null` rows on RED-phase reminders. - * Reads collapse the trailing N-day window into a single bounded - * `findMany` against `medication_compliance_rollups`; the route-level - * coverage probe falls back to live + fires a boot backfill when a - * user has intake events but zero rollup rows. + * The dashboard tile it was built for reads the schedule-anchored + * compliance (`src/lib/analytics/schedule-anchored-compliance.ts`) since + * v1.26, and the per-day recompute below is now what keeps a day's + * supplement nutrients in step with its doses; the table itself has no + * reader left. * * v1.15.18 DESIGN DECISION — this tier stays a RAW per-day COVERAGE count, * NOT the band-attributed compliance %. It aggregates, across ALL of a @@ -77,14 +78,6 @@ export interface MedicationComplianceBackfillPayload { enqueuedAt: string; } -/** Reader-result row shape — matches `buildComplianceBuckets` output. */ -export interface ComplianceBucket { - /** YYYY-MM-DD in the user's tz. */ - date: string; - scheduled: number; - taken: number; -} - /** * Resolve a safe tz string — fall back to the server default when the * supplied value is missing or not an IANA zone the host recognises. @@ -254,138 +247,6 @@ export async function recomputeMedicationComplianceForEvent(input: { } } -/** - * Read trailing `days` of compliance buckets for the user, oldest day - * first. Returns the same `{date, scheduled, taken}` shape the legacy - * `buildComplianceBuckets` helper produced so the route swap is - * byte-identical. - * - * Days without a rollup row land as zero-filled buckets. The caller's - * cache wrapper carries the user-tz in its key so a tz change forces - * a fresh read. - */ -export async function readMedicationCompliance( - userId: string, - days: number, - tz: string | null | undefined, - now: Date = new Date(), -): Promise { - const safeTz = safeTimezone(tz); - - // Build the expected day-key window oldest → newest so the response - // is sorted and zero-filled regardless of which days hold rollups. - const todayKey = userDayKey(now, safeTz); - const expectedKeys: string[] = []; - for (let i = days - 1; i >= 0; i--) { - const probe = new Date(now.getTime() - i * 86_400_000); - expectedKeys.push(userDayKey(probe, safeTz)); - } - // Guard against a DST day-of-the-year landing the loop on the same - // dayKey twice — dedup by Map insertion order. - const dedup = new Set(); - const orderedKeys: string[] = []; - for (const key of expectedKeys) { - if (!dedup.has(key)) { - dedup.add(key); - orderedKeys.push(key); - } - } - // Always include today even if the trailing window dropped it (e.g. - // days===1 on a DST boundary). Cheap defensive insert. - if (!dedup.has(todayKey)) { - orderedKeys.push(todayKey); - } - - const oldestKey = orderedKeys[0]; - const newestKey = orderedKeys[orderedKeys.length - 1]; - - const rows = await prisma.medicationComplianceRollup.findMany({ - where: { - userId, - day: { gte: oldestKey, lte: newestKey }, - }, - select: { day: true, scheduled: true, taken: true, skipped: true }, - }); - - // Fold per-medication rows into one per-day total. The legacy route - // shape was aggregated across all the user's medications; preserving - // that surface keeps the front-end byte-stable. - const totals = new Map(); - for (const key of orderedKeys) { - totals.set(key, { scheduled: 0, taken: 0 }); - } - for (const row of rows) { - const bucket = totals.get(row.day); - if (!bucket) continue; - bucket.scheduled += row.scheduled; - bucket.taken += row.taken; - } - - return orderedKeys.map((date) => { - const bucket = totals.get(date) ?? { scheduled: 0, taken: 0 }; - return { date, scheduled: bucket.scheduled, taken: bucket.taken }; - }); -} - -/** - * Returns true when the rollup tier covers every day inside the - * trailing `days` window that has at least one intake event. Coverage - * is full when the count of DISTINCT rolled days equals the count of - * DISTINCT days that hold a `medication_intake_events` row in window — - * partial coverage (boot backfill mid-fold on a multi-medication - * account) returns false so the route falls through to the legacy - * aggregator instead of serving zero-filled tiles for un-rolled days. - * - * v1.4.39 hotfix (QA F-H-01): the previous "any row exists" probe - * could short-circuit to the rollup path while only the first few days - * had been backfilled, exposing zero-filled tiles for the un-rolled - * days until the boot fold completed. - */ -export async function hasMedicationComplianceCoverage( - userId: string, - days: number, - tz: string | null | undefined, - now: Date = new Date(), -): Promise { - const safeTz = safeTimezone(tz); - const oldestKey = userDayKey( - new Date(now.getTime() - (days - 1) * 86_400_000), - safeTz, - ); - const oldestStart = startOfLocalDayKey(oldestKey, safeTz); - - // Single SQL aggregate: count DISTINCT rolled days vs DISTINCT - // event-days in window. The event side is anchored on the same - // user-tz day-key (`to_char(... AT TIME ZONE $tz, 'YYYY-MM-DD')`) so - // the comparison stays apples-to-apples even on DST boundaries. - const result = await prisma.$queryRaw< - Array<{ rolled_days: bigint; event_days: bigint }> - >` - SELECT - ( - SELECT COUNT(DISTINCT "day")::bigint - FROM "medication_compliance_rollups" - WHERE "user_id" = ${userId} - AND "day" >= ${oldestKey} - ) AS rolled_days, - ( - SELECT COUNT(DISTINCT to_char("scheduled_for" AT TIME ZONE ${safeTz}, 'YYYY-MM-DD'))::bigint - FROM "medication_intake_events" - WHERE "user_id" = ${userId} - AND "deleted_at" IS NULL - AND "scheduled_for" >= ${oldestStart} - ) AS event_days - `; - if (result.length === 0) return true; - const { rolled_days, event_days } = result[0]; - const rolled = Number(rolled_days); - const events = Number(event_days); - // Zero events in window → trivially covered (the read path produces - // a zero-filled trailing window from the empty rollup table). - if (events === 0) return true; - return rolled >= events; -} - /** * Full-fold helper. Walks every `(medicationId, day)` pair the user * has logged across the trailing `days` window and upserts each @@ -395,7 +256,7 @@ export async function hasMedicationComplianceCoverage( * * Bounded SQL: one `DISTINCT (medicationId, dayKey)` $queryRaw scan * over the trailing window, then one upsert per row. The dayKey is - * computed in Postgres via `to_char(... AT TIME ZONE $tz)` so we + * computed in Postgres in the user's zone so we * don't have to ship millions of rows back to Node. */ export async function recomputeUserMedicationCompliance( @@ -414,14 +275,20 @@ export async function recomputeUserMedicationCompliance( const oldestStart = startOfLocalDayKey(oldestKey, safeTz); // Distinct (medication_id, dayKey) pairs the user has any intake - // event for inside the trailing window. `to_char(... AT TIME ZONE - // $tz)` keeps the day-bucket on the user's wall-clock. + // event for inside the trailing window. `scheduled_for` is a naive UTC + // timestamp: it is read as UTC first and only then shifted into the + // user's zone. A single `AT TIME ZONE $tz` reads it as local time and + // shifts it the wrong way, which keyed every slot within the zone's + // offset of midnight onto the neighbouring day. const pairs = await prisma.$queryRaw< Array<{ medication_id: string; day: string }> >` SELECT DISTINCT "medication_id", - to_char("scheduled_for" AT TIME ZONE ${safeTz}, 'YYYY-MM-DD') AS "day" + to_char( + ("scheduled_for" AT TIME ZONE 'UTC') AT TIME ZONE ${safeTz}, + 'YYYY-MM-DD' + ) AS "day" FROM "medication_intake_events" WHERE "user_id" = ${userId} AND "deleted_at" IS NULL diff --git a/tests/integration/local-day-keys.test.ts b/tests/integration/local-day-keys.test.ts new file mode 100644 index 000000000..04b891ea6 --- /dev/null +++ b/tests/integration/local-day-keys.test.ts @@ -0,0 +1,124 @@ +/** + * Local day keys from naive UTC timestamps, against a real Postgres. + * + * `measured_at` and `scheduled_for` hold UTC wall-clock time in `timestamp` + * columns. The dashboard streak and the medication-compliance backfill keyed + * them with a single `AT TIME ZONE tz`, which reads the value as local time + * and shifts it the wrong way. Both now agree with `userDayKey`: a reading at + * 23:30 UTC is the next day in Berlin and still the same day in Los Angeles, + * a reading at 03:00 UTC is the previous day in Los Angeles. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { readActivityDayKeys } from "@/lib/dashboard/summary-reads"; +import { recomputeUserMedicationCompliance } from "@/lib/rollups/medication-compliance-rollups"; +import { userDayKey } from "@/lib/tz/format"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const USER = "local-day-keys"; +const now = new Date(); +/** UTC midnight ten days back. */ +const D = Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() - 10, +); +const LATE = new Date(D + 23.5 * 3_600_000); // 23:30 UTC on D +const EARLY = new Date(D + 3 * 3_600_000); // 03:00 UTC on D +const key = (ms: number) => new Date(ms).toISOString().slice(0, 10); + +beforeEach(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { + id: USER, + username: USER, + email: `${USER}@example.test`, + timezone: "Europe/Berlin", + }, + }); +}); + +describe("readActivityDayKeys", () => { + beforeEach(async () => { + await prisma.measurement.createMany({ + data: [ + { at: LATE, deleted: false }, + { at: EARLY, deleted: false }, + { at: new Date(D - 3 * 86_400_000), deleted: true }, + ].map(({ at, deleted }, i) => ({ + id: `${USER}-m${i}`, + userId: USER, + type: "WEIGHT" as const, + source: "MANUAL" as const, + value: 80, + unit: "kg", + measuredAt: at, + deletedAt: deleted ? new Date() : null, + })), + }); + }); + + const since = new Date(D - 5 * 86_400_000); + + it("keys a late reading onto the next day east of UTC", async () => { + const keys = await readActivityDayKeys( + prisma, + USER, + "Europe/Berlin", + since, + now, + ); + expect(keys).toEqual([key(D), key(D + 86_400_000)]); + expect(keys).toEqual( + [EARLY, LATE].map((at) => userDayKey(at, "Europe/Berlin")), + ); + }); + + it("keys an early reading onto the previous day west of UTC", async () => { + const keys = await readActivityDayKeys( + prisma, + USER, + "America/Los_Angeles", + since, + now, + ); + expect(keys).toEqual([key(D - 86_400_000), key(D)]); + expect(keys).toEqual( + [EARLY, LATE].map((at) => userDayKey(at, "America/Los_Angeles")), + ); + }); +}); + +describe("recomputeUserMedicationCompliance", () => { + it("keys a late dose onto the person's own day", async () => { + const med = await prisma.medication.create({ + data: { userId: USER, name: "Ramipril", dose: "5mg", active: true }, + }); + await prisma.medicationIntakeEvent.create({ + data: { + userId: USER, + medicationId: med.id, + scheduledFor: LATE, + takenAt: LATE, + skipped: false, + source: "API", + }, + }); + + await recomputeUserMedicationCompliance(USER, 30, "Europe/Berlin"); + + const rows = await prisma.medicationComplianceRollup.findMany({ + where: { userId: USER }, + select: { day: true, scheduled: true, taken: true }, + }); + expect(rows).toEqual([ + { day: key(D + 86_400_000), scheduled: 1, taken: 1 }, + ]); + }); +}); From 04855b1a67bfef17726a44a701b493fdbf5a408c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 03:36:44 +0200 Subject: [PATCH 03/11] perf(day): read the day view's comparison history once while paging Every day view compares its day with the two years before it to find extremes, so stepping from one day to the next read the same two years of readings again. The history is now cached per record, zone, source ladder and readable type set, starting on a 90-day block boundary so neighbouring days share one cell, and each day takes its own slice cut at local day keys, which is exactly the window a live read would see. Measurement writes sweep the cell with the rest of the analytics bucket. A lookback floor inside the window and a window reaching past today still read live, and a type the modules switch off is never read. --- .../__tests__/notable-history-cache.test.ts | 121 ++++++++++++++++++ src/lib/day/notable.ts | 116 +++++++++++++++-- 2 files changed, 224 insertions(+), 13 deletions(-) create mode 100644 src/lib/day/__tests__/notable-history-cache.test.ts diff --git a/src/lib/day/__tests__/notable-history-cache.test.ts b/src/lib/day/__tests__/notable-history-cache.test.ts new file mode 100644 index 000000000..dd050aea9 --- /dev/null +++ b/src/lib/day/__tests__/notable-history-cache.test.ts @@ -0,0 +1,121 @@ +/** + * The day view's comparison history is read once per record, zone, ladder + * and readable type set, and each day takes its own slice of it. + * + * Paging from one day to the next used to read the same two years again. + * The slice has to give exactly what a live read of the window would: the + * extremes depend on where the series starts. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const readLocalDailyStats = vi.fn(); +vi.mock("@/lib/day/daily-stats", () => ({ + readLocalDailyStats: (...a: unknown[]) => readLocalDailyStats(...a), +})); +vi.mock("@/lib/db", () => ({ + prisma: { $queryRaw: vi.fn(async () => []) }, +})); + +import type { MeasurementType } from "@/generated/prisma/enums"; +import { __resetAllCachesForTests } from "@/lib/cache/server-cache"; +import { shiftDateKey, userDayKey } from "@/lib/tz/format"; + +import { + NOTABLE_LOOKBACK_DAYS, + extremesInSeries, + loadNotableRange, +} from "../notable"; + +const TZ = "Europe/Berlin"; +const TODAY = userDayKey(new Date(), TZ); +const SPIKE = shiftDateKey(TODAY, -20); +const FIRST = shiftDateKey(TODAY, -900); + +/** Systolic 120 every day from FIRST to TODAY, 150 on SPIKE. */ +function value(day: string): number { + return day === SPIKE ? 150 : 120; +} + +beforeEach(() => { + __resetAllCachesForTests(); + readLocalDailyStats.mockReset(); + readLocalDailyStats.mockImplementation( + async (args: { types: MeasurementType[]; from: Date; to: Date }) => { + const days = new Map(); + const end = userDayKey(args.to, TZ); + for ( + let day = userDayKey(args.from, TZ); + day < end; + day = shiftDateKey(day, 1) + ) { + if (day >= FIRST && day <= TODAY) days.set(day, value(day)); + } + return new Map( + args.types + .filter((t) => t === "BLOOD_PRESSURE_SYS") + .map((t) => [t, days]), + ); + }, + ); +}); + +function load(day: string, extra: { floor?: Date } = {}) { + return loadNotableRange({ + userId: "user-1", + from: day, + to: day, + tz: TZ, + priorityJson: null, + typeVisible: (t) => t !== "BLOOD_GLUCOSE", + gaps: false, + extremeTypes: ["BLOOD_PRESSURE_SYS"], + firstValueTypes: [], + ...extra, + }); +} + +/** What a live read of the window gives. */ +function expected(day: string) { + const series: Array<[string, number]> = []; + for ( + let d = shiftDateKey(day, -NOTABLE_LOOKBACK_DAYS); + d <= day; + d = shiftDateKey(d, 1) + ) { + if (d >= FIRST) series.push([d, value(d)]); + } + return extremesInSeries("BLOOD_PRESSURE_SYS", series, day); +} + +describe("notable history cache", () => { + it("reads the history once while paging through neighbouring days", async () => { + const spike = await load(SPIKE); + const before = await load(shiftDateKey(SPIKE, -1)); + const after = await load(shiftDateKey(SPIKE, 1)); + + expect(readLocalDailyStats).toHaveBeenCalledTimes(1); + expect(spike).toEqual(expected(SPIKE)); + expect(spike.map((n) => n.kind)).toEqual(["extremeHigh"]); + expect(before).toEqual(expected(shiftDateKey(SPIKE, -1))); + expect(after).toEqual(expected(shiftDateKey(SPIKE, 1))); + }); + + it("never reads a type the modules switch off", async () => { + await load(SPIKE); + const [args] = readLocalDailyStats.mock.calls[0] as [ + { types: MeasurementType[] }, + ]; + expect(args.types).not.toContain("BLOOD_GLUCOSE"); + expect(args.types).toContain("BLOOD_PRESSURE_SYS"); + }); + + it("reads live from a lookback floor inside the window", async () => { + const floor = new Date(Date.now() - 60 * 86_400_000); + await load(SPIKE, { floor }); + await load(SPIKE, { floor }); + + expect(readLocalDailyStats).toHaveBeenCalledTimes(2); + const [args] = readLocalDailyStats.mock.calls[0] as [{ from: Date }]; + expect(args.from).toEqual(floor); + }); +}); diff --git a/src/lib/day/notable.ts b/src/lib/day/notable.ts index 527c8d60d..4f275680a 100644 --- a/src/lib/day/notable.ts +++ b/src/lib/day/notable.ts @@ -25,10 +25,13 @@ * thousands of background samples, which says little as an extreme and * costs a scan of every one of them over two years. */ +import { createHash } from "node:crypto"; + import type { MeasurementType } from "@/generated/prisma/enums"; import type { DayNotable } from "@/lib/day/contract"; import { readLocalDailyStats, type DailyStats } from "@/lib/day/daily-stats"; +import { caches, type ServerCache } from "@/lib/cache/server-cache"; import { prisma } from "@/lib/db"; import { startOfLocalDayKey } from "@/lib/tz/local-day"; import { daysBetweenDateKeys, shiftDateKey, userDayKey } from "@/lib/tz/format"; @@ -184,6 +187,95 @@ export async function readFirstReadings( return out; } +/** Cached history starts on a block boundary, so neighbouring days share it. */ +const HISTORY_BLOCK_DAYS = 90; +const HISTORY_EPOCH = "1970-01-01"; +/** Measurement writes sweep the cell; the TTL only bounds an idle one. */ +const HISTORY_TTL_MS = 15 * 60_000; + +/** + * The daily series behind the extremes, for the local days `[from, to]`. + * + * Every day view compares its day with the two years before it, so paging + * from one day to the next read the same two years again. The history is + * cached per record, zone, source ladder and readable type set, from a block + * boundary at or before `from` up to today, and each caller takes its own + * slice: a day's value does not depend on the window around it, and a slice + * cut at local day keys is exactly the window the live read would see. A + * type the record's modules switch off is never read. A lookback floor + * inside the window, or a window that reaches past today, reads live. + */ +async function readExtremeHistory(args: { + userId: string; + readable: readonly MeasurementType[]; + from: string; + to: string; + tz: string; + priorityJson: unknown; + floor: Date | null; +}): Promise { + const fromInstant = startOfLocalDayKey(args.from, args.tz); + const today = userDayKey(new Date(), args.tz); + const live = (from: Date) => + readLocalDailyStats({ + userId: args.userId, + types: args.readable, + from, + to: startOfLocalDayKey(shiftDateKey(args.to, 1), args.tz), + tz: args.tz, + priorityJson: args.priorityJson, + }); + if (args.floor && args.floor > fromInstant) return live(args.floor); + if (args.to > today) return live(fromInstant); + + const offset = daysBetweenDateKeys(HISTORY_EPOCH, args.from); + const blockStart = shiftDateKey( + HISTORY_EPOCH, + Math.floor(offset / HISTORY_BLOCK_DAYS) * HISTORY_BLOCK_DAYS, + ); + const ladder = createHash("sha256") + .update(JSON.stringify(args.priorityJson ?? null)) + .digest("base64url") + .slice(0, 16); + const key = [ + args.userId, + "notable-history", + args.tz, + ladder, + [...args.readable].sort().join(","), + blockStart, + today, + ].join("|"); + const cache = caches.analytics as ServerCache; + const { value } = await cache.wrap( + key, + () => + readLocalDailyStats({ + userId: args.userId, + types: args.readable, + from: startOfLocalDayKey(blockStart, args.tz), + to: startOfLocalDayKey(shiftDateKey(today, 1), args.tz), + tz: args.tz, + priorityJson: args.priorityJson, + }), + HISTORY_TTL_MS, + ); + return sliceDays(value, args.from, args.to); +} + +/** The days of `stats` inside `[from, to]`, a copy. */ +function sliceDays(stats: DailyStats, from: string, to: string): DailyStats { + const out: DailyStats = new Map(); + for (const [type, days] of stats) { + const kept = new Map(); + for (const [day, value] of days) { + if (day >= from && day <= to) kept.set(day, value); + } + out.set(type, kept); + } + return out; +} + /** * The observations for a window of local days `[from, to]`. * @@ -212,20 +304,18 @@ export async function loadNotableRange(args: { (t) => args.typeVisible(t) && (wanted === null || wanted.has(t)), ); const firstTypes = (args.firstValueTypes ?? types).filter(args.typeVisible); - let historyFrom = startOfLocalDayKey( - shiftDateKey(args.from, -NOTABLE_LOOKBACK_DAYS), - args.tz, - ); - if (args.floor && args.floor > historyFrom) historyFrom = args.floor; const [stats, firsts] = await Promise.all([ - readLocalDailyStats({ - userId: args.userId, - types, - from: historyFrom, - to: startOfLocalDayKey(shiftDateKey(args.to, 1), args.tz), - tz: args.tz, - priorityJson: args.priorityJson, - }), + types.length === 0 + ? new Map() + : readExtremeHistory({ + userId: args.userId, + readable: NOTABLE_EXTREME_TYPES.filter(args.typeVisible), + from: shiftDateKey(args.from, -NOTABLE_LOOKBACK_DAYS), + to: args.to, + tz: args.tz, + priorityJson: args.priorityJson, + floor: args.floor ?? null, + }), readFirstReadings(args.userId, firstTypes), ]); From a22bf91311a811509c43f9357d079fc76dfd55b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 03:41:54 +0200 Subject: [PATCH 04/11] fix(insights): report every correlation through the significance gate The /insights envelope (weight x BP, mood x BP, mood x weight, mood x pulse, BP medication continuity) and the AI feature payload computed their coefficients with an ungated Pearson that banded |r| above five pairs, so five days of noise sloping the right way read as a strong correlation there while every other surface refused the same points. Both now use `significantPearsonCorrelation` (n >= 20 and p < 0.05, through the engine's exact p-value), and the ungated function is gone. The allowlist guard that tracked its remaining callers becomes a guard that there is one Pearson implementation. --- ...lation-single-implementation-guard.test.ts | 85 +++++++++++++ .../unguarded-correlation-guard.test.ts | 117 ------------------ .../comprehensive/__tests__/route.test.ts | 77 +++++++----- src/app/api/insights/comprehensive/route.ts | 14 +-- .../analytics/__tests__/correlations.test.ts | 66 +--------- src/lib/analytics/correlations.ts | 48 +------ src/lib/insights/features.ts | 4 +- src/lib/insights/mood-aggregates.ts | 6 +- .../insights-comprehensive-cache.test.ts | 2 +- 9 files changed, 150 insertions(+), 269 deletions(-) create mode 100644 src/__tests__/correlation-single-implementation-guard.test.ts delete mode 100644 src/__tests__/unguarded-correlation-guard.test.ts diff --git a/src/__tests__/correlation-single-implementation-guard.test.ts b/src/__tests__/correlation-single-implementation-guard.test.ts new file mode 100644 index 000000000..2a4d1e09c --- /dev/null +++ b/src/__tests__/correlation-single-implementation-guard.test.ts @@ -0,0 +1,85 @@ +/** + * Structural guard: one Pearson correlation, one bar. + * + * Three implementations used to answer "how correlated are these two + * series": the engine's `pearson` (exact p-value, `lib/insights/ + * correlations.ts`), its gated wrapper `significantPearsonCorrelation` + * (n >= 20 and p < 0.05), and an ungated `pearsonCorrelation` that banded a + * coefficient from |r| alone above five pairs. The ungated one fed the + * /insights envelope and the AI feature payload, so five days of noise + * sloping the right way reached a reader labelled "strong" there while every + * other surface refused the same five points. It is gone; every surface goes + * through the wrapper, and the wrapper through the engine. + * + * The matcher flags a function whose name says Pearson or correlation + * coefficient, declared anywhere but the engine and the wrapper's file, and + * any mention of the deleted name. It is a tripwire: a coefficient computed + * inline under another name slips it. Comments are stripped first; the + * planted cases prove it fires. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +const HOMES = new Set([ + "lib/insights/correlations.ts", + "lib/analytics/correlations.ts", +]); + +const PEARSON_DEFINITION = + /(?:function\s+|const\s+)\w*(?:pearson|correlationCoefficient)\w*\s*(?:=|\()/i; +const DELETED = /(? + !p.startsWith("generated/") && + !p.includes("__tests__/") && + !/\.test\.tsx?$/.test(p), + ); +} + +describe("one Pearson correlation", () => { + it("the matchers fire on a second implementation", () => { + expect(PEARSON_DEFINITION.test("export function pearsonR(xs, ys) {")).toBe( + true, + ); + expect(PEARSON_DEFINITION.test("const quickPearson = (a, b) =>")).toBe( + true, + ); + expect(DELETED.test("pearsonCorrelation(pairs)")).toBe(true); + expect(DELETED.test("significantPearsonCorrelation(pairs)")).toBe(false); + }); + + it("both homes still hold their function", () => { + const engine = readFileSync( + join(SRC, "lib/insights/correlations.ts"), + "utf8", + ); + const wrapper = readFileSync( + join(SRC, "lib/analytics/correlations.ts"), + "utf8", + ); + expect(engine).toMatch(/export function pearson\s*\(/); + expect(wrapper).toMatch( + /export function significantPearsonCorrelation\s*\(/, + ); + }); + + it("no other file defines a Pearson coefficient or calls the deleted one", () => { + const offenders = sourceFiles().filter((file) => { + const code = stripComments(readFileSync(join(SRC, file), "utf8")); + if (DELETED.test(code)) return true; + return !HOMES.has(file) && PEARSON_DEFINITION.test(code); + }); + expect(offenders).toEqual([]); + }); +}); diff --git a/src/__tests__/unguarded-correlation-guard.test.ts b/src/__tests__/unguarded-correlation-guard.test.ts deleted file mode 100644 index 96f0993d4..000000000 --- a/src/__tests__/unguarded-correlation-guard.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Structural guard on the unguarded Pearson helper. - * - * `pearsonCorrelation` bands a coefficient from |r| alone above a floor of - * five pairs, with no significance test. Its guarded twin - * `significantPearsonCorrelation` sits directly beneath it in the same file - * and exists because of exactly one failure: five days of noise sloping the - * right way reaching a reader labelled "strong". The comment above the twin - * says so in as many words. - * - * The twin was written once and adopted one surface at a time, which is how - * the mood page kept the naive call for four minor lines after the surfaces - * either side of it had moved. Nothing failed when it was skipped, because - * nothing was watching. This is what watches: a new caller of the unguarded - * helper has to be added to the list below, in a diff a reviewer reads, next - * to a written reason. - * - * It is a tripwire, not a proof. It cannot say the remaining entries are - * right — only that the set has not grown without someone editing this file. - * The two entries it carries today are named as open work, not as blessings. - */ -import { describe, it, expect } from "vitest"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - -import { walkSourceFiles } from "./helpers/source-files"; - -const SRC = join(process.cwd(), "src"); - -/** - * Files that may still call `pearsonCorrelation` directly, and why. - * - * Neither entry is settled. Both are the same migration the user-facing - * surfaces already made, still owed by two payload builders whose consumers - * pin the n >= 5 shape; moving them changes what those payloads contain and - * belongs in its own change with its own tests, not smuggled into this one. - */ -const ALLOWED: Record = { - "app/api/insights/comprehensive/route.ts": - "Five correlations on the /insights envelope (weight x BP, mood x BP, " + - "mood x weight, mood x pulse, BP-medication continuity). Open: they " + - "carry the same small-n exposure the mood page had, and the page's " + - "cards read `n` straight off the envelope.", - "lib/insights/features.ts": - "Eight cross-metric coefficients on the AI feature payload. Open: the " + - "prompt reads them as findings, so the same significance bar applies; " + - "the snapshot shape and its budget tests move with it.", -}; - -/** Every non-test, non-generated source file under `src/`. */ -function sourceFiles(): string[] { - return walkSourceFiles(SRC, { floor: 3000 }) - .filter((p) => !p.startsWith("generated/")) - .filter((p) => !p.includes("__tests__")) - .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx")) - .sort(); -} - -function read(rel: string): string { - return readFileSync(join(SRC, rel), "utf8"); -} - -/** - * A call to the unguarded helper. Whitespace-tolerant and anchored on a word - * boundary so `significantPearsonCorrelation(` cannot match: a matcher that - * quietly matches nothing is the failure mode this repository has shipped - * before, so the count is asserted below as well. - */ -const CALL = /(? { - it("finds callers at all, so an empty result cannot read as a pass", () => { - const definition = read("lib/analytics/correlations.ts"); - expect(definition).toMatch(/export function pearsonCorrelation\s*\(/); - expect( - definition, - "the guarded twin must exist for this guard to mean anything", - ).toMatch(/export function significantPearsonCorrelation\s*\(/); - - const callers = sourceFiles().filter( - (rel) => rel !== "lib/analytics/correlations.ts" && CALL.test(read(rel)), - ); - expect(callers.length).toBeGreaterThan(0); - }); - - it("names every caller outside the allowlist", () => { - const callers = sourceFiles().filter( - (rel) => rel !== "lib/analytics/correlations.ts" && CALL.test(read(rel)), - ); - - expect( - callers.filter((rel) => !(rel in ALLOWED)), - "A new caller of the unguarded `pearsonCorrelation`. A surface a " + - "person reads must use `significantPearsonCorrelation` (n >= 20 " + - "AND p < 0.05). If this one genuinely cannot, add it to ALLOWED " + - "in this file with the reason, so the exception is reviewed rather " + - "than assumed.", - ).toEqual([]); - }); - - it("keeps the allowlist honest by refusing stale entries", () => { - const callers = new Set( - sourceFiles().filter( - (rel) => - rel !== "lib/analytics/correlations.ts" && CALL.test(read(rel)), - ), - ); - - // An entry whose file no longer calls the helper is a licence nobody - // needs any more; leaving it standing is how the next one gets waved in. - expect( - Object.keys(ALLOWED).filter((rel) => !callers.has(rel)), - "this file is on the allowlist but no longer calls the unguarded " + - "helper — drop the entry", - ).toEqual([]); - }); -}); diff --git a/src/app/api/insights/comprehensive/__tests__/route.test.ts b/src/app/api/insights/comprehensive/__tests__/route.test.ts index 2e9758306..2f1a185ab 100644 --- a/src/app/api/insights/comprehensive/__tests__/route.test.ts +++ b/src/app/api/insights/comprehensive/__tests__/route.test.ts @@ -569,16 +569,10 @@ describe("GET /api/insights/comprehensive — envelope shape", () => { expect(body.data.moodSummary?.count).toBe(2); }); - it("derives weight × BP scatter from daily-bucketed series", async () => { - vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); - // Five day-aligned pairs (n >= 5 unlocks pearsonCorrelation). - const days = [ - "2026-05-01", - "2026-05-02", - "2026-05-03", - "2026-05-04", - "2026-05-05", - ]; + function mockWeightBpDays(count: number) { + const days = Array.from({ length: count }, (_, i) => + new Date(Date.UTC(2026, 4, 1 + i)).toISOString().slice(0, 10), + ); (buildComprehensiveAggregate as ReturnType).mockResolvedValue( { summaries: {}, @@ -592,22 +586,41 @@ describe("GET /api/insights/comprehensive — envelope shape", () => { })), }, firstMeasurementAt: new Date(), - totalMeasurements: 10, + totalMeasurements: count * 2, }, ); + } + + type WeightBpBody = { + data: { + scatterData: Array<{ weight: number; sysBP: number }>; + weightBpCorrelation: { r: number; n: number } | null; + }; + }; + + it("derives weight × BP scatter from daily-bucketed series", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + mockWeightBpDays(25); const res = await callGet(makeReq()); - const body = (await res.json()) as { - data: { - scatterData: Array<{ weight: number; sysBP: number }>; - weightBpCorrelation: { r: number; n: number } | null; - }; - }; - expect(body.data.scatterData).toHaveLength(5); + const body = (await res.json()) as WeightBpBody; + expect(body.data.scatterData).toHaveLength(25); expect(body.data.scatterData[0]).toEqual({ weight: 80, sysBP: 120 }); - // Perfect linear relationship → r === 1. + // Perfect linear relationship over 25 days → r === 1. expect(body.data.weightBpCorrelation?.r).toBe(1); - expect(body.data.weightBpCorrelation?.n).toBe(5); + expect(body.data.weightBpCorrelation?.n).toBe(25); + }); + + it("reports no correlation below the significance bar every surface uses", async () => { + // Five perfectly aligned days: the scatter shows them, but five points + // cannot carry a correlation (n >= 20 and p < 0.05). + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + mockWeightBpDays(5); + + const res = await callGet(makeReq()); + const body = (await res.json()) as WeightBpBody; + expect(body.data.scatterData).toHaveLength(5); + expect(body.data.weightBpCorrelation).toBeNull(); }); }); @@ -849,13 +862,10 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing ( // tier in R12. Both read the UTC `dailyByType` / UTC `scheduledFor` // slice, so their output must be invariant to the caller's timezone. // Run the SAME fixture under UTC then America/New_York and deep-equal. - const days = [ - "2026-05-01", - "2026-05-02", - "2026-05-03", - "2026-05-04", - "2026-05-05", - ]; + // 25 days: a correlation has to clear n >= 20 and p < 0.05 to exist. + const days = Array.from({ length: 25 }, (_, i) => + new Date(Date.UTC(2026, 4, 1 + i)).toISOString().slice(0, 10), + ); const aggregate = { summaries: {}, bpRawRows: { sys: [], dia: [] }, @@ -896,20 +906,20 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing ( scheduleRevisions: [], pauseEras: [], }; - // Intake taken on the first three days, logged as skipped on the last - // two → a non-degenerate continuity series. The skips used to be absent + // Intake taken on the first fifteen days, logged as skipped on the + // last ten → a non-degenerate continuity series. The skips used to be absent // rows: a day with no record read as a measured 0 % continuity, which is // the fabrication the route no longer makes, and a fixture that relies on // it would be pinning the defect rather than the timezone question this - // test asks. Recording the two days as skips keeps five paired days AND + // test asks. Recording the days as skips keeps every day paired AND // keeps every one of them a thing the person actually logged. const intakeEvents = days.map((d, i) => ({ medicationId: "med-bp", userId: "user-comp-1", deletedAt: null, scheduledFor: new Date(`${d}T07:00:00Z`), - takenAt: i < 3 ? new Date(`${d}T07:05:00Z`) : null, - skipped: i >= 3, + takenAt: i < 15 ? new Date(`${d}T07:05:00Z`) : null, + skipped: i >= 15, })); function primeMocks() { @@ -948,7 +958,8 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing ( ); // And the guard is meaningful only if these joins actually produced // output to compare. - expect(utcBody.data.scatterData).toHaveLength(5); + expect(utcBody.data.scatterData).toHaveLength(25); + expect(utcBody.data.weightBpCorrelation).not.toBeNull(); expect(utcBody.data.bpMedicationCorrelation).not.toBeNull(); }); }); diff --git a/src/app/api/insights/comprehensive/route.ts b/src/app/api/insights/comprehensive/route.ts index f15b92687..6d3adac3a 100644 --- a/src/app/api/insights/comprehensive/route.ts +++ b/src/app/api/insights/comprehensive/route.ts @@ -12,7 +12,7 @@ import { generateAlerts, } from "@/lib/analytics/classifications"; import { - pearsonCorrelation, + significantPearsonCorrelation, type PairedPoint, } from "@/lib/analytics/correlations"; import { @@ -285,7 +285,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) { const weightDaily = aggregate.dailyByType.WEIGHT ?? []; const sysDaily = aggregate.dailyByType.BLOOD_PRESSURE_SYS ?? []; const weightBpPairs: PairedPoint[] = joinDailyByDay(weightDaily, sysDaily); - const weightBpCorrelation = pearsonCorrelation(weightBpPairs); + const weightBpCorrelation = significantPearsonCorrelation(weightBpPairs); const scatterData = weightBpPairs.map((p) => ({ weight: p.a, sysBP: p.b, @@ -308,11 +308,11 @@ export async function buildComprehensiveResponse(user: AuthedUser) { ); const moodBpPairs = pairMoodWithDaily(dailyMoodEntries, sysDailyLocal); - const moodBpCorrelation = pearsonCorrelation(moodBpPairs); + const moodBpCorrelation = significantPearsonCorrelation(moodBpPairs); const moodBpScatterData = moodBpPairs.map((p) => ({ mood: p.a, sysBP: p.b })); const moodWeightPairs = pairMoodWithDaily(dailyMoodEntries, weightDailyLocal); - const moodWeightCorrelation = pearsonCorrelation(moodWeightPairs); + const moodWeightCorrelation = significantPearsonCorrelation(moodWeightPairs); const moodWeightScatterData = moodWeightPairs.map((p) => ({ mood: p.a, weight: p.b, @@ -374,7 +374,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) { .sort(([a], [b]) => a.localeCompare(b)) .map(([day, stats]) => ({ day, value: stats.sum / stats.count })); const moodPulsePairs = pairMoodWithDaily(dailyMoodEntries, restingPulseDaily); - const moodPulseCorrelation = pearsonCorrelation(moodPulsePairs); + const moodPulseCorrelation = significantPearsonCorrelation(moodPulsePairs); const moodPulseScatterData = moodPulsePairs.map((p) => ({ mood: p.a, pulse: p.b, @@ -558,7 +558,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) { }); } - const corr = pearsonCorrelation(pairs); + const corr = significantPearsonCorrelation(pairs); if (corr) { bpMedicationCorrelation = { ...corr, @@ -667,7 +667,7 @@ function deriveDailyLocalMeans( /** * Daily-key inner join between two daily-aggregated series. Returns - * `PairedPoint` shapes compatible with `pearsonCorrelation`. + * `PairedPoint` shapes compatible with `significantPearsonCorrelation`. * * Inputs are sorted ASC by `day` (the aggregator's ORDER BY * guarantees that). The walk is O(n + m) merge-join style instead of diff --git a/src/lib/analytics/__tests__/correlations.test.ts b/src/lib/analytics/__tests__/correlations.test.ts index 1e91db9f4..26be92f55 100644 --- a/src/lib/analytics/__tests__/correlations.test.ts +++ b/src/lib/analytics/__tests__/correlations.test.ts @@ -2,7 +2,6 @@ import { describe, it, expect } from "vitest"; import type { DataPoint } from "../trends"; import { pairByTimestamp, - pearsonCorrelation, significantPearsonCorrelation, } from "../correlations"; import type { PairedPoint } from "../correlations"; @@ -52,62 +51,6 @@ describe("pairByTimestamp", () => { }); }); -describe("pearsonCorrelation", () => { - it("returns null for fewer than minPairs", () => { - const pairs = [ - { a: 1, b: 2, date: new Date() }, - { a: 2, b: 4, date: new Date() }, - ]; - expect(pearsonCorrelation(pairs)).toBeNull(); - }); - - it("detects perfect positive correlation", () => { - const pairs = Array.from({ length: 10 }, (_, i) => ({ - a: i, - b: i * 2, - date: new Date(), - })); - const result = pearsonCorrelation(pairs)!; - expect(result.r).toBe(1); - expect(result.strength).toBe("stark"); - expect(result.n).toBe(10); - }); - - it("detects perfect negative correlation", () => { - const pairs = Array.from({ length: 10 }, (_, i) => ({ - a: i, - b: 100 - i * 2, - date: new Date(), - })); - const result = pearsonCorrelation(pairs)!; - expect(result.r).toBe(-1); - expect(result.strength).toBe("stark"); - }); - - it("detects no correlation for random-like data", () => { - const pairs = [ - { a: 1, b: 5, date: new Date() }, - { a: 2, b: 3, date: new Date() }, - { a: 3, b: 7, date: new Date() }, - { a: 4, b: 2, date: new Date() }, - { a: 5, b: 6, date: new Date() }, - ]; - const result = pearsonCorrelation(pairs)!; - expect(Math.abs(result.r)).toBeLessThan(0.4); - }); - - it("returns keine for constant values", () => { - const pairs = Array.from({ length: 5 }, () => ({ - a: 5, - b: 5, - date: new Date(), - })); - const result = pearsonCorrelation(pairs)!; - expect(result.r).toBe(0); - expect(result.strength).toBe("keine"); - }); -}); - describe("significantPearsonCorrelation (M-CS3 — n>=20 AND p<0.05)", () => { const mk = (a: number, b: number): PairedPoint => ({ a, @@ -116,14 +59,9 @@ describe("significantPearsonCorrelation (M-CS3 — n>=20 AND p<0.05)", () => { }); it("suppresses a 5-pair r≈0.7 dataset (below the n>=20 floor)", () => { - // Five points with a strong-looking slope — exactly the small-n fluke the - // plain `pearsonCorrelation` would surface as a "stark" correlation. + // Five points with a strong-looking slope: the small-n fluke an + // ungated band would call a "stark" correlation. const pairs = [mk(1, 1), mk(2, 2), mk(3, 2), mk(4, 4), mk(5, 3)]; - // Sanity-check the legacy path WOULD have surfaced it (non-null, n=5). - const legacy = pearsonCorrelation(pairs); - expect(legacy).not.toBeNull(); - expect(legacy!.n).toBe(5); - // The gated path refuses it — too few pairs. expect(significantPearsonCorrelation(pairs)).toBeNull(); }); diff --git a/src/lib/analytics/correlations.ts b/src/lib/analytics/correlations.ts index 10270f1f5..6065af944 100644 --- a/src/lib/analytics/correlations.ts +++ b/src/lib/analytics/correlations.ts @@ -72,54 +72,18 @@ export function pairByTimestamp( } /** - * Pearson correlation coefficient. - * Returns null if fewer than minPairs data points. - */ -export function pearsonCorrelation( - pairs: PairedPoint[], - minPairs: number = 5, -): CorrelationResult | null { - if (pairs.length < minPairs) return null; - - const n = pairs.length; - const sumA = pairs.reduce((s, p) => s + p.a, 0); - const sumB = pairs.reduce((s, p) => s + p.b, 0); - const sumAB = pairs.reduce((s, p) => s + p.a * p.b, 0); - const sumAA = pairs.reduce((s, p) => s + p.a * p.a, 0); - const sumBB = pairs.reduce((s, p) => s + p.b * p.b, 0); - - const denominator = Math.sqrt( - (n * sumAA - sumA * sumA) * (n * sumBB - sumB * sumB), - ); - - if (denominator === 0) return { r: 0, strength: "keine", n }; - - const r = Math.round(((n * sumAB - sumA * sumB) / denominator) * 1000) / 1000; - const absR = Math.abs(r); - - let strength: CorrelationResult["strength"]; - if (absR >= 0.7) strength = "stark"; - else if (absR >= 0.4) strength = "moderat"; - else if (absR >= 0.2) strength = "schwach"; - else strength = "keine"; - - return { r, strength, n }; -} - -/** - * v1.2.5 (M-CS3) — significance-gated Pearson for USER-FACING surfaces. + * v1.2.5 (M-CS3) — significance-gated Pearson, the one correlation every + * surface reports. * - * The plain `pearsonCorrelation` above assigns a strength band from |r| - * alone with no significance test and a default floor of only 5 pairs — - * so a 5-point r ≈ 0.7 fluke surfaces as a "stark" correlation. Every - * surfaced insight-card / status-snapshot correlation must instead clear - * the same rigorous bar the `/insights` correlation engine enforces: + * A strength band read from |r| alone, above a floor of five pairs, let a + * 5-point r ≈ 0.7 fluke surface as a "stark" correlation. Every reported + * correlation clears the bar the `/insights` correlation engine enforces: * * - n >= 20 paired points (the engine's `MIN_PAIRED_N`), AND * - a two-sided Student-t p-value < 0.05 (`MAX_P_VALUE`). * * Below either bar this returns `null` so the caller suppresses the - * correlation exactly as it already does for the < minPairs case — no + * correlation — no * call-site shape change. The returned `{ r, strength, n }` keeps the * legacy shape (German strength band) so the snapshot byte-shape is * unchanged on the happy path. diff --git a/src/lib/insights/features.ts b/src/lib/insights/features.ts index 091714530..cc24519de 100644 --- a/src/lib/insights/features.ts +++ b/src/lib/insights/features.ts @@ -32,7 +32,7 @@ import { getBpTargets } from "@/lib/analytics/bp-targets"; import { isBpReadingInTarget } from "@/lib/analytics/bp-in-target"; import { pairByTimestamp, - pearsonCorrelation, + significantPearsonCorrelation, type CorrelationResult, } from "@/lib/analytics/correlations"; import { getMedicationCategories } from "@/lib/medication-category"; @@ -1656,7 +1656,7 @@ export async function extractFeatures( })); const computeCorr = (a: DataPoint[], b: DataPoint[]) => - pearsonCorrelation(pairByTimestamp(a, b)); + significantPearsonCorrelation(pairByTimestamp(a, b)); const sleepPoints = toDataPoints(sleepData); diff --git a/src/lib/insights/mood-aggregates.ts b/src/lib/insights/mood-aggregates.ts index d8f4e1fa7..333e7a60d 100644 --- a/src/lib/insights/mood-aggregates.ts +++ b/src/lib/insights/mood-aggregates.ts @@ -428,9 +428,9 @@ export interface MoodMetricCorrelation { * * This routes through `significantPearsonCorrelation` — n >= 20 AND * p < 0.05 — for the same reason the weight, blood-pressure and - * mood-status surfaces do: the plain `pearsonCorrelation` bands from |r| - * alone above a floor of five pairs, so five days of noise sloping the - * right way arrive at the reader labelled "strong". A mood page is where + * mood-status surfaces do: a band read from |r| alone above a floor of + * five pairs lets five days of noise sloping the right way arrive at the + * reader labelled "strong". A mood page is where * that costs the most, because the reader is looking for something to * change about their week. */ diff --git a/tests/integration/insights-comprehensive-cache.test.ts b/tests/integration/insights-comprehensive-cache.test.ts index acd0cced9..fec430c3d 100644 --- a/tests/integration/insights-comprehensive-cache.test.ts +++ b/tests/integration/insights-comprehensive-cache.test.ts @@ -159,7 +159,7 @@ describe("GET /api/insights/comprehensive — server cache", () => { // A tight fixture: 5 paired BP readings + 5 weight rows + 5 mood // entries spread across 5 distinct days. Enough to populate every - // correlation block (n >= 5 is the pearsonCorrelation gate). + // correlation key (each is null below the significance bar). const now = Date.now(); const DAY_MS = 24 * 60 * 60 * 1000; const dayDates = Array.from( From 7535235b8232a265850c6d2f5da2dec1ac301f78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 04:36:25 +0200 Subject: [PATCH 05/11] test(guards): give the new guards a floor on what they find The zone-shift and single-Pearson guards now assert that their matchers find the correct shapes before concluding nothing else matches, and the dense-read guard lists the streak's per-day existence probe with the reason it stays bounded. --- ...lation-single-implementation-guard.test.ts | 8 + .../dense-measurement-read-guard.test.ts | 2 + .../naive-timestamp-zone-shift-guard.test.ts | 26 +- .../__tests__/pick-canonical-workout.test.ts | 478 ------------------ src/lib/sources/pick-canonical-workout.ts | 437 ---------------- .../workouts/__tests__/canonical-rows.test.ts | 396 --------------- src/lib/workouts/canonical-rows.ts | 308 ----------- 7 files changed, 25 insertions(+), 1630 deletions(-) delete mode 100644 src/lib/sources/__tests__/pick-canonical-workout.test.ts delete mode 100644 src/lib/sources/pick-canonical-workout.ts delete mode 100644 src/lib/workouts/__tests__/canonical-rows.test.ts delete mode 100644 src/lib/workouts/canonical-rows.ts diff --git a/src/__tests__/correlation-single-implementation-guard.test.ts b/src/__tests__/correlation-single-implementation-guard.test.ts index 2a4d1e09c..f9697ab84 100644 --- a/src/__tests__/correlation-single-implementation-guard.test.ts +++ b/src/__tests__/correlation-single-implementation-guard.test.ts @@ -75,6 +75,14 @@ describe("one Pearson correlation", () => { }); it("no other file defines a Pearson coefficient or calls the deleted one", () => { + // The homes match the definition matcher, so a matcher gone blind fails here. + const definers = sourceFiles().filter((file) => + PEARSON_DEFINITION.test( + stripComments(readFileSync(join(SRC, file), "utf8")), + ), + ); + expect(definers.length).toBeGreaterThanOrEqual(2); + const offenders = sourceFiles().filter((file) => { const code = stripComments(readFileSync(join(SRC, file), "utf8")); if (DELETED.test(code)) return true; diff --git a/src/__tests__/dense-measurement-read-guard.test.ts b/src/__tests__/dense-measurement-read-guard.test.ts index eb46135e0..75c083e73 100644 --- a/src/__tests__/dense-measurement-read-guard.test.ts +++ b/src/__tests__/dense-measurement-read-guard.test.ts @@ -133,6 +133,8 @@ const ALLOWED_SQL: Record = { "Selects account ids; `measurements` is read inside `EXISTS`, one row per account at most.", "lib/rollups/measurement-coverage.ts::1": "Selects from `unnest` of the caller's type list; `measurements` is read inside `EXISTS`, one row per type at most.", + "lib/dashboard/summary-reads.ts::1": + "Selects from `generate_series` of the streak window's local days; `measurements` is read inside `EXISTS`, one row per day at most.", }; interface UntakenRead { diff --git a/src/__tests__/naive-timestamp-zone-shift-guard.test.ts b/src/__tests__/naive-timestamp-zone-shift-guard.test.ts index d0e55846c..cfb111b5a 100644 --- a/src/__tests__/naive-timestamp-zone-shift-guard.test.ts +++ b/src/__tests__/naive-timestamp-zone-shift-guard.test.ts @@ -28,6 +28,9 @@ const SRC = join(process.cwd(), "src"); const SINGLE_SHIFT = /"(?:[a-z_]+_at|scheduled_for)"\s*\)?\s*AT\s+TIME\s+ZONE\s+(?:\$\{|\$\d)/i; +const DOUBLE_SHIFT = + /AT\s+TIME\s+ZONE\s+'UTC'\s*\)?\s*AT\s+TIME\s+ZONE\s+(?:\$\{|\$\d)/i; + function stripComments(text: string): string { return text .replace(/\/\*[\s\S]*?\*\//g, "") @@ -52,16 +55,17 @@ describe("naive timestamp zone shifts", () => { }); it("no production read shifts a naive column once", () => { - const offenders = walkSourceFiles(SRC, { floor: 3000 }) - .filter( - (file) => - !file.startsWith("generated/") && - !file.includes("__tests__/") && - !/\.test\.tsx?$/.test(file), - ) - .filter((file) => - SINGLE_SHIFT.test(stripComments(readFileSync(join(SRC, file), "utf8"))), - ); - expect(offenders).toEqual([]); + const files = walkSourceFiles(SRC, { floor: 3000 }).filter( + (file) => + !file.startsWith("generated/") && + !file.includes("__tests__/") && + !/\.test\.tsx?$/.test(file), + ); + const code = (file: string) => + stripComments(readFileSync(join(SRC, file), "utf8")); + // The reads that shift correctly, so a sweep that reads nothing fails. + const doubleShifts = files.filter((file) => DOUBLE_SHIFT.test(code(file))); + expect(doubleShifts.length).toBeGreaterThanOrEqual(5); + expect(files.filter((file) => SINGLE_SHIFT.test(code(file)))).toEqual([]); }); }); diff --git a/src/lib/sources/__tests__/pick-canonical-workout.test.ts b/src/lib/sources/__tests__/pick-canonical-workout.test.ts deleted file mode 100644 index ecb9afbda..000000000 --- a/src/lib/sources/__tests__/pick-canonical-workout.test.ts +++ /dev/null @@ -1,478 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - DEFAULT_WORKOUT_PROXIMITY_MINUTES, - DEFAULT_WORKOUT_SOURCE_PRIORITY, - pickCanonicalWorkout, - type WorkoutPickerRow, -} from "../pick-canonical-workout"; - -/** - * v1.4.25 W16a — `pickCanonicalWorkout` unit tests. - * - * Three primary fixtures match the W16a brief: - * (a) two sources record the SAME workout (within proximity, same - * sport) — picker keeps the canonical one. - * (b) two sources record DIFFERENT workouts close in time (within - * proximity but different sport types) — picker keeps both. - * (c) single source — pass-through. - * - * Edge fixtures verify the proximity-window boundary, the determinism - * contract, and the empty-input fast path. - */ - -const D = (iso: string): Date => new Date(iso); - -const row = ( - id: string, - source: WorkoutPickerRow["source"], - startedAt: string, - sportType: WorkoutPickerRow["sportType"] = "running", - extra: Partial = {}, -): WorkoutPickerRow => ({ - id, - source, - startedAt: D(startedAt), - sportType, - ...extra, -}); - -describe("DEFAULT_WORKOUT_SOURCE_PRIORITY", () => { - it("places APPLE_HEALTH ahead of WITHINGS for cumulative workouts", () => { - const apple = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("APPLE_HEALTH"); - const withings = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("WITHINGS"); - expect(apple).toBeGreaterThanOrEqual(0); - expect(withings).toBeGreaterThan(apple); - }); - - it("places MANUAL ahead of IMPORT so user-typed entries beat bulk loads", () => { - const manual = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("MANUAL"); - const importIdx = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("IMPORT"); - expect(importIdx).toBeGreaterThan(manual); - }); -}); - -describe("pickCanonicalWorkout — fixture (a) two-source same workout", () => { - it("keeps the APPLE_HEALTH row when both sources record the same run", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - // Withings ingested the same workout 60 seconds later (typical - // API-pull lag). Same sport, well within the 5-minute window. - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - ]); - expect(result.canonical).toHaveLength(1); - expect(result.canonical[0].id).toBe("apple-1"); - expect(result.canonical[0].source).toBe("APPLE_HEALTH"); - expect(result.clusters).toHaveLength(1); - expect(result.clusters[0].members).toHaveLength(2); - expect(result.clusters[0].pickedSource).toBe("APPLE_HEALTH"); - }); - - it("keeps every workout from the winning source in one cluster", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - row("apple-2", "APPLE_HEALTH", "2026-05-14T07:32:00.000Z"), - ]); - - expect(result.canonical.map((item) => item.id)).toEqual([ - "apple-1", - "apple-2", - ]); - }); - - it("still picks APPLE_HEALTH when the Withings row arrives first", () => { - const result = pickCanonicalWorkout([ - row("withings-1", "WITHINGS", "2026-05-14T07:30:00.000Z"), - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:31:30.000Z"), - ]); - expect(result.canonical.map((r) => r.id)).toEqual(["apple-1"]); - }); - - it("honours a per-user override that flips the ladder", () => { - const result = pickCanonicalWorkout( - [ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - ], - { sourcePriority: ["WITHINGS", "APPLE_HEALTH", "MANUAL", "IMPORT"] }, - ); - expect(result.canonical.map((r) => r.id)).toEqual(["withings-1"]); - }); -}); - -describe("pickCanonicalWorkout — fixture (b) two-source different workouts close in time", () => { - it("keeps both rows when sport types differ within the proximity window", () => { - const result = pickCanonicalWorkout([ - // 07:30 Walking + 07:32 Running — same wall-clock burst (user - // tapped one workout, the watch logged a separate one) but - // different sport types, so they must stay distinct rows. - row("walk-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "walking"), - row("run-1", "WITHINGS", "2026-05-14T07:32:00.000Z", "running"), - ]); - expect(result.canonical).toHaveLength(2); - expect(result.canonical.map((r) => r.id).sort()).toEqual([ - "run-1", - "walk-1", - ]); - expect(result.clusters).toHaveLength(2); - }); - - it("keeps both rows when the same sport occurs outside the proximity window", () => { - const result = pickCanonicalWorkout([ - row("run-am", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - // 2 hours later — well past the 5-minute cluster window. A - // morning run and an afternoon run on the same day must NOT - // collapse into one canonical row. - row("run-pm", "WITHINGS", "2026-05-14T09:30:00.000Z"), - ]); - expect(result.canonical).toHaveLength(2); - expect(result.clusters).toHaveLength(2); - }); -}); - -describe("pickCanonicalWorkout — fixture (c) single source pass-through", () => { - it("returns the only row unchanged when no other source contributed", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - ]); - expect(result.canonical).toHaveLength(1); - expect(result.canonical[0].id).toBe("apple-1"); - expect(result.clusters[0].pickedSource).toBe("APPLE_HEALTH"); - }); - - it("passes through a batch of disjoint single-source workouts", () => { - const result = pickCanonicalWorkout([ - row("run-1", "APPLE_HEALTH", "2026-05-13T07:30:00.000Z"), - row("run-2", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("run-3", "APPLE_HEALTH", "2026-05-15T07:30:00.000Z"), - ]); - expect(result.canonical.map((r) => r.id)).toEqual([ - "run-1", - "run-2", - "run-3", - ]); - }); - - it("returns empty for empty input", () => { - const result = pickCanonicalWorkout([]); - expect(result.canonical).toHaveLength(0); - expect(result.clusters).toHaveLength(0); - }); -}); - -describe("pickCanonicalWorkout — proximity-window boundary", () => { - it("exposes the default proximity window as 5 minutes", () => { - expect(DEFAULT_WORKOUT_PROXIMITY_MINUTES).toBe(5); - }); - - it("clusters two rows exactly at the proximity boundary", () => { - // 5-minute gap — picker accepts the boundary as in-window so a - // server's coarse cron tick (Withings hourly pull at HH:00 paired - // with HKWorkout export at HH:05) still dedups cleanly. - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:35:00.000Z"), - ]); - expect(result.canonical).toHaveLength(1); - }); - - it("anchors a cluster to its first row instead of chaining neighbours", () => { - const result = pickCanonicalWorkout([ - row("anchor", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("neighbour", "WITHINGS", "2026-05-14T07:34:00.000Z"), - row("chained", "WITHINGS", "2026-05-14T07:38:00.000Z"), - ]); - - expect(result.clusters).toHaveLength(2); - expect(result.canonical.map((item) => item.id)).toEqual([ - "anchor", - "chained", - ]); - }); - - it("keeps two rows just outside the proximity window", () => { - // 5 minutes and 1 second — picker rejects, two clusters. - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:35:01.000Z"), - ]); - expect(result.canonical).toHaveLength(2); - }); - - it("respects a custom proximity window", () => { - // Same fixture as the same-workout case but with a 30-second - // window — the 60-second-later Withings row falls outside and - // both rows survive. - const result = pickCanonicalWorkout( - [ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - ], - { proximityMinutes: 0.5 }, - ); - expect(result.canonical).toHaveLength(2); - }); -}); - -describe("pickCanonicalWorkout — determinism", () => { - it("returns the same canonical list regardless of input order", () => { - const a = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - row("manual-1", "MANUAL", "2026-05-14T07:30:30.000Z"), - ]); - const b = pickCanonicalWorkout([ - row("manual-1", "MANUAL", "2026-05-14T07:30:30.000Z"), - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - ]); - expect(a.canonical.map((r) => r.id)).toEqual(b.canonical.map((r) => r.id)); - }); - - it("preserves same-source workouts at the same timestamp", () => { - const result = pickCanonicalWorkout([ - row("a-2", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("a-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - ]); - - expect(result.canonical.map((item) => item.id)).toEqual(["a-1", "a-2"]); - }); -}); - -describe("pickCanonicalWorkout — field-merge (v1.29.x, WHOOP HR-loss fix)", () => { - it("backfills avgHeartRate/maxHeartRate from a lower-priority twin when the winner's is null", () => { - // Regression fixture from the brief: a WHOOP cycling row carries live - // HR; a higher-priority Apple Health twin for the SAME ride has no HR - // (e.g. a manually-logged / GPS-only Apple entry). Before the merge, - // the picker kept the Apple row whole and threw the WHOOP HR away. - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling", { - avgHeartRate: null, - maxHeartRate: null, - }), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling", { - avgHeartRate: 142, - maxHeartRate: 168, - }), - ]); - expect(result.canonical).toHaveLength(1); - const winner = result.canonical[0]; - expect(winner.id).toBe("apple-1"); // ladder winner stays the base row - expect(winner.avgHeartRate).toBe(142); - expect(winner.maxHeartRate).toBe(168); - expect(result.clusters[0].picked.avgHeartRate).toBe(142); - }); - - it("backfills a lower-priority donor only onto its nearest winning-source workout", () => { - const fixtures = [ - row( - "apple-earlier", - "APPLE_HEALTH", - "2026-05-14T07:30:00.000Z", - "running", - { - avgHeartRate: null, - }, - ), - row( - "apple-later", - "APPLE_HEALTH", - "2026-05-14T07:34:00.000Z", - "running", - { - avgHeartRate: null, - }, - ), - row("whoop-later", "WHOOP", "2026-05-14T07:34:00.000Z", "running", { - avgHeartRate: 151, - }), - ]; - - for (const input of [fixtures, [...fixtures].reverse()]) { - const result = pickCanonicalWorkout(input); - - expect( - result.canonical.map(({ id, source, avgHeartRate }) => ({ - id, - source, - avgHeartRate, - })), - ).toEqual([ - { - id: "apple-earlier", - source: "APPLE_HEALTH", - avgHeartRate: null, - }, - { - id: "apple-later", - source: "APPLE_HEALTH", - avgHeartRate: 151, - }, - ]); - } - }); - - it("never overwrites a field the base row already has", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling", { - avgHeartRate: 120, - }), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling", { - avgHeartRate: 142, - }), - ]); - expect(result.canonical[0].avgHeartRate).toBe(120); - }); - - it("backfills totalEnergyKcal, totalDistanceM, and elevationM independently", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling", { - totalEnergyKcal: null, - totalDistanceM: 15000, - elevationM: null, - }), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling", { - totalEnergyKcal: 480, - totalDistanceM: 14800, - elevationM: 220, - }), - ]); - const winner = result.canonical[0]; - expect(winner.totalEnergyKcal).toBe(480); // backfilled - expect(winner.totalDistanceM).toBe(15000); // base row's own value kept - expect(winner.elevationM).toBe(220); // backfilled - }); - - it("prefers the ladder-highest member that HAS a field over a lower one, when the base row lacks it", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling", { - avgHeartRate: null, - }), - row("whoop-1", "WHOOP", "2026-05-14T07:30:30.000Z", "cycling", { - avgHeartRate: 142, - }), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z", "cycling", { - avgHeartRate: 99, - }), - ]); - // WHOOP outranks WITHINGS on the default ladder — its HR wins the - // backfill even though WITHINGS also had one. - expect(result.canonical[0].avgHeartRate).toBe(142); - }); - - it("adopts a specific sport type from a member when the base row's own sportType is generic", () => { - // The confirmed bug's downstream symptom: before the WHOOP sport-map - // fix, a WHOOP cycling workout could carry sportType "other" (or a - // non-canonical raw label) while a lower-priority twin correctly - // tagged "cycling". The merge adopts the specific sport so the - // cluster still surfaces as a bike ride. - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "other"), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling"), - ]); - expect(result.canonical[0].sportType).toBe("cycling"); - }); - - it("keeps the base row's own specific sportType even when a member disagrees", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling"), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling"), - ]); - expect(result.canonical[0].sportType).toBe("cycling"); - }); - - it("does not run the merge on a single-element cluster", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling", { - avgHeartRate: null, - }), - ]); - expect(result.canonical[0].avgHeartRate).toBeNull(); - }); - - it("end-to-end: a WHOOP-cycling+HR row clustered with a generic no-HR twin yields a cycling canonical row WITH the HR", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "other", { - avgHeartRate: null, - totalEnergyKcal: null, - }), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling", { - avgHeartRate: 145, - totalEnergyKcal: 512, - }), - ]); - expect(result.canonical).toHaveLength(1); - const winner = result.canonical[0]; - expect(winner.source).toBe("APPLE_HEALTH"); // ladder still picks Apple as the base - expect(winner.sportType).toBe("cycling"); // adopted from WHOOP - expect(winner.avgHeartRate).toBe(145); // backfilled from WHOOP - expect(winner.totalEnergyKcal).toBe(512); // backfilled from WHOOP - }); -}); - -describe("pickCanonicalWorkout — Strava brick-workout regression", () => { - // Legacy Strava labels ("Ride", "Run") are both generic to this picker, - // so they share a cluster. The source is identical, however, which means - // both legs are distinct records and must survive. - it("preserves two same-source raw Strava workouts in one cluster", () => { - const result = pickCanonicalWorkout([ - row("strava-ride", "STRAVA", "2026-06-01T15:00:00.000Z", "Ride"), - row("strava-run", "STRAVA", "2026-06-01T15:03:00.000Z", "Run"), - ]); - - expect(result.clusters).toHaveLength(1); - expect(result.canonical.map((item) => item.id)).toEqual([ - "strava-ride", - "strava-run", - ]); - }); - - it("post-fix: a canonical Strava Ride + Run pair survives as two distinct rows", () => { - // Same timing as the pre-fix fixture above, but with the sportType - // values `mapActivity()` now writes (via `mapStravaSportType()`): - // "Ride" → "cycling", "Run" → "running". Both are SPECIFIC canonical - // buckets and differ from each other, so `isSpecificSportType()` no - // longer treats them as sport-compatible and the brick session - // survives as two workouts. - const result = pickCanonicalWorkout([ - row("strava-ride", "STRAVA", "2026-06-01T15:00:00.000Z", "cycling"), - row("strava-run", "STRAVA", "2026-06-01T15:03:00.000Z", "running"), - ]); - expect(result.clusters).toHaveLength(2); - expect(result.canonical).toHaveLength(2); - const sports = result.canonical.map((w) => w.sportType).sort(); - expect(sports).toEqual(["cycling", "running"]); - }); -}); - -/** - * #1054 — workouts a `workouts:write` bridge pushes in carry `EXTERNAL`, which - * is on no default ladder. Pinned because the settings copy promises what - * follows from that: a bridged session the phone also recorded shows once, as - * the phone's copy, and a bridged session nothing else recorded still shows. - */ -describe("pickCanonicalWorkout — the unranked EXTERNAL source", () => { - it("is on no default ladder", () => { - expect(DEFAULT_WORKOUT_SOURCE_PRIORITY).not.toContain("EXTERNAL"); - }); - - it("yields to the phone's copy of the same session", () => { - const result = pickCanonicalWorkout([ - row("bridge-1", "EXTERNAL", "2026-09-20T06:30:00.000Z"), - row("apple-1", "APPLE_HEALTH", "2026-09-20T06:31:00.000Z"), - ]); - expect(result.canonical.map((w) => w.id)).toEqual(["apple-1"]); - }); - - it("survives when nothing else recorded the session", () => { - const result = pickCanonicalWorkout([ - row("bridge-1", "EXTERNAL", "2026-09-20T06:30:00.000Z"), - row("apple-1", "APPLE_HEALTH", "2026-09-20T09:00:00.000Z"), - ]); - expect(result.canonical.map((w) => w.id).sort()).toEqual([ - "apple-1", - "bridge-1", - ]); - }); -}); diff --git a/src/lib/sources/pick-canonical-workout.ts b/src/lib/sources/pick-canonical-workout.ts deleted file mode 100644 index 9f1da7274..000000000 --- a/src/lib/sources/pick-canonical-workout.ts +++ /dev/null @@ -1,437 +0,0 @@ -/** - * v1.4.25 W16a — cross-source workout canonical picker. - * - * Companion to `pickCanonicalSourceRows()` (the Measurement-row picker - * in `src/lib/analytics/source-priority.ts`). Workouts have a similar - * double-ingest problem coming in v1.5: the iOS app will write an - * HKWorkout row at the same wall-clock time the Withings Activity API - * already supplied — same run, two rows. - * - * This helper is scaffolding only — it returns the canonical workout - * list, doesn't mutate the DB, and is not yet wired into any query - * path. The wiring lands alongside the workout ingest endpoint - * (W16b in the v1.4.25 plan) or with the v1.5 iOS rollout, whichever - * arrives first. - * - * Algorithm - * ───────── - * - * 1. Cluster workouts whose start timestamps land within - * `proximityMinutes` of each other (default 5 min — Apple's - * ScanWatch passthrough lag plus a small clock-skew buffer). - * Sport type must match across the cluster; a "Walking" workout - * and a "Running" workout starting at the same instant stay - * distinct rows so the cardio-vs-strength split stays clean. Once - * an integration writes a canonical `sportType` (every source does, - * as of the WHOOP fix below), a cross-source pair of the SAME - * workout always shares a sport bucket and clusters correctly — - * before that fix a WHOOP "whoop_sport_1" row could never cluster - * with an Apple Health "cycling" row. - * 2. Per cluster, walk the per-user source-priority ladder for - * workouts (default `APPLE_HEALTH > WITHINGS > MANUAL > IMPORT`) - * and keep the first source that has a row in the cluster as the - * BASE of the canonical row. All other rows from the same cluster - * are dropped from the canonical list. - * 3. Field-merge (multi-element clusters only): each workout from the - * winning source is backfilled only from losing-source members matched - * to that workout by sport identity and start-time proximity. A donor is - * assigned to one deterministic best winner (exact sport before generic - * compatibility, then nearest timestamp). Missing `avgHeartRate`, - * `maxHeartRate`, `totalEnergyKcal`, `totalDistanceM`, and `elevationM` - * fields come from the ladder-highest matched donor that has the field. - * The same step adopts a specific sport type when the base row is generic - * (`"other"` / unrecognised). Every OTHER field on the base row (id, - * source, startedAt, route, metadata, ...) is untouched — the merge only - * ever fills a gap, never overwrites a value the base row already has. - * 4. Single-element clusters (no near-duplicate) pass through - * unchanged regardless of source — no merge step runs. - * - * Rationale for the default ladder - * ──────────────────────────────── - * - * - `APPLE_HEALTH` first: the iOS-app passthrough carries the - * watch's per-second sample stream (energy, HR, optional GPS), - * so when both sources have the same workout it's the richer - * row. - * - `WITHINGS` second: ScanWatch syncs into Withings cloud too, - * but the API caps each workout at coarser aggregates (mean HR, - * totals, no per-sample HR series). Loses cleanly when paired - * with the HKWorkout twin. - * - `MANUAL` third: the user typed it; trust beats anything from - * `IMPORT` (CSV / XML) but loses to a sensor stream. - * - `IMPORT` last: bulk historical imports (Apple Health XML, - * Strava export, Garmin TCX) — useful when no live sensor source - * covers the period but the lowest fidelity when one does. - * - * The picker is deterministic on `(rows, ladder, proximityMinutes)` - * and stable on input order. The caller is responsible for sorting - * rows by `startedAt ASC, id ASC` before invocation so the picker's - * cluster-building walks a fixed sequence. - */ - -import type { MeasurementSource } from "@/generated/prisma/client"; - -import { - workoutSportTypeEnum, - type WorkoutSportType, -} from "@/lib/validations/workout"; - -/** - * Minimum row shape the helper consults. Exposed as an interface so the - * Prisma `Workout` model and the not-yet-existing ingest DTO both - * compose with the picker without a transform step. Anything else on - * the row (route blob, metadata, ...) flows through untouched — ONLY the - * fields declared below are eligible for the step-3 field-merge, and even - * those are backfilled (fills a gap on the base row), never overwritten. - */ -export interface WorkoutPickerRow { - /** Stable identifier — `id` for DB rows, externalId+source for - * not-yet-persisted ingest payloads. Used for deterministic - * tie-breaking when two sources both win the priority round. */ - id: string; - startedAt: Date; - source: MeasurementSource; - /** Canonical sport string. Workouts of different sport types never - * collapse into one cluster — see step 1 of the algorithm. May be - * adopted from a losing cluster member by the step-3 merge when the - * base row's own value is generic — see `mergeClusterFields()`. */ - sportType: WorkoutSportType | string; - /** Backfillable richer fields — present when the source carries live - * sensor data (HR strap, GPS), absent/null on a coarser summary from - * another source recording the same workout. */ - avgHeartRate?: number | null; - maxHeartRate?: number | null; - totalEnergyKcal?: number | null; - totalDistanceM?: number | null; - elevationM?: number | null; -} - -/** - * The numeric fields step 3 of the algorithm may backfill onto the base - * row. Listed explicitly (not derived via `keyof`) so each field's copy - * stays independently type-checked in `mergeClusterFields()`. - */ -const MERGEABLE_NUMERIC_FIELDS = [ - "avgHeartRate", - "maxHeartRate", - "totalEnergyKcal", - "totalDistanceM", - "elevationM", -] as const; - -/** - * `true` for a specific canonical sport bucket; `false` for the generic - * `"other"` bucket or a string outside `workoutSportTypeEnum` entirely (a - * not-yet-normalised integration's raw label). Drives the sport-type - * adoption half of the merge. - */ -function isSpecificSportType(value: string): boolean { - return ( - value !== "other" && - (workoutSportTypeEnum.options as readonly string[]).includes(value) - ); -} - -function areSportTypesCompatible(left: string, right: string): boolean { - return ( - left === right || !isSpecificSportType(left) || !isSpecificSportType(right) - ); -} - -/** - * Match one losing-source donor to one winning-source workout. Exact sport - * identity is stronger than generic compatibility; within the same identity - * strength, the nearest start wins. `winners` is already startedAt/id-sorted, - * so an exact tie resolves deterministically to the first winner. - */ -function findMatchingWinner( - donor: T, - winners: readonly T[], - proximityMs: number, -): T | undefined { - let best: T | undefined; - let bestHasExactSport = false; - let bestDeltaMs = Infinity; - - for (const winner of winners) { - if (!areSportTypesCompatible(winner.sportType, donor.sportType)) continue; - - const deltaMs = Math.abs( - winner.startedAt.getTime() - donor.startedAt.getTime(), - ); - if (deltaMs > proximityMs) continue; - - const hasExactSport = winner.sportType === donor.sportType; - if ( - !best || - (hasExactSport && !bestHasExactSport) || - (hasExactSport === bestHasExactSport && deltaMs < bestDeltaMs) - ) { - best = winner; - bestHasExactSport = hasExactSport; - bestDeltaMs = deltaMs; - } - } - - return best; -} - -/** Ladder position of `source`, or `ladder.length` (last) when the source - * isn't on the ladder at all — an unranked source only wins a merge field - * when no ranked member has it. */ -function ladderRank( - source: MeasurementSource, - ladder: readonly MeasurementSource[], -): number { - const idx = ladder.indexOf(source); - return idx === -1 ? ladder.length : idx; -} - -/** - * Step 3 of the algorithm: backfill the ladder-winning base row's missing - * richer fields, and its generic sport type, from other cluster members. - * - * Deterministic: for each candidate field, the ladder-highest member that - * HAS a non-null value wins that field. `members` is already - * startedAt/id-sorted by the caller, so a same-rank tie resolves to - * whichever member sorts first. - * - * A single-element cluster returns `winning` unchanged — nothing to merge. - */ -function mergeClusterFields( - winning: T, - members: readonly T[], - ladder: readonly MeasurementSource[], -): T { - if (members.length <= 1) return winning; - - const merged: T = { ...winning }; - - for (const field of MERGEABLE_NUMERIC_FIELDS) { - if (merged[field] != null) continue; - let best: T | undefined; - let bestRank = Infinity; - for (const member of members) { - const value = member[field]; - if (value == null) continue; - const rank = ladderRank(member.source, ladder); - if (rank < bestRank) { - bestRank = rank; - best = member; - } - } - if (best) merged[field] = best[field]; - } - - if (!isSpecificSportType(merged.sportType)) { - let best: T | undefined; - let bestRank = Infinity; - for (const member of members) { - if (!isSpecificSportType(member.sportType)) continue; - const rank = ladderRank(member.source, ladder); - if (rank < bestRank) { - bestRank = rank; - best = member; - } - } - if (best) merged.sportType = best.sportType; - } - - return merged; -} - -/** - * Per-user, per-source priority list for workouts. The shape is - * parallel to `SOURCE_PRIORITY_METRIC_KEYS` entries but stored - * separately because workouts are their own entity (not a Measurement - * row), and the metric-priority Json blob on `User.sourcePriorityJson` - * never carries a `workouts` key today. - * - * Persistence of the per-user value lands when the dedup wiring lands; - * the helper accepts an explicit ladder so the future caller can pass - * the merged value without dragging schema work into this file. - */ -export const DEFAULT_WORKOUT_SOURCE_PRIORITY: readonly MeasurementSource[] = [ - "APPLE_HEALTH", - // v1.11.0 — WHOOP ranks second. Apple Watch GPS + HR is the richest run - // record; a WHOOP strap is the next-best when no watch logged the same - // session. The read-time picker clusters a WHOOP run and an Apple-Health - // run by (activityType, startedAt ± window) and keeps the ladder winner. - "WHOOP", - // v1.12.0 — Fitbit/Pixel exercise logs rank just below WHOOP: a wrist - // wearable in the same class, the next-best when no watch logged the same - // session. - "FITBIT", - "WITHINGS", - // v1.32.15 — Polar ranks below Withings and above Strava: a Polar watch is a - // device-native HR capture (unlike a Strava re-upload with degraded/absent - // HR), so a Polar exercise outranks a Strava twin of the same session but - // sits below the primary on-wrist wearables. Matches Polar's slot in the - // read-time `steps` ladder (`DEFAULT_SOURCE_PRIORITY.steps`). - "POLAR", - // v1.28.x — Strava ranks below the device-native captures (Apple/WHOOP/ - // Fitbit/Withings on-wrist HR) but above MANUAL/IMPORT: a Strava activity is - // often itself a re-upload of an Apple/Garmin recording, so its HR/calories - // are lower-fidelity or absent on the summary. A user who records primarily - // on Strava (phone GPS) can promote it in Settings → Sources. - "STRAVA", - "MANUAL", - "IMPORT", -] as const; - -/** - * Default proximity window. Apple's HealthKit passthrough is observed - * to arrive 0–60 seconds after the watch closes the workout; Withings - * Activity API rounds to the minute. 5 minutes covers both lags plus a - * small clock-skew buffer between the iOS app's clock and the server. - */ -export const DEFAULT_WORKOUT_PROXIMITY_MINUTES = 5; - -export interface PickCanonicalWorkoutOptions { - /** - * Per-user source ladder. Defaults to - * `DEFAULT_WORKOUT_SOURCE_PRIORITY`. The picker walks left-to-right - * and keeps the first source present in each cluster. - */ - sourcePriority?: readonly MeasurementSource[]; - /** - * Cluster window. Defaults to `DEFAULT_WORKOUT_PROXIMITY_MINUTES`. - * Workouts whose start timestamps are within this many minutes of - * the cluster anchor join that cluster; later neighbours cannot extend - * the window. - */ - proximityMinutes?: number; -} - -/** - * Pick the canonical-source workout list. See file-level docs for the - * algorithm; returns the surviving input rows plus one representative - * record per overlap cluster for debug overlays and audit logging. - */ -export function pickCanonicalWorkout( - rows: readonly T[], - options: PickCanonicalWorkoutOptions = {}, -): { - canonical: T[]; - clusters: Array<{ - /** Every row that clustered together, unmerged. */ - members: T[]; - pickedSource: MeasurementSource; - /** Representative winning row after cross-source field merge. */ - picked: T; - }>; -} { - if (rows.length === 0) { - return { canonical: [], clusters: [] }; - } - - const ladder = options.sourcePriority ?? DEFAULT_WORKOUT_SOURCE_PRIORITY; - const proximityMinutes = - options.proximityMinutes ?? DEFAULT_WORKOUT_PROXIMITY_MINUTES; - const proximityMs = proximityMinutes * 60_000; - - // Sort by startedAt so cluster building is a single linear pass. - // Stable secondary sort on `id` keeps determinism on ties — two rows - // recorded at the exact same instant with the same id ordering give - // the same picker output on every call. - const sorted = [...rows].sort((a, b) => { - const dt = a.startedAt.getTime() - b.startedAt.getTime(); - if (dt !== 0) return dt; - return a.id.localeCompare(b.id); - }); - - // Build clusters keyed by sport type (a Walking + Running pair - // starting at the same instant must stay distinct — different - // workouts entirely) — WITH one relaxation: a GENERIC sport - // ("other" / an unrecognised raw string) is treated as compatible - // with any SPECIFIC sport, so a thinly-classified row from one - // source (an unmapped import, a not-yet-normalised integration) can - // still cluster with a specific-sport twin from another source - // recording the SAME real-world workout — the merge step below then - // adopts the specific sport onto the canonical row. Two DIFFERENT - // specific sports (Walking vs Running) never become compatible this - // way — the core invariant is unchanged. - const clusters: T[][] = []; - for (const row of sorted) { - // Compare against the first row in the cluster. A later neighbour - // inside the window must not extend the cluster beyond its anchor. - let placed = false; - for (let i = clusters.length - 1; i >= 0; i--) { - const cluster = clusters[i]; - const head = cluster[0]; - const sameSport = head.sportType === row.sportType; - if (!areSportTypesCompatible(head.sportType, row.sportType)) continue; - if (row.startedAt.getTime() - head.startedAt.getTime() <= proximityMs) { - cluster.push(row); - placed = true; - break; - } - if (sameSport) break; - } - if (!placed) { - clusters.push([row]); - } - } - - // Preserve single-source clusters. For a real cross-source overlap, - // select the winning source and retain every workout it contributed. - const canonical: T[] = []; - const clusterRecords: Array<{ - members: T[]; - pickedSource: MeasurementSource; - picked: T; - }> = []; - for (const cluster of clusters) { - const sources = new Set(cluster.map((row) => row.source)); - if (sources.size === 1) { - canonical.push(...cluster); - clusterRecords.push({ - members: cluster, - pickedSource: cluster[0].source, - picked: cluster[0], - }); - continue; - } - - let pickedSource: MeasurementSource | undefined; - for (const candidate of ladder) { - if (cluster.some((row) => row.source === candidate)) { - pickedSource = candidate; - break; - } - } - if (!pickedSource) { - canonical.push(...cluster); - clusterRecords.push({ - members: cluster, - pickedSource: cluster[0].source, - picked: cluster[0], - }); - continue; - } - const winningRows = cluster.filter((row) => row.source === pickedSource); - const membersByWinner = new Map( - winningRows.map((winner) => [winner, [winner]]), - ); - for (const donor of cluster) { - if (donor.source === pickedSource) continue; - const matchedWinner = findMatchingWinner(donor, winningRows, proximityMs); - if (matchedWinner) membersByWinner.get(matchedWinner)?.push(donor); - } - const winners = winningRows.map((winner) => - mergeClusterFields( - winner, - membersByWinner.get(winner) ?? [winner], - ladder, - ), - ); - canonical.push(...winners); - clusterRecords.push({ - members: cluster, - pickedSource, - picked: winners[0], - }); - } - - return { canonical, clusters: clusterRecords }; -} diff --git a/src/lib/workouts/__tests__/canonical-rows.test.ts b/src/lib/workouts/__tests__/canonical-rows.test.ts deleted file mode 100644 index 4250b906a..000000000 --- a/src/lib/workouts/__tests__/canonical-rows.test.ts +++ /dev/null @@ -1,396 +0,0 @@ -/** - * v1.4.42 W5 — write-time canonical-row picker tests. - * - * Pairs with `src/lib/workouts/canonical-rows.ts`. The helper is the - * pre-`createMany` dedup pass invoked by `POST /api/workouts/batch` - * when the v1.5 iOS sprint posts a batch carrying both an Apple Watch - * row and a Withings ScanWatch row for the same Sunday-morning run. - * The tests pin the algorithm contract so a future tightening of the - * window or a tie-breaker tweak surfaces a failing assertion rather - * than a silent regression in the duplicate-drop count. - */ -import { describe, expect, it } from "vitest"; - -import { - dedupeWorkoutBatch, - WORKOUT_DEDUP_WINDOW_MS, - type WorkoutRow, -} from "../canonical-rows"; - -import type { MeasurementSource } from "@/generated/prisma/client"; - -interface TestRow extends WorkoutRow { - /** Marker so the test can assert which input row survived. */ - id: string; -} - -function row(partial: Partial & { id: string }): TestRow { - return { - id: partial.id, - userId: partial.userId ?? "user-1", - activityType: partial.activityType ?? "running", - startedAt: partial.startedAt ?? new Date("2026-05-21T08:00:00Z"), - source: partial.source ?? "APPLE_HEALTH", - caloriesKcal: partial.caloriesKcal ?? null, - createdAt: partial.createdAt ?? null, - index: partial.index, - }; -} - -describe("dedupeWorkoutBatch", () => { - it("returns an empty array on empty input", () => { - expect(dedupeWorkoutBatch([])).toEqual([]); - }); - - it("preserves overlapping workouts from the same source", () => { - const a = row({ - id: "a", - source: "APPLE_HEALTH", - caloriesKcal: 420, - createdAt: new Date("2026-05-21T08:30:00Z"), - }); - const b = row({ - id: "b", - source: "APPLE_HEALTH", - caloriesKcal: 420, - createdAt: new Date("2026-05-21T08:31:00Z"), - }); - - expect(dedupeWorkoutBatch([a, b]).map((item) => item.id)).toEqual([ - "a", - "b", - ]); - }); - - it("prefers the higher-priority source on a cross-source overlap inside the window", () => { - // Apple Watch + Withings ScanWatch both record the same run. The - // window is satisfied (rows are 60 s apart, well under the 90 s - // ceiling) and the activityType matches → group dedups to one - // survivor. APPLE_HEALTH outranks WITHINGS per the ladder. - const apple = row({ - id: "apple", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - caloriesKcal: 400, - }); - const withings = row({ - id: "withings", - source: "WITHINGS", - startedAt: new Date("2026-05-21T08:01:00Z"), - caloriesKcal: 500, - }); - - const out = dedupeWorkoutBatch([apple, withings]); - expect(out).toHaveLength(1); - // Source-axis wins outright — APPLE_HEALTH beats WITHINGS even - // when WITHINGS has higher calories. The calories tie-break only - // fires WITHIN a single source tier. - expect(out[0]!.id).toBe("apple"); - }); - - it("retains every row from the winning source in a cross-source overlap", () => { - const appleFirst = row({ - id: "apple-first", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const withings = row({ - id: "withings", - source: "WITHINGS", - startedAt: new Date("2026-05-21T08:00:30Z"), - }); - const appleSecond = row({ - id: "apple-second", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:01:00Z"), - }); - - expect( - dedupeWorkoutBatch([appleFirst, withings, appleSecond]).map( - (item) => item.id, - ), - ).toEqual(["apple-first", "apple-second"]); - }); - - it("returns every row unchanged when no overlap exists", () => { - // Two runs 10 min apart from the SAME source — well outside the - // 90 s window. Both must survive; the helper is not a generic - // "one run per day" filter. - const first = row({ - id: "first", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const second = row({ - id: "second", - startedAt: new Date("2026-05-21T08:10:00Z"), - }); - - const out = dedupeWorkoutBatch([first, second]); - expect(out.map((r) => r.id)).toEqual(["first", "second"]); - }); - - it("returns rows from different users untouched even on identical timestamps", () => { - // A multi-user batch (theoretically possible from an admin path) - // must never collapse rows across `userId`. The window check - // includes the userId in the grouping key. - const u1 = row({ - id: "u1", - userId: "user-1", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const u2 = row({ - id: "u2", - userId: "user-2", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - - const out = dedupeWorkoutBatch([u1, u2]); - expect(out.map((r) => r.id).sort()).toEqual(["u1", "u2"]); - }); - - it("returns rows of different activity types untouched at the same instant", () => { - // A user opens a "Walk" workout on the Watch and a "Strength" - // workout on the iPhone at the same instant. The window matches - // but the activityType doesn't — two distinct rows, both survive. - const walk = row({ - id: "walk", - activityType: "walking", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const strength = row({ - id: "strength", - activityType: "strength", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - - const out = dedupeWorkoutBatch([walk, strength]); - expect(out.map((r) => r.id).sort()).toEqual(["strength", "walk"]); - }); - - it("preserves same-source workouts when calories differ", () => { - const lo = row({ - id: "lo", - source: "APPLE_HEALTH", - caloriesKcal: 200, - }); - const hi = row({ - id: "hi", - source: "APPLE_HEALTH", - caloriesKcal: 500, - createdAt: new Date("2026-05-21T09:00:00Z"), - }); - - expect(dedupeWorkoutBatch([lo, hi]).map((item) => item.id)).toEqual([ - "lo", - "hi", - ]); - }); - - it("preserves same-source input order when creation times differ", () => { - const early = row({ - id: "early", - source: "WITHINGS", - caloriesKcal: 350, - createdAt: new Date("2026-05-21T08:30:00Z"), - }); - const late = row({ - id: "late", - source: "WITHINGS", - caloriesKcal: 350, - createdAt: new Date("2026-05-21T08:45:00Z"), - }); - - expect(dedupeWorkoutBatch([late, early]).map((item) => item.id)).toEqual([ - "late", - "early", - ]); - }); - - it("preserves same-source input order when every field ties", () => { - const first = row({ id: "first", source: "MANUAL" }); - const second = row({ id: "second", source: "MANUAL" }); - - expect(dedupeWorkoutBatch([first, second]).map((item) => item.id)).toEqual([ - "first", - "second", - ]); - expect(dedupeWorkoutBatch([second, first]).map((item) => item.id)).toEqual([ - "second", - "first", - ]); - }); - - it("treats a row exactly at the ±90 s boundary as in-window", () => { - // The window is inclusive (`<=`). A row landing on the boundary - // joins the group; one millisecond past stays separate. - const anchor = row({ - id: "anchor", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const onBoundary = row({ - id: "boundary", - source: "WITHINGS", - startedAt: new Date(anchor.startedAt.getTime() + WORKOUT_DEDUP_WINDOW_MS), - }); - const justOutside = row({ - id: "outside", - source: "WITHINGS", - startedAt: new Date( - anchor.startedAt.getTime() + WORKOUT_DEDUP_WINDOW_MS + 1, - ), - }); - - // anchor + onBoundary → one group; APPLE_HEALTH wins. - expect(dedupeWorkoutBatch([anchor, onBoundary]).map((r) => r.id)).toEqual([ - "anchor", - ]); - - // anchor + justOutside → two distinct groups, both survive. - expect(dedupeWorkoutBatch([anchor, justOutside]).map((r) => r.id)).toEqual([ - "anchor", - "outside", - ]); - }); - - it("anchors overlap groups to the first row instead of chaining neighbours", () => { - const anchor = row({ - id: "anchor", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const neighbour = row({ - id: "neighbour", - source: "WITHINGS", - startedAt: new Date( - anchor.startedAt.getTime() + WORKOUT_DEDUP_WINDOW_MS - 1_000, - ), - }); - const chained = row({ - id: "chained", - source: "WITHINGS", - startedAt: new Date( - anchor.startedAt.getTime() + 2 * (WORKOUT_DEDUP_WINDOW_MS - 1_000), - ), - }); - - expect( - dedupeWorkoutBatch([anchor, neighbour, chained]).map((item) => item.id), - ).toEqual(["anchor", "chained"]); - }); - - it("keeps anchor windows stable when the batch arrives out of order", () => { - const anchor = row({ - id: "anchor", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const neighbour = row({ - id: "neighbour", - source: "WITHINGS", - startedAt: new Date( - anchor.startedAt.getTime() + WORKOUT_DEDUP_WINDOW_MS - 1_000, - ), - }); - const chained = row({ - id: "chained", - source: "WITHINGS", - startedAt: new Date( - anchor.startedAt.getTime() + 2 * (WORKOUT_DEDUP_WINDOW_MS - 1_000), - ), - }); - - expect( - dedupeWorkoutBatch([neighbour, chained, anchor]).map((item) => item.id), - ).toEqual(["chained", "anchor"]); - }); - - it("walks the full source ladder (APPLE_HEALTH > WITHINGS > MANUAL > IMPORT)", () => { - const sources: MeasurementSource[] = [ - "IMPORT", - "MANUAL", - "WITHINGS", - "APPLE_HEALTH", - ]; - const rows = sources.map((source, idx) => - row({ - id: source, - source, - // Stagger by 10 s so each row sits inside the 90 s window - // relative to the anchor (the first row inserted). - startedAt: new Date( - new Date("2026-05-21T08:00:00Z").getTime() + idx * 10_000, - ), - }), - ); - - expect(dedupeWorkoutBatch(rows).map((r) => r.id)).toEqual(["APPLE_HEALTH"]); - }); - - it("honours a custom user ladder that promotes MANUAL above APPLE_HEALTH", () => { - // v1.4.43 W9 — when the user has customised Settings → Sources to - // promote MANUAL above APPLE_HEALTH, a Manual row inside the 90 s - // window MUST win over the Apple Watch twin. Without the - // user-priority wiring this helper would silently drop the - // user's preferred row at write-time before the read-time picker - // ever saw it. - const apple = row({ - id: "apple", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - caloriesKcal: 500, - }); - const manual = row({ - id: "manual", - source: "MANUAL", - startedAt: new Date("2026-05-21T08:01:00Z"), - caloriesKcal: 300, - }); - - const userPriorityJson = { - steps: ["MANUAL", "APPLE_HEALTH", "WITHINGS"] as MeasurementSource[], - }; - - const out = dedupeWorkoutBatch([apple, manual], userPriorityJson); - expect(out).toHaveLength(1); - expect(out[0]!.id).toBe("manual"); - }); - - it("falls back to the canonical default ladder when userPriorityJson is null", () => { - // Null = no user preference persisted. The helper must walk the - // canonical `DEFAULT_WORKOUT_SOURCE_PRIORITY` ladder so existing - // callers that don't supply the blob keep their v1.4.42 behaviour. - const apple = row({ - id: "apple", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const manual = row({ - id: "manual", - source: "MANUAL", - startedAt: new Date("2026-05-21T08:01:00Z"), - }); - - expect(dedupeWorkoutBatch([apple, manual], null).map((r) => r.id)).toEqual([ - "apple", - ]); - expect( - dedupeWorkoutBatch([apple, manual], undefined).map((r) => r.id), - ).toEqual(["apple"]); - // Argument omitted entirely — same behaviour as null/undefined. - expect(dedupeWorkoutBatch([apple, manual]).map((r) => r.id)).toEqual([ - "apple", - ]); - }); - - it("does not mutate caller rows", () => { - // The picker is pure — the caller's array reference and row - // identity survive the call so a future audit-overlay can - // re-inspect the dropped rows. - const a = row({ id: "a", source: "APPLE_HEALTH", caloriesKcal: 100 }); - const b = row({ id: "b", source: "WITHINGS", caloriesKcal: 100 }); - const before = JSON.stringify([a, b]); - dedupeWorkoutBatch([a, b]); - expect(JSON.stringify([a, b])).toBe(before); - }); -}); diff --git a/src/lib/workouts/canonical-rows.ts b/src/lib/workouts/canonical-rows.ts deleted file mode 100644 index b6fbf1688..000000000 --- a/src/lib/workouts/canonical-rows.ts +++ /dev/null @@ -1,308 +0,0 @@ -/** - * v1.4.42 W5 — write-time cross-source canonical-row picker for - * incoming `Workout` ingest payloads. - * - * Why this exists alongside the read-time helper - * ────────────────────────────────────────────── - * - * Two related-but-distinct dedup paths exist in the codebase: - * - * - `pickCanonicalWorkoutRows()` in `src/lib/measurements/` (v1.4.30) - * — READ-time picker invoked by `GET /api/workouts` / - * `GET /api/workouts/[id]`. It walks DB rows, looks up the user's - * persisted `sourcePriorityJson`, buckets by 5-minute slot, and - * filters the read result so the dashboard tile shows one row per - * logical workout. - * - * - This helper (`dedupeWorkoutBatch`, v1.4.42) — WRITE-time picker - * invoked by `POST /api/workouts/batch` BEFORE - * `prisma.workout.createMany`. The name intentionally diverges - * from the read-time picker to make the auto-completion path - * unambiguous (write-time payload-internal dedup vs. read-time - * cross-batch dedup with user-priority resolution). - * The v1.5 iOS sprint will ingest `HKWorkoutType` data in batches - * against `/api/workouts/batch`; Apple Watch + Withings ScanWatch - * paired to the same iPhone frequently capture the same workout - * simultaneously (e.g. a Sunday-morning run) and the iOS app may - * post both rows in one batch. Without write-time dedup, both - * rows would persist (their `externalId`s differ, so the - * `(userId, source, externalId)` unique index doesn't fire) and - * the read-time picker would carry the duplicate forward at every - * downstream read. - * - * The two pickers are layered, not redundant: - * - Write-time picker drops PAYLOAD-internal duplicates inside a - * single batch so we don't pay the storage cost. - * - Read-time picker still runs to collapse cross-batch duplicates - * (e.g. Apple Watch row from Monday's batch + ScanWatch row from - * Tuesday's backfill batch). - * - * Algorithm - * ───────── - * - * 1. Group by `(userId, activityType, startedAt ± 90 s window)`. - * The 90-second window is tighter than the read-time picker's - * 5-minute slot because a single batch from one iOS client has - * no cross-device clock skew to absorb — both rows are stamped - * with HealthKit's `startDate`, which Apple aggregates to the - * same instant across paired sensors. A 90-second cushion - * covers HK's documented ±60s smoothing plus a small buffer. - * 2. Groups containing one source are not duplicates and remain - * unchanged. For a cross-source group, select the winning source - * in order `APPLE_HEALTH ≻ WITHINGS ≻ MANUAL ≻ IMPORT`, then keep - * every row from that source. The literal source ladder comes from - * `DEFAULT_WORKOUT_SOURCE_PRIORITY` so both pickers share one source - * of truth. - * 3. When two sources share a priority tier, prefer the source whose - * best candidate has higher `caloriesKcal`, then earliest `createdAt`, - * then input order. These tie-breakers choose a source; they never - * collapse multiple workouts recorded by that winning source. - * - * Purity contract - * ─────────────── - * - * Zero Prisma imports, zero IO, zero user lookups. The helper takes - * an array, returns an array. Testable in isolation; safely invoked - * from a hot ingest path with no DB round-trip cost. - * - * Source-priority parity with the read-time picker - * ──────────────────────────────────────────────── - * - * CLOSED in v1.4.43 (W9-WORKOUTS-PRIORITY). The read-time picker - * resolves the per-user `sourcePriorityJson` (from - * `User.sourcePriorityJson`) for every read so a user who promoted - * MANUAL above APPLE_HEALTH in Settings → Sources sees their Manual - * rows surface above the Apple Watch ones. This helper now mirrors - * that contract — the optional `userPriorityJson` argument feeds - * through `parseSourcePriority()` + `getSourceLadder(..., "steps")`, - * the same chain the read-time picker walks, so a customised user - * keeps their preferred source at BOTH write- and read-time. - * - * When the argument is omitted (or null/undefined) the helper falls - * back to the canonical `DEFAULT_WORKOUT_SOURCE_PRIORITY` ladder so - * callers that don't yet have the user blob handy stay on the - * v1.4.42 behaviour. The batch route at - * `src/app/api/workouts/batch/route.ts` does one indexed - * `User.sourcePriorityJson` lookup at the start of the handler and - * passes the blob through here; the cost is a single `findUnique` - * per batch, well under the cost of letting a customised user's - * preferred row drop at write-time. - */ - -import { DEFAULT_WORKOUT_SOURCE_PRIORITY } from "@/lib/sources/pick-canonical-workout"; -import { - getSourceLadder, - parseSourcePriority, -} from "@/lib/validations/source-priority"; -import type { MeasurementSource } from "@/generated/prisma/client"; - -/** - * Minimum row shape consulted by the picker. Field names mirror the - * batch-route's prepared `Prisma.WorkoutCreateManyInput` plus a - * `createdAt` slot for the deterministic tie-breaker. Anything else - * the caller carried alongside the row (route geometry, HR series, - * metadata) survives untouched — the picker filters but never - * mutates and is generic in `T` so call sites keep their own - * carrier type. - * - * Naming aligned with the v1.5 iOS contract: - * - `activityType` — the sport type (`"running"`, `"cycling"`, - * ...). Matches `Workout.sportType` at the DB layer and - * `WorkoutSportType` at the validator layer; cross-source - * overlap requires identical sport so a "Walking" + "Running" - * pair at the same instant stays as two distinct rows. - * - `caloriesKcal` — the active-energy aggregate. The v1.5 iOS - * contract uses `caloriesKcal` for parity with HealthKit's - * `HKQuantityTypeIdentifierActiveEnergyBurned`; the server's - * write path maps it to `Workout.totalEnergyKcal`. - * - `createdAt` — when the row was minted. The batch path doesn't - * have this yet (it's set by Prisma's `@default(now())` at - * insert), so the helper falls back to insertion order via the - * `index` field when `createdAt` is undefined on every member. - */ -export interface WorkoutRow { - userId: string; - activityType: string; - startedAt: Date; - source: MeasurementSource; - caloriesKcal?: number | null; - /** - * Optional row-mint timestamp. The write-time picker is called - * BEFORE `createMany`, so prepared rows usually have no - * `createdAt` yet — undefined falls through to the next - * tie-breaker (insertion order via the `index` field). - */ - createdAt?: Date | null; - /** - * Stable per-batch ordinal. Set by the caller (e.g. the position - * of the entry in the inbound payload) so two rows with identical - * priority + calories + createdAt still resolve deterministically. - * Optional; defaults to 0 so single-row groups never need it. - */ - index?: number; -} - -/** - * Cross-source dedup window. Within a single ingest batch from one - * iOS client, two rows whose `startedAt` falls within ±90 s AND - * share the same `activityType` AND belong to the same user are - * treated as the same logical workout. The window is symmetric: - * the comparison uses `|a.startedAt - b.startedAt| ≤ 90 000 ms`. - */ -export const WORKOUT_DEDUP_WINDOW_MS = 90 * 1000; - -/** - * Source-priority rank lookup. Lower rank = higher priority. A - * source absent from the ladder gets a rank past the end of the - * array — the picker still keeps that row when no ladder-listed - * row competes for the slot (single-source group), but loses to - * any ladder-listed competitor. - * - * The ladder is resolved per-call (from the user's - * `sourcePriorityJson` or the canonical default), so the rank - * function is parameterised on the ladder rather than reading a - * module-level constant. - */ -function sourceRank( - source: MeasurementSource, - ladder: readonly MeasurementSource[], -): number { - const idx = ladder.indexOf(source); - return idx === -1 ? ladder.length : idx; -} - -/** - * Comparator: returns negative when `a` should win over `b`. - * Priority order: source rank → higher calories → earliest - * createdAt → earliest input index. - */ -function compareCandidates( - a: T, - b: T, - ladder: readonly MeasurementSource[], -): number { - const sourceDelta = - sourceRank(a.source, ladder) - sourceRank(b.source, ladder); - if (sourceDelta !== 0) return sourceDelta; - - // Tie-breaker 1 — higher caloriesKcal wins (more sensor data). - // `null` / `undefined` calories sort below any numeric value. - const aCal = a.caloriesKcal ?? -1; - const bCal = b.caloriesKcal ?? -1; - if (aCal !== bCal) return bCal - aCal; - - // Tie-breaker 2 — earliest `createdAt` wins (stable across - // re-runs). Undefined sorts below any concrete date so a - // pre-insert row (no createdAt yet) doesn't beat an already- - // persisted row carrying a real timestamp. - const aCreated = a.createdAt?.getTime() ?? Number.POSITIVE_INFINITY; - const bCreated = b.createdAt?.getTime() ?? Number.POSITIVE_INFINITY; - if (aCreated !== bCreated) return aCreated - bCreated; - - // Tie-breaker 3 — input order. Guarantees determinism even when - // every other field ties (rare; same source, same calories, - // same / no createdAt). Default of 0 keeps single-row groups - // untouched. - return (a.index ?? 0) - (b.index ?? 0); -} - -/** - * Pick canonical rows across sources for an inbound ingest batch. - * - * Group by `(userId, activityType, startedAt ± 90 s)`; a group with - * multiple sources keeps every row from its highest-priority source. - * Single-source overlaps pass through unchanged. The returned - * array preserves the input order of the surviving rows so a - * caller that pre-sorted by `startedAt ASC` keeps that ordering - * downstream. - * - * Implementation walks the input once with a union-find-style - * pass: each row joins an existing group whose anchor falls - * within the dedup window AND matches `(userId, activityType)`, - * or seeds a new group. The anchor is the FIRST row that landed - * in the group; subsequent rows compare against the anchor so the - * window can't drift across an arbitrarily long chain of - * within-window-of-neighbour rows. This is O(n²) worst case but - * a typical batch is ≤ 100 entries so the constant is fine. - */ -export function dedupeWorkoutBatch( - rows: readonly T[], - userPriorityJson?: unknown, -): T[] { - if (rows.length === 0) return []; - - // Resolve the source ladder from the user's persisted preference - // when one was supplied. The read-time picker - // (`pickCanonicalWorkoutRows`) keys workouts off the `steps` metric - // ladder — the canonical-activity bucket — so both pickers consult - // the same per-user configuration surface. When no preference is - // supplied (legacy callers, tests that don't care, or a route that - // hasn't been wired yet), fall back to the canonical default - // ladder so behaviour matches the v1.4.42 baseline. - const ladder: readonly MeasurementSource[] = - userPriorityJson == null - ? DEFAULT_WORKOUT_SOURCE_PRIORITY - : getSourceLadder(parseSourcePriority(userPriorityJson), "steps"); - - // Stamp each input with its original index so the comparator - // has a stable tie-breaker without mutating the caller's row. - type Tagged = { row: T; origIndex: number }; - const tagged: Tagged[] = rows.map((row, origIndex) => ({ row, origIndex })); - - // Group anchors: each group is a list of (row, origIndex). The - // anchor row sets the window comparison reference. - const groups: Tagged[][] = []; - - for (const entry of [...tagged].sort((a, b) => { - const delta = a.row.startedAt.getTime() - b.row.startedAt.getTime(); - return delta || a.origIndex - b.origIndex; - })) { - let placed = false; - for (const group of groups) { - const anchor = group[0]!.row; - if ( - anchor.userId === entry.row.userId && - anchor.activityType === entry.row.activityType && - Math.abs(anchor.startedAt.getTime() - entry.row.startedAt.getTime()) <= - WORKOUT_DEDUP_WINDOW_MS - ) { - group.push(entry); - placed = true; - break; - } - } - if (!placed) { - groups.push([entry]); - } - } - - // Preserve single-source groups. For genuine cross-source overlaps, - // select the winning source and retain every row it contributed. - const survivors: Tagged[] = []; - for (const group of groups) { - if (group.length === 1) { - survivors.push(group[0]!); - continue; - } - const sources = new Set(group.map((entry) => entry.row.source)); - if (sources.size === 1) { - survivors.push(...group); - continue; - } - const annotated = group.map((entry) => ({ - ...entry, - row: { - ...entry.row, - index: entry.row.index ?? entry.origIndex, - } as T, - })); - annotated.sort((a, b) => compareCandidates(a.row, b.row, ladder)); - const winningSource = annotated[0]!.row.source; - survivors.push( - ...group.filter((entry) => entry.row.source === winningSource), - ); - } - - survivors.sort((a, b) => a.origIndex - b.origIndex); - return survivors.map((entry) => entry.row); -} From ac861998081e75961a97793d3da5f49b20164c91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 04:36:54 +0200 Subject: [PATCH 06/11] fix(stats): one standard deviation with its kind named at the call site About a dozen hand-rolled spreads each picked population or sample variance without saying so, and the two surfaces a person compares picked differently: the series API reported the population SD of a blood-pressure series, the MCP tools and the AI payload the sample SD of the same readings, about 22 % apart at three readings. `lib/stats/spread` holds `variance` and `standardDeviation` with an explicit `population | sample` kind and `READING_SPREAD`, the kind every surface uses to report the spread of a person's readings. The series route and the feature payload both report it; the spreads that feed a verdict (the normal-swing checks, the changepoint threshold) and the clinical glucose SD stay the sample estimate, now said out loud. A guard fails on a new hand-rolled spread and on the two surfaces drifting apart. --- ...spread-single-implementation-guard.test.ts | 85 +++++++++++++++++++ src/app/api/measurements/series/route.ts | 7 +- src/lib/analytics/glucose-metrics.ts | 11 +-- src/lib/analytics/mood-prognosis/features.ts | 6 +- src/lib/analytics/score/activity.ts | 4 +- src/lib/analytics/score/glycaemia.ts | 4 +- src/lib/analytics/score/shared.ts | 7 +- src/lib/analytics/score/sleep.ts | 4 +- src/lib/analytics/trends.ts | 6 +- src/lib/insights/__tests__/features.test.ts | 21 +++++ src/lib/insights/features.ts | 24 ++++-- src/lib/insights/metric-signal.ts | 14 +-- src/lib/insights/mood-patterns.ts | 11 +-- src/lib/insights/seasonal-adjust.ts | 11 ++- src/lib/insights/signals-of-day.ts | 17 +++- src/lib/mcp/rich-reads.ts | 7 +- src/lib/rollups/tiered-context.ts | 6 +- src/lib/stats/__tests__/spread.test.ts | 23 +++++ src/lib/stats/spread.ts | 49 +++++++++++ src/lib/targets/mood-builder.ts | 42 +++------ 20 files changed, 263 insertions(+), 96 deletions(-) create mode 100644 src/__tests__/spread-single-implementation-guard.test.ts create mode 100644 src/lib/stats/__tests__/spread.test.ts create mode 100644 src/lib/stats/spread.ts diff --git a/src/__tests__/spread-single-implementation-guard.test.ts b/src/__tests__/spread-single-implementation-guard.test.ts new file mode 100644 index 000000000..46b5dfa5a --- /dev/null +++ b/src/__tests__/spread-single-implementation-guard.test.ts @@ -0,0 +1,85 @@ +/** + * Structural guard: a standard deviation is computed in `lib/stats/spread`, + * with its kind named at the call site. + * + * About a dozen hand-rolled spreads each picked population (÷ n) or sample + * (÷ n − 1) without saying so, and the two surfaces a person compares picked + * differently: the series API reported the population SD of a blood-pressure + * series, the MCP and AI payload the sample SD of the same readings, about + * 22 % apart at three readings. The reading spread both report is now + * `READING_SPREAD`. + * + * The matcher flags a squared deviation from a mean in a file that also takes + * a square root. The files below compute one for a test statistic or a fit, + * never as a reported spread, and say so. The doctor report's glucose panel + * composes its SD from per-bucket sums of squares (sample, the same rule as + * `glucoseSD`) and is out of the matcher's reach by shape. Comments are + * stripped first; the planted case proves the matcher fires. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +const ALLOWED: Record = { + "lib/stats/spread.ts": "The implementation.", + "lib/insights/correlations.ts": + "The correlation engine's centred sums and the Welch t-test variances.", + "lib/insights/mood-tag-influence.ts": "Welch t-test variances.", + "lib/insights/mood-pattern-cards.ts": "A pooled variance for a t statistic.", + "lib/insights/derived/trajectory.ts": + "The residual standard error of a regression fit (÷ n − 2).", +}; + +const SQUARED_DEVIATION = + /\(\s*[\w.[\]]+\s*-\s*[\w.[\]]+\s*\)\s*\*\*\s*2|\(\s*([\w.[\]]+)\s*-\s*([\w.[\]]+)\s*\)\s*\*\s*\(\s*\1\s*-\s*\2\s*\)/; + +function stripComments(text: string): string { + return text.replace(/\/\*[\s\S]*?\*\//g, "").replace(/^\s*\/\/.*$/gm, ""); +} + +function computesSpread(code: string): boolean { + return SQUARED_DEVIATION.test(code) && code.includes("Math.sqrt("); +} + +function read(file: string): string { + return stripComments(readFileSync(join(SRC, file), "utf8")); +} + +describe("one standard deviation", () => { + it("the matcher fires on a hand-rolled spread", () => { + expect( + computesSpread( + "const v = xs.reduce((s, x) => s + (x - m) ** 2, 0) / n; Math.sqrt(v);", + ), + ).toBe(true); + expect( + computesSpread("acc += (v - mean) * (v - mean); Math.sqrt(acc / n);"), + ).toBe(true); + }); + + it("no file outside the list computes its own", () => { + const offenders = walkSourceFiles(SRC, { floor: 3000 }) + .filter( + (file) => + !file.startsWith("generated/") && + !file.includes("__tests__/") && + !/\.test\.tsx?$/.test(file), + ) + .filter((file) => computesSpread(read(file)) && !(file in ALLOWED)); + expect(offenders).toEqual([]); + }); + + it("the series API and the AI and MCP payload report the same spread", () => { + expect(read("app/api/measurements/series/route.ts")).toMatch( + /standardDeviation\(\s*values,\s*READING_SPREAD\s*\)/, + ); + const features = read("lib/insights/features.ts"); + expect( + features.match(/roundedSpread\(\s*vals,\s*READING_SPREAD\s*\)/g), + ).toHaveLength(2); + }); +}); diff --git a/src/app/api/measurements/series/route.ts b/src/app/api/measurements/series/route.ts index b3abf5504..45bb64bb1 100644 --- a/src/app/api/measurements/series/route.ts +++ b/src/app/api/measurements/series/route.ts @@ -54,6 +54,7 @@ import { import { resolveSeriesReferenceLines } from "@/lib/measurements/series-reference-lines"; import { getServerTranslator } from "@/lib/i18n/server-translator"; import { resolveJobLocale } from "@/lib/i18n/job-locale"; +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; /** * v1.11.4 — sleep is read row-per-stage (not from the day rollup), so its @@ -209,10 +210,8 @@ interface SeriesPoint { function stdDev(values: number[]): number { if (values.length < 2) return 0; - const mean = values.reduce((sum, v) => sum + v, 0) / values.length; - const variance = - values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length; - return Math.round(Math.sqrt(variance) * 100) / 100; + const sd = standardDeviation(values, READING_SPREAD) ?? 0; + return Math.round(sd * 100) / 100; } export const GET = apiHandler(async (request: NextRequest) => { diff --git a/src/lib/analytics/glucose-metrics.ts b/src/lib/analytics/glucose-metrics.ts index a0f566735..06eca2cee 100644 --- a/src/lib/analytics/glucose-metrics.ts +++ b/src/lib/analytics/glucose-metrics.ts @@ -40,6 +40,7 @@ */ import { GLUCOSE_TIR_CUT_POINTS } from "@/lib/glucose"; +import { standardDeviation } from "@/lib/stats/spread"; // ── Inputs ─────────────────────────────────────────────── @@ -236,15 +237,7 @@ function mean(values: number[]): number { * Returns null for n < 2 (undefined sample SD). */ export function glucoseSD(values: number[]): number | null { - const n = values.length; - if (n < 2) return null; - const m = mean(values); - let ss = 0; - for (const v of values) { - const d = v - m; - ss += d * d; - } - return Math.sqrt(ss / (n - 1)); + return standardDeviation(values, "sample"); } /** diff --git a/src/lib/analytics/mood-prognosis/features.ts b/src/lib/analytics/mood-prognosis/features.ts index 467b71236..53f012998 100644 --- a/src/lib/analytics/mood-prognosis/features.ts +++ b/src/lib/analytics/mood-prognosis/features.ts @@ -68,6 +68,7 @@ import { LINKED_FEATURE_METRICS, } from "./feature-keys"; import { occurrencesSuffice } from "./thresholds"; +import { standardDeviation } from "@/lib/stats/spread"; /** * Most features the fit may carry. @@ -444,10 +445,7 @@ export function buildFeatureMatrix( const standardisation = selected.map((key, column) => { const values = rawRows.map((row) => row[column]); const mean = values.reduce((sum, v) => sum + v, 0) / values.length; - const variance = - values.reduce((sum, v) => sum + (v - mean) * (v - mean), 0) / - values.length; - return { key, mean, sd: Math.sqrt(variance) }; + return { key, mean, sd: standardDeviation(values, "population") ?? 0 }; }); // A column with no spread inside the complete-case set carries no diff --git a/src/lib/analytics/score/activity.ts b/src/lib/analytics/score/activity.ts index 4601eb786..326363dfd 100644 --- a/src/lib/analytics/score/activity.ts +++ b/src/lib/analytics/score/activity.ts @@ -9,7 +9,7 @@ import { insufficientPillar, mean, okPillar, - standardDeviation, + readingSpread, } from "./shared"; export const ACTIVITY_WINDOW_DAYS = 28; @@ -74,7 +74,7 @@ export function computeActivityPillar( const score = clampScore((observed / plateau) * 100); const noiseFloor = Math.max( 1, - Math.round(((0.5 * standardDeviation(values)) / plateau) * 100), + Math.round(((0.5 * readingSpread(values)) / plateau) * 100), ); const latestDay = [...points].sort((a, b) => b.day.localeCompare(a.day))[0]; diff --git a/src/lib/analytics/score/glycaemia.ts b/src/lib/analytics/score/glycaemia.ts index b62462a02..dca0bb893 100644 --- a/src/lib/analytics/score/glycaemia.ts +++ b/src/lib/analytics/score/glycaemia.ts @@ -10,7 +10,7 @@ import { median, okPillar, scoreReferenceBand, - standardDeviation, + readingSpread, uniqueSources, withinWindow, } from "./shared"; @@ -126,7 +126,7 @@ export function computeGlycaemiaPillar( const observed = median(values); // The behavioural/device noise convention from the design is one half of // within-person SD, translated onto this pillar's proportional score scale. - const halfSd = standardDeviation(values) * 0.5; + const halfSd = readingSpread(values) * 0.5; const floorScore = scoreReferenceBand( Math.max(0, observed - halfSd), FASTING_GLUCOSE_REFERENCE.low, diff --git a/src/lib/analytics/score/shared.ts b/src/lib/analytics/score/shared.ts index e5dfbbdfa..1fa3bff2d 100644 --- a/src/lib/analytics/score/shared.ts +++ b/src/lib/analytics/score/shared.ts @@ -11,6 +11,7 @@ import type { } from "@/lib/insights/derived/types"; import type { PillarValue, ScoreBand, ScorePillarId } from "./types"; +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; export const DAY_MS = 86_400_000; @@ -88,11 +89,9 @@ export function median(values: readonly number[]): number { : ordered[middle]; } -export function standardDeviation(values: readonly number[]): number { +export function readingSpread(values: readonly number[]): number { if (values.length < 2) return 0; - const center = mean(values); - const variance = mean(values.map((value) => (value - center) ** 2)); - return Math.sqrt(variance); + return standardDeviation(values, READING_SPREAD) ?? 0; } export function uniqueSources(values: readonly string[]): string[] { diff --git a/src/lib/analytics/score/sleep.ts b/src/lib/analytics/score/sleep.ts index 086fd887c..bebc81462 100644 --- a/src/lib/analytics/score/sleep.ts +++ b/src/lib/analytics/score/sleep.ts @@ -13,7 +13,7 @@ import { insufficientPillar, mean, okPillar, - standardDeviation, + readingSpread, } from "./shared"; export const SLEEP_WINDOW_DAYS = 28; @@ -75,7 +75,7 @@ export function computeSleepPillar( : Math.round(sufficiency * 0.6 + regularity * 0.4); const noiseFloor = Math.max( 1, - Math.round(((0.5 * standardDeviation(asleepValues)) / need) * 100), + Math.round(((0.5 * readingSpread(asleepValues)) / need) * 100), ); const latest = nights.at(-1)!; diff --git a/src/lib/analytics/trends.ts b/src/lib/analytics/trends.ts index 242f30bea..08b17d4d7 100644 --- a/src/lib/analytics/trends.ts +++ b/src/lib/analytics/trends.ts @@ -3,6 +3,8 @@ * All functions are pure and work on sorted arrays of {date, value} pairs. */ +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; + export interface DataPoint { date: Date; value: number; @@ -95,9 +97,7 @@ export function detectAnomalies(data: DataPoint[], threshold = 2.0): Anomaly[] { const values = data.map((p) => p.value); const mean = values.reduce((s, v) => s + v, 0) / values.length; - const variance = - values.reduce((s, v) => s + (v - mean) ** 2, 0) / values.length; - const stdDev = Math.sqrt(variance); + const stdDev = standardDeviation(values, READING_SPREAD) ?? 0; if (stdDev === 0) return []; diff --git a/src/lib/insights/__tests__/features.test.ts b/src/lib/insights/__tests__/features.test.ts index f43d56adf..0f582b50c 100644 --- a/src/lib/insights/__tests__/features.test.ts +++ b/src/lib/insights/__tests__/features.test.ts @@ -682,3 +682,24 @@ describe("extractFeatures — a day of pulse is the mean of its hours' means", ( expect(f.historicalComparison?.pulse?.current7dAvg).toBe(71.25); }); }); + +describe("extractFeatures — blood-pressure spread", () => { + it("reports the same population spread the series API shows", async () => { + // 120 / 130 / 140 over the last three days: population SD 8.165, the + // sample SD (the old rule here) would be 10. + const day = 24 * 60 * 60 * 1000; + const rows = [120, 130, 140].map((value, i) => ({ + type: "BLOOD_PRESSURE_SYS", + value, + measuredAt: new Date(Date.now() - (i + 1) * day), + sleepStage: null, + source: "MANUAL", + deviceType: null, + })); + prismaMock.measurement.findMany.mockResolvedValue(rows); + + const features = await extractFeatures("user-1", false); + + expect(features.bloodPressure?.sdSys30).toBe(8.2); + }); +}); diff --git a/src/lib/insights/features.ts b/src/lib/insights/features.ts index cc24519de..ee6c70f76 100644 --- a/src/lib/insights/features.ts +++ b/src/lib/insights/features.ts @@ -30,6 +30,11 @@ import { import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; import { getBpTargets } from "@/lib/analytics/bp-targets"; import { isBpReadingInTarget } from "@/lib/analytics/bp-in-target"; +import { + READING_SPREAD, + standardDeviation, + type VarianceKind, +} from "@/lib/stats/spread"; import { pairByTimestamp, significantPearsonCorrelation, @@ -894,12 +899,17 @@ export class FeaturesPayloadTooLargeError extends Error { } } -export function stdDev(values: number[]): number | null { +/** + * A spread rounded to the payload's one decimal, `null` below two values: a + * single reading has no spread to report, not a spread of zero. + */ +export function roundedSpread( + values: readonly number[], + kind: VarianceKind, +): number | null { if (values.length < 2) return null; - const mean = values.reduce((s, v) => s + v, 0) / values.length; - const variance = - values.reduce((s, v) => s + (v - mean) ** 2, 0) / (values.length - 1); - return Math.round(Math.sqrt(variance) * 10) / 10; + const sd = standardDeviation(values, kind); + return sd === null ? null : Math.round(sd * 10) / 10; } export function toDataPoints( @@ -1468,14 +1478,14 @@ export async function extractFeatures( const vals = sysData .filter((m) => m.measuredAt.getTime() >= fromMs) .map((m) => m.value); - return stdDev(vals); + return roundedSpread(vals, READING_SPREAD); })(), sdDia30: (() => { const fromMs = now - 30 * 24 * 60 * 60 * 1000; const vals = diaData .filter((m) => m.measuredAt.getTime() >= fromMs) .map((m) => m.value); - return stdDev(vals); + return roundedSpread(vals, READING_SPREAD); })(), pulsePressure30: (() => { const avgSys = sysSummary?.avg30 ?? null; diff --git a/src/lib/insights/metric-signal.ts b/src/lib/insights/metric-signal.ts index 1dbf261fd..8f943b256 100644 --- a/src/lib/insights/metric-signal.ts +++ b/src/lib/insights/metric-signal.ts @@ -25,6 +25,7 @@ */ import type { GradedSeries } from "@/lib/insights/graded-series"; import type { NormRange } from "@/lib/insights/derived/norms"; +import { standardDeviation } from "@/lib/stats/spread"; /** Favourable-direction framing — mirrors the metric-status registry. */ export type MetricDirection = "higher-better" | "lower-better" | "target-band"; @@ -96,16 +97,6 @@ function meanOf(values: number[]): number | null { return sum / values.length; } -/** Sample standard deviation, or null with fewer than two points. */ -function sampleSd(values: number[]): number | null { - if (values.length < 2) return null; - const m = meanOf(values); - if (m === null) return null; - let acc = 0; - for (const v of values) acc += (v - m) ** 2; - return Math.sqrt(acc / (values.length - 1)); -} - /** Placement of a value against a coarse normal range. */ export function placeInBand( value: number, @@ -169,7 +160,8 @@ export function buildMetricSignal( } const baseline = meanOf(baselineMeans); - const spread = sampleSd(baselineMeans); + // The spread decides the verdict, so it is the sample estimate. + const spread = standardDeviation(baselineMeans, "sample"); let delta: number | null = null; let deltaPct: number | null = null; diff --git a/src/lib/insights/mood-patterns.ts b/src/lib/insights/mood-patterns.ts index 7dc5fb218..289748095 100644 --- a/src/lib/insights/mood-patterns.ts +++ b/src/lib/insights/mood-patterns.ts @@ -20,6 +20,7 @@ import { import { round } from "@/lib/insights/status-shared"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; import { wallClockInTz } from "@/lib/tz/wall-clock"; +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; // --- In-target % over the last 30 daily points (lifted from mood-status) --- @@ -289,11 +290,11 @@ export function computeMoodStability( ): MoodStability | null { if (daily.length < STABILITY_MIN_DAYS) return null; - const values = daily.map((bucket) => bucket.value); - const avg = values.reduce((sum, v) => sum + v, 0) / values.length; - const variance = - values.reduce((sum, v) => sum + (v - avg) ** 2, 0) / values.length; - const stdDev = Math.sqrt(variance); + const stdDev = + standardDeviation( + daily.map((bucket) => bucket.value), + READING_SPREAD, + ) ?? 0; const normalised = Math.min(stdDev, STABILITY_SD_FULL_SCALE) / STABILITY_SD_FULL_SCALE; diff --git a/src/lib/insights/seasonal-adjust.ts b/src/lib/insights/seasonal-adjust.ts index 4db5e1c67..5be724265 100644 --- a/src/lib/insights/seasonal-adjust.ts +++ b/src/lib/insights/seasonal-adjust.ts @@ -29,6 +29,8 @@ * the regression runs over the days that exist and nothing is imputed. */ +import { variance as varianceOf } from "@/lib/stats/spread"; + /** Span (first to last paired day, inclusive) below which only the trend is removed. */ export const SEASONAL_MIN_SPAN_DAYS = 120; @@ -205,10 +207,11 @@ function densify( dense[days[i] - lo] = values[i]; sum += values[i]; } - const mean = sum / values.length; - let ss = 0; - for (const v of values) ss += (v - mean) ** 2; - return { dense, mean, variance: ss / values.length }; + return { + dense, + mean: sum / values.length, + variance: varianceOf(values, "population") ?? 0, + }; } /** diff --git a/src/lib/insights/signals-of-day.ts b/src/lib/insights/signals-of-day.ts index b522db563..3be2c7e0c 100644 --- a/src/lib/insights/signals-of-day.ts +++ b/src/lib/insights/signals-of-day.ts @@ -9,7 +9,11 @@ * so every existing call site keeps importing from there. */ import { trendSlope } from "@/lib/analytics/trends"; -import { avgInWindow, stdDev, toDataPoints } from "@/lib/insights/features"; +import { + avgInWindow, + roundedSpread, + toDataPoints, +} from "@/lib/insights/features"; import { SAME_HOURS_MIN_DAYS, SAME_HOURS_TYPES, @@ -177,7 +181,11 @@ function buildSignal( // spread and the anomaly stay over the readings. const avg7 = avgInWindow(records, now, 7, 0, day); const avg30 = avgInWindow(records, now, 30, 0, day); - const spread30 = stdDev(win30.map((rec) => rec.value)); + // The spread decides the verdict below, so it is the sample estimate. + const spread30 = roundedSpread( + win30.map((rec) => rec.value), + "sample", + ); const deltaVs7 = avg7 !== null ? shown(newest.value - avg7) : null; const deltaVs30 = avg30 !== null ? shown(newest.value - avg30) : null; const outsideNormalSwing = @@ -310,7 +318,10 @@ function buildSameHoursSignal( const avg30 = mean(earlier.map((d) => d.value)); const avg7 = mean(earlier.filter((d) => d.age <= 7).map((d) => d.value)); - const spread30 = stdDev(earlier.map((d) => d.value)); + const spread30 = roundedSpread( + earlier.map((d) => d.value), + "sample", + ); const latest = today.value; const outsideNormalSwing = avg30 !== null && spread30 !== null && spread30 > 0 diff --git a/src/lib/mcp/rich-reads.ts b/src/lib/mcp/rich-reads.ts index 4c7318286..53329ae9e 100644 --- a/src/lib/mcp/rich-reads.ts +++ b/src/lib/mcp/rich-reads.ts @@ -63,6 +63,7 @@ import { sanitizeValueText } from "@/lib/ai/coach/labs-snapshot"; import { scrubFenceMarkers } from "@/lib/ai/coach/data-fence"; import { encodeOffsetCursor } from "@/lib/mcp/pagination"; import type { CoachScopeWindow } from "@/lib/ai/coach/types"; +import { standardDeviation } from "@/lib/stats/spread"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -1158,11 +1159,9 @@ function mean(xs: number[]): number { return xs.reduce((s, x) => s + x, 0) / xs.length; } +/** The CUSUM threshold estimates the series' noise, so it is the sample SD. */ function stdDev(xs: number[]): number { - if (xs.length < 2) return 0; - const m = mean(xs); - const variance = xs.reduce((s, x) => s + (x - m) ** 2, 0) / (xs.length - 1); - return Math.sqrt(variance); + return standardDeviation(xs, "sample") ?? 0; } /** diff --git a/src/lib/rollups/tiered-context.ts b/src/lib/rollups/tiered-context.ts index b73f6073d..931911e5b 100644 --- a/src/lib/rollups/tiered-context.ts +++ b/src/lib/rollups/tiered-context.ts @@ -43,6 +43,7 @@ import { readDayAggregates } from "@/lib/measurements/day-aggregates"; import { dayWeightedRowsSql, windowMeanSql } from "@/lib/measurements/day-mean"; import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { Prisma } from "@/generated/prisma/client"; +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -203,9 +204,8 @@ function windowStats(buckets: RollupBucket[]): { const means = usable.map((b) => b.mean); const mean = means.reduce((s, m) => s + m, 0) / means.length; if (means.length < 2) return { mean: round(mean), sd: null, n }; - const variance = - means.reduce((s, m) => s + (m - mean) ** 2, 0) / means.length; - return { mean: round(mean), sd: round(Math.sqrt(variance)), n }; + const sd = standardDeviation(means, READING_SPREAD) ?? 0; + return { mean: round(mean), sd: round(sd), n }; } /** diff --git a/src/lib/stats/__tests__/spread.test.ts b/src/lib/stats/__tests__/spread.test.ts new file mode 100644 index 000000000..5192b17f7 --- /dev/null +++ b/src/lib/stats/__tests__/spread.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; + +import { READING_SPREAD, standardDeviation, variance } from "../spread"; + +describe("spread", () => { + const values = [120, 130, 140]; + + it("divides by n for a population and by n - 1 for a sample", () => { + expect(variance(values, "population")).toBeCloseTo(200 / 3, 10); + expect(variance(values, "sample")).toBe(100); + expect(standardDeviation(values, "sample")).toBe(10); + }); + + it("is undefined below what the kind needs", () => { + expect(variance([], "population")).toBeNull(); + expect(variance([5], "population")).toBe(0); + expect(variance([5], "sample")).toBeNull(); + }); + + it("reports readings as a population", () => { + expect(READING_SPREAD).toBe("population"); + }); +}); diff --git a/src/lib/stats/spread.ts b/src/lib/stats/spread.ts new file mode 100644 index 000000000..c06c944c5 --- /dev/null +++ b/src/lib/stats/spread.ts @@ -0,0 +1,49 @@ +/** + * Variance and standard deviation, with the kind named at every call site. + * + * - `population` divides by n: the spread of exactly these readings. It is + * what a surface reports when it describes a series the person can see. + * - `sample` divides by n − 1: an estimate of the spread that readings like + * these come from. It belongs where the spread feeds a decision (a + * significance verdict, a changepoint threshold) or where a clinical + * definition asks for it. + * + * The two differ by sqrt(n / (n − 1)), about 22 % at n = 3, so one series + * shown with both rules looks like two different series. Every hand-rolled + * spread used to pick its rule silently; the API reported the population SD + * of a blood-pressure series while the MCP and AI payload reported the + * sample SD of the same readings. + */ + +export type VarianceKind = "population" | "sample"; + +/** + * The kind every surface uses to report the spread of a person's readings, + * so the API, the MCP tools, the Coach and the AI payload say the same + * number for the same series. + */ +export const READING_SPREAD: VarianceKind = "population"; + +/** `null` when the kind is undefined for this many values (n < 1, or n < 2 for a sample). */ +export function variance( + values: readonly number[], + kind: VarianceKind, +): number | null { + const n = values.length; + const divisor = kind === "sample" ? n - 1 : n; + if (divisor < 1) return null; + let sum = 0; + for (const v of values) sum += v; + const mean = sum / n; + let squares = 0; + for (const v of values) squares += (v - mean) ** 2; + return squares / divisor; +} + +export function standardDeviation( + values: readonly number[], + kind: VarianceKind, +): number | null { + const v = variance(values, kind); + return v === null ? null : Math.sqrt(v); +} diff --git a/src/lib/targets/mood-builder.ts b/src/lib/targets/mood-builder.ts index 2e48e76c7..fdb917ae7 100644 --- a/src/lib/targets/mood-builder.ts +++ b/src/lib/targets/mood-builder.ts @@ -5,6 +5,7 @@ import type { TargetMoodRollup, TargetTrend, } from "./types"; +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; interface MoodTargetInput { moodRollups: TargetMoodRollup[]; @@ -24,9 +25,6 @@ export function buildMoodTargets({ const thirtyDaysAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); let dailyEvents: Array<{ measuredAt: Date; value: number }>; let entryCount: number; - let sumScores: number; - let varianceCount: number; - let sumSquares: number; if (moodRollups.length > 0) { entryCount = moodRollups.reduce((sum, rollup) => sum + rollup.count, 0); @@ -36,12 +34,6 @@ export function buildMoodTargets({ measuredAt: rollup.bucketStart, value: rollup.mean, })); - sumScores = dailyEvents.reduce((sum, event) => sum + event.value, 0); - varianceCount = dailyEvents.length; - sumSquares = dailyEvents.reduce( - (sum, event) => sum + event.value * event.value, - 0, - ); } else if (latestMoodEntry !== null) { const rawMood = recentRawMood ?? []; entryCount = rawMood.length; @@ -49,26 +41,20 @@ export function buildMoodTargets({ measuredAt: entry.moodLoggedAt, value: entry.score, })); - sumScores = rawMood.reduce((sum, entry) => sum + entry.score, 0); - varianceCount = rawMood.length; - sumSquares = rawMood.reduce( - (sum, entry) => sum + entry.score * entry.score, - 0, - ); } else { entryCount = 0; dailyEvents = []; - sumScores = 0; - varianceCount = 0; - sumSquares = 0; } + const values = dailyEvents.map((event) => event.value); if (entryCount < 3) return []; const current = latestMoodEntry?.score ?? null; const average30 = - dailyEvents.length > 0 - ? Math.round((sumScores / dailyEvents.length) * 10) / 10 + values.length > 0 + ? Math.round( + (values.reduce((sum, v) => sum + v, 0) / values.length) * 10, + ) / 10 : null; let trend: TargetTrend = null; if (dailyEvents.length >= 4) { @@ -118,23 +104,21 @@ export function buildMoodTargets({ }, ]; - if (varianceCount >= 5) { - const mean = sumScores / varianceCount; - const variance = sumSquares / varianceCount - mean * mean; - const standardDeviation = - Math.round(Math.sqrt(Math.max(0, variance)) * 100) / 100; + if (values.length >= 5) { + const spread = + Math.round((standardDeviation(values, READING_SPREAD) ?? 0) * 100) / 100; targets.push({ type: "MOOD_STABILITY", label: "Mood stability", - current: standardDeviation, - average30: standardDeviation, + current: spread, + average30: spread, trend: null, unit: "σ", range: { min: 0, max: 0.5 }, classification: - standardDeviation <= 0.5 + spread <= 0.5 ? { category: "Very stable", color: "var(--success)" } - : standardDeviation <= 1 + : spread <= 1 ? { category: "Stable", color: "var(--dracula-yellow)" } : { category: "Fluctuating", color: "var(--destructive)" }, source: "Mood entries", From 9ffcdea312aa737e7d18a294ef0f449e512cdfd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 04:37:08 +0200 Subject: [PATCH 07/11] fix(insights): keep chronic conditions out of the symptom series fill The symptom-severity series fills healthy days with zero across each illness episode's span. A chronic condition has no end, so its span covered the whole window and the correlation engine read a near-constant run of zeros. Only episodes that can put the account into Rest Mode (`REST_MODE_EPISODE_WHERE`) carry a span now. --- .../__tests__/symptom-series-chronic.test.ts | 78 +++++++++++++++++++ .../insights/correlation-channel-series.ts | 11 ++- 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 src/lib/insights/__tests__/symptom-series-chronic.test.ts diff --git a/src/lib/insights/__tests__/symptom-series-chronic.test.ts b/src/lib/insights/__tests__/symptom-series-chronic.test.ts new file mode 100644 index 000000000..c949b3380 --- /dev/null +++ b/src/lib/insights/__tests__/symptom-series-chronic.test.ts @@ -0,0 +1,78 @@ +/** + * A chronic condition does not zero-fill the symptom-severity series. + * + * The series fills healthy days with 0 across an episode's span. A chronic + * condition has no end, so its span covered the whole window and the engine + * correlated a near-constant run of zeros. Only episodes that can put the + * account into Rest Mode carry a span; the fake store below honours the + * lifecycle filter the way Postgres would. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +type Episode = { + id: string; + lifecycle: string; + onsetAt: Date; + resolvedAt: Date | null; +}; +const store: { episodes: Episode[] } = { episodes: [] }; + +vi.mock("@/lib/db", () => ({ + prisma: { + illnessEpisode: { + findMany: vi.fn( + async (args: { where: { lifecycle?: { not: string } } }) => + store.episodes.filter( + (e) => + args.where.lifecycle === undefined || + e.lifecycle !== args.where.lifecycle.not, + ), + ), + }, + illnessDayLog: { findMany: vi.fn(async () => []) }, + }, +})); + +import { fetchSymptomSeries } from "../correlation-channel-series"; + +const DAY = 24 * 60 * 60 * 1000; +const since = new Date(Date.now() - 60 * DAY); + +beforeEach(() => { + store.episodes = []; +}); + +describe("fetchSymptomSeries", () => { + it("has no series for a chronic condition alone", async () => { + store.episodes = [ + { + id: "chronic", + lifecycle: "CHRONIC_ONGOING", + onsetAt: new Date(Date.now() - 900 * DAY), + resolvedAt: null, + }, + ]; + const series = await fetchSymptomSeries("user-1", "UTC", since); + expect(series.points).toEqual([]); + }); + + it("fills only the acute episode's span beside a chronic one", async () => { + store.episodes = [ + { + id: "chronic", + lifecycle: "CHRONIC_ONGOING", + onsetAt: new Date(Date.now() - 900 * DAY), + resolvedAt: null, + }, + { + id: "cold", + lifecycle: "ACUTE", + onsetAt: new Date(Date.now() - 20 * DAY), + resolvedAt: new Date(Date.now() - 14 * DAY), + }, + ]; + const series = await fetchSymptomSeries("user-1", "UTC", since); + expect(series.points.length).toBeGreaterThanOrEqual(6); + expect(series.points.length).toBeLessThanOrEqual(8); + }); +}); diff --git a/src/lib/insights/correlation-channel-series.ts b/src/lib/insights/correlation-channel-series.ts index 57635b873..d383307fe 100644 --- a/src/lib/insights/correlation-channel-series.ts +++ b/src/lib/insights/correlation-channel-series.ts @@ -20,6 +20,7 @@ * fabricating a constant. */ import { prisma } from "@/lib/db"; +import { REST_MODE_EPISODE_WHERE } from "@/lib/illness/rest-mode"; import { buildComplianceMedicationContext, COURSES_COMPLIANCE_SELECT, @@ -152,9 +153,9 @@ export async function fetchComplianceSeries( * v1.21.0 (FDREXTEND) — build the user's SYMPTOM_SEVERITY daily series in the * `outcome` role (callers that need the behaviour role re-tag the returned * series — the points are role-invariant). Reads every in-window illness episode - * + its day-logs; the builder zero-fills healthy days ONLY across real episode - * spans, so a user with no episodes yields an empty series that degrades to - * absent. + * that is not a chronic condition + its day-logs; the builder zero-fills healthy + * days ONLY across those spans, so a user with no such episode yields an empty + * series that degrades to absent. */ export async function fetchSymptomSeries( userId: string, @@ -165,11 +166,15 @@ export async function fetchSymptomSeries( const episodes = await prisma.illnessEpisode.findMany({ // An episode overlaps the window when it onset before `now` and either is // ongoing or resolved at/after the window start. + // A chronic condition has no end, so its span would zero-fill every day + // of the window and hand the engine a near-constant series. Only the + // episodes that can put the account into Rest Mode carry a span. where: { userId, deletedAt: null, onsetAt: { lte: now }, OR: [{ resolvedAt: null }, { resolvedAt: { gte: since } }], + ...REST_MODE_EPISODE_WHERE, }, select: { id: true, onsetAt: true, resolvedAt: true }, }); From 2861075676c3d554b3bfb22e655de5e085a5c6ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 04:37:15 +0200 Subject: [PATCH 08/11] fix(mood): bound the mood series fallback to the rollup window Without mood rollups the series walked every mood entry the account ever logged, while the rollup path reads the trailing five years. The live fallback now reads the same window. --- src/lib/analytics/__tests__/mood-series.test.ts | 13 +++++++++++++ src/lib/analytics/mood-series.ts | 7 ++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/lib/analytics/__tests__/mood-series.test.ts b/src/lib/analytics/__tests__/mood-series.test.ts index 07ec24220..e29667876 100644 --- a/src/lib/analytics/__tests__/mood-series.test.ts +++ b/src/lib/analytics/__tests__/mood-series.test.ts @@ -83,6 +83,19 @@ describe("buildMoodDailySeries", () => { expect(series.summary.count).toBe(2); }); + it("bounds the live walk to the window the rollup read covers", async () => { + await buildMoodDailySeries("user-1"); + const [rollupUser, since] = vi.mocked(readMoodDayRollups).mock.calls[0] as [ + string, + Date, + ]; + expect(rollupUser).toBe("user-1"); + const args = vi.mocked(prisma.moodEntry.findMany).mock.calls[0]?.[0] as { + where: { moodLoggedAt?: { gte: Date } }; + }; + expect(args.where.moodLoggedAt?.gte).toEqual(since); + }); + it("returns an empty series for a user with no rollups and no raw entries", async () => { const series = await buildMoodDailySeries("user-1"); expect(series.entries).toEqual([]); diff --git a/src/lib/analytics/mood-series.ts b/src/lib/analytics/mood-series.ts index 676eb62e4..8e57d64e3 100644 --- a/src/lib/analytics/mood-series.ts +++ b/src/lib/analytics/mood-series.ts @@ -116,11 +116,12 @@ export async function buildMoodDailySeries( return { entries, summary, entryCount, source: "rollup" }; } - // Coverage fallback — no rollup rows yet. Run the legacy live walk ONCE; - // the warm-up above mints the rollups for the next request. + // Coverage fallback — no rollup rows yet. Run the live walk ONCE over the + // same window the rollup read covers; the warm-up above mints the rollups + // for the next request. const moodEntries = await client.moodEntry.findMany({ // v1.7.0 sync — exclude tombstoned rows from the fallback. - where: { userId, deletedAt: null }, + where: { userId, deletedAt: null, moodLoggedAt: { gte: since } }, orderBy: { moodLoggedAt: "asc" }, select: { date: true, score: true }, }); From e355c6120ffe56c1ffac983ff1804cbe751dbbb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 04:37:39 +0200 Subject: [PATCH 09/11] fix(workouts): one canonical-workout picker with one source order Three pickers decided which of two overlapping workouts to keep: the read-time picker the list, the detail page and the Coach use, a write-time batch picker with its own 90-second window, and an unwired field-merge picker. They used different source orders (the `steps` ladder, or a separate workout default when no order was saved), so the batch route could keep a row the list then ranked below another. The batch route now runs the read-time picker over the rows it is about to insert, with the ladder from `workoutSourceLadder`, and reports what it hides as duplicates. A session HealthKit hands over twice under two ids is stored once, as the list already showed it. The write-time and the unwired picker are removed. --- .../workout-canonical-identity-guard.test.ts | 24 ++++++++--- src/app/api/workouts/batch/route.ts | 42 +++++++------------ .../pick-canonical-workout-rows.ts | 36 ++++++++-------- src/lib/polar/__tests__/dedup.test.ts | 14 ++++--- src/lib/strava/__tests__/dedup.test.ts | 14 ++++--- src/lib/strava/sport-map.ts | 9 ++-- .../integration/workout-batch-create.test.ts | 35 +++++++++++++++- tests/integration/workout-batch-race.test.ts | 6 +-- 8 files changed, 109 insertions(+), 71 deletions(-) diff --git a/src/__tests__/workout-canonical-identity-guard.test.ts b/src/__tests__/workout-canonical-identity-guard.test.ts index 73fb493fb..416faa7e5 100644 --- a/src/__tests__/workout-canonical-identity-guard.test.ts +++ b/src/__tests__/workout-canonical-identity-guard.test.ts @@ -31,7 +31,7 @@ const SRC = join(process.cwd(), "src"); const PICKER = "lib/measurements/pick-canonical-workout-rows.ts"; /** - * Every production call site of the read-time picker, frozen. + * Every production call site of the picker, frozen. * * Adding a surface that collapses workouts is a deliberate act: it must fetch * the identity fields or it silently disagrees with every other surface. Adding @@ -48,6 +48,18 @@ const IDENTITY_OWNER: Record = { // block's data. Recording the owner here rather than assuming the call site // owns its own select is the whole point of this map. "lib/ai/coach/snapshot-blocks/workouts-block.ts": "lib/ai/coach/snapshot.ts", + // The write path: the batch route runs the same picker over the rows it is + // about to insert, so a duplicate inside one batch is dropped by the rule + // the list later applies. + "app/api/workouts/batch/route.ts": "app/api/workouts/batch/route.ts", +}; + +/** + * How a call site proves its rows carry the duration. A read selects the + * column; the batch route maps it off the row it computed for the insert. + */ +const IDENTITY_PROOF: Record = { + "app/api/workouts/batch/route.ts": "durationSec: p.row.durationSec", }; const EXPECTED_CALL_SITES = Object.keys(IDENTITY_OWNER).sort(); @@ -65,10 +77,8 @@ function read(rel: string): string { } /** - * Lines that actually invoke the picker. Comment lines are excluded because the - * write-time sibling (`lib/workouts/canonical-rows.ts`) names the read-time - * picker in its docblock to explain how the two layer, and that mention is - * documentation rather than a call. + * Lines that actually invoke the picker. Comment lines are excluded: a + * docblock that names the picker is documentation rather than a call. */ function invokesPicker(source: string): boolean { return source @@ -104,7 +114,9 @@ describe("every call site fetches the session identity", () => { // rows came from. Coarse in the safe direction — it can only fail to catch // a second unrelated select in the same file, never pass a caller that // fetches no duration at all. - expect(read(IDENTITY_OWNER[rel])).toContain("durationSec: true"); + expect(read(IDENTITY_OWNER[rel])).toContain( + IDENTITY_PROOF[rel] ?? "durationSec: true", + ); }); }); diff --git a/src/app/api/workouts/batch/route.ts b/src/app/api/workouts/batch/route.ts index 6d3e30d85..db2d1b1c7 100644 --- a/src/app/api/workouts/batch/route.ts +++ b/src/app/api/workouts/batch/route.ts @@ -91,7 +91,7 @@ import { UNSTABLE_EXTERNAL_ID_REASON, type UnstableExternalIdShape, } from "@/lib/validations/external-id"; -import { dedupeWorkoutBatch } from "@/lib/workouts/canonical-rows"; +import { pickCanonicalWorkoutRows } from "@/lib/measurements/pick-canonical-workout-rows"; import { WORKOUTS_WRITE_SCOPE } from "@/lib/workouts/scopes"; import { EXTERNAL_SOURCE } from "@/lib/measurements/external-source"; import { encryptRouteGeometry } from "@/lib/workouts/route-geometry-cipher"; @@ -494,41 +494,29 @@ async function postBatch(request: NextRequest): Promise { }); } - // v1.4.42 W5 — write-time cross-source dedup. The v1.5 iOS app - // drains a HealthKit observer queue that frequently carries the - // SAME logical workout from two paired sensors (Apple Watch + - // Withings ScanWatch). Without this pass, both rows persist (their - // `externalId`s differ → the `(userId, source, externalId)` unique - // index never fires) and the read-time canonical-picker has to - // carry the duplicate forward on every dashboard render. - // - // The helper is pure (no Prisma, no user lookups). It groups by - // `(userId, activityType, startedAt ± 90 s)`, prefers the canonical - // source ladder (APPLE_HEALTH > WITHINGS > MANUAL > IMPORT), and - // breaks ties on calories > earliest createdAt > input order. Rows - // dropped here surface to the iOS client as `duplicate` so the - // sync cursor advances past them identically to the externalId - // case below. - const canonicalPrepared = dedupeWorkoutBatch( + // Write-time cross-source dedup through the one workout picker the reads + // use, with the same source ladder: one logical workout from two sources + // in one batch (or one HealthKit session re-sent under a new id) lands + // once. Rows the picker hides surface to the client as `duplicate`, so + // the sync cursor advances past them like the externalId case below. + const canonicalPrepared = pickCanonicalWorkoutRows( prepared.map((p) => ({ + index: p.index, userId: p.row.userId, - activityType: p.row.sportType, + sportType: p.row.sportType, startedAt: p.row.startedAt as Date, + durationSec: p.row.durationSec, // Zod's `measurementSourceEnum.optional().default("MANUAL")` // guarantees `source` is set post-parse, but Prisma's - // `WorkoutCreateManyInput` types it optional. Default through - // the same fallback the validator uses so the picker's source - // ladder still resolves on the rare row where TypeScript - // can't prove the union. + // `WorkoutCreateManyInput` types it optional. source: (p.row.source ?? "MANUAL") as MeasurementSource, - caloriesKcal: p.row.totalEnergyKcal ?? null, - // No createdAt yet — the row is pre-insert; the tie-breaker - // falls through to input order via the `index` field below. - index: p.index, + avgHeartRate: p.row.avgHeartRate ?? null, + maxHeartRate: p.row.maxHeartRate ?? null, + metadata: p.row.metadata ?? null, })), userPriority?.sourcePriorityJson ?? null, ); - const survivingIndices = new Set(canonicalPrepared.map((r) => r.index ?? -1)); + const survivingIndices = new Set(canonicalPrepared.map((r) => r.index)); const droppedByWriteDedup: number[] = []; for (const p of prepared) { if (!survivingIndices.has(p.index)) { diff --git a/src/lib/measurements/pick-canonical-workout-rows.ts b/src/lib/measurements/pick-canonical-workout-rows.ts index cb97fd0ad..50fda8fdf 100644 --- a/src/lib/measurements/pick-canonical-workout-rows.ts +++ b/src/lib/measurements/pick-canonical-workout-rows.ts @@ -14,11 +14,9 @@ * logical run start within a minute of each other on both * sources; the rounding gives the cross-source matcher slack * without collapsing distinct back-to-back workouts. - * 2. For each bucket, walk the SOURCE priority ladder - * (`APPLE_HEALTH ≻ WITHINGS ≻ MANUAL ≻ IMPORT` per the existing - * source-priority validator's default) and pick the FIRST - * source that has any row in that bucket. Keep every row from - * the winning source; drop the rest. + * 2. For each bucket, walk the source ladder (`workoutSourceLadder`) + * and pick the FIRST source that has any row in that bucket. Keep + * every row from the winning source; drop the rest. * 3. Rows whose source isn't in the ladder pass through untouched * (fallback intent — never silently drop signal). * @@ -26,10 +24,10 @@ * (typically `ORDER BY startedAt ASC, id ASC` from the caller's * Prisma read). The output preserves input order. * - * Future v1.5.x tune: metric-aware picker per R-F open question #4 — - * route → Apple wins, HR zones → Withings wins. Out of scope for - * v1.4.30; the existing ladder ships first so the dashboard tile is - * unambiguous. + * This is the one workout picker. The reads use it, and so does the batch + * write route before it inserts: a write-time picker with its own window + * and a different default ladder kept or dropped a different row than the + * list then showed. */ import type { MeasurementSource } from "@/generated/prisma/client"; @@ -249,6 +247,17 @@ function enrichWinnersFromWhoop( return enriched; } +/** + * The source order for workouts: the `steps` metric's ladder, the + * canonical-activity bucket the Sources settings configure, falling back to + * its default when the user saved none or it does not parse. + */ +export function workoutSourceLadder( + userPriorityJson: unknown, +): readonly MeasurementSource[] { + return getSourceLadder(parseSourcePriority(userPriorityJson), "steps"); +} + /** * Pick canonical rows across sources for a user's workouts. * @@ -266,14 +275,7 @@ export function pickCanonicalWorkoutRows( ): T[] { if (rows.length === 0) return []; - // Workouts ride the `steps` metric's source ladder — the - // canonical-activity bucket. The user's iOS Settings → Sources UI - // configures the same ladder for the workout dashboard tile; - // unifying on "steps" keeps a single configuration surface rather - // than introducing a new `workouts` metric key. Future ladders for - // workout-specific tunes (route-vs-zones) ride a v1.5.x change. - const resolved = parseSourcePriority(userPriorityJson); - const sourceLadder = getSourceLadder(resolved, "steps"); + const sourceLadder = workoutSourceLadder(userPriorityJson); const buckets = new Map< string, diff --git a/src/lib/polar/__tests__/dedup.test.ts b/src/lib/polar/__tests__/dedup.test.ts index 0352b40be..21b379f01 100644 --- a/src/lib/polar/__tests__/dedup.test.ts +++ b/src/lib/polar/__tests__/dedup.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; -import { pickCanonicalWorkoutRows } from "@/lib/measurements/pick-canonical-workout-rows"; -import { DEFAULT_WORKOUT_SOURCE_PRIORITY } from "@/lib/sources/pick-canonical-workout"; +import { + pickCanonicalWorkoutRows, + workoutSourceLadder, +} from "@/lib/measurements/pick-canonical-workout-rows"; import { DEFAULT_SOURCE_PRIORITY } from "@/lib/validations/source-priority"; /** @@ -68,10 +70,10 @@ describe("POLAR workout dedup — read-time canonical picker", () => { }); describe("POLAR sits in the workout source ladders", () => { - it("is present in the workout default ladder, below WITHINGS, above STRAVA", () => { - const polar = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("POLAR"); - const withings = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("WITHINGS"); - const strava = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("STRAVA"); + it("is present in the workout ladder, below WITHINGS, above STRAVA", () => { + const polar = workoutSourceLadder(null).indexOf("POLAR"); + const withings = workoutSourceLadder(null).indexOf("WITHINGS"); + const strava = workoutSourceLadder(null).indexOf("STRAVA"); expect(polar).toBeGreaterThan(withings); expect(polar).toBeLessThan(strava); }); diff --git a/src/lib/strava/__tests__/dedup.test.ts b/src/lib/strava/__tests__/dedup.test.ts index 889c2e2a2..e74f9075f 100644 --- a/src/lib/strava/__tests__/dedup.test.ts +++ b/src/lib/strava/__tests__/dedup.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; -import { pickCanonicalWorkoutRows } from "@/lib/measurements/pick-canonical-workout-rows"; -import { DEFAULT_WORKOUT_SOURCE_PRIORITY } from "@/lib/sources/pick-canonical-workout"; +import { + pickCanonicalWorkoutRows, + workoutSourceLadder, +} from "@/lib/measurements/pick-canonical-workout-rows"; import { DEFAULT_SOURCE_PRIORITY } from "@/lib/validations/source-priority"; /** @@ -67,10 +69,10 @@ describe("STRAVA workout dedup — read-time canonical picker", () => { }); describe("STRAVA sits in the workout source ladders", () => { - it("is present in the workout default ladder, below the wearables, above MANUAL", () => { - const strava = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("STRAVA"); - const withings = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("WITHINGS"); - const manual = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("MANUAL"); + it("is present in the workout ladder, below the wearables, above MANUAL", () => { + const strava = workoutSourceLadder(null).indexOf("STRAVA"); + const withings = workoutSourceLadder(null).indexOf("WITHINGS"); + const manual = workoutSourceLadder(null).indexOf("MANUAL"); expect(strava).toBeGreaterThan(withings); expect(strava).toBeLessThan(manual); }); diff --git a/src/lib/strava/sport-map.ts b/src/lib/strava/sport-map.ts index c3702e9f8..b60c7a675 100644 --- a/src/lib/strava/sport-map.ts +++ b/src/lib/strava/sport-map.ts @@ -7,11 +7,10 @@ * to write `summary.sport_type ?? summary.type ?? detail?.sport_type ?? * "workout"` straight into `Workout.sportType`, so a Strava "Run" or * "VirtualRide" never matched a canonical bucket — `/insights/workouts` - * rendered every Strava row generic, and the read-time picker in - * `src/lib/sources/pick-canonical-workout.ts` treats any two GENERIC rows - * as sport-compatible, so a same-source brick session (a Strava "Ride" - * ending 15:00 immediately followed by a Strava "Run" starting 15:03) - * clustered into one and silently dropped a leg. Fitbit + * rendered every Strava row generic, and a picker that treats two generic + * rows as the same sport clusters a same-source brick session (a Strava + * "Ride" ending 15:00 immediately followed by a Strava "Run" starting + * 15:03) into one and silently drops a leg. Fitbit * (`mapFitbitSportType()` in `src/lib/fitbit/client.ts`), Google Health * (`mapGoogleHealthSportType()` in `src/lib/google-health/mappers.ts`), and * WHOOP (`mapWhoopSportType()` in `src/lib/whoop/sport-map.ts`) all diff --git a/tests/integration/workout-batch-create.test.ts b/tests/integration/workout-batch-create.test.ts index b7f7f3274..190fa34ef 100644 --- a/tests/integration/workout-batch-create.test.ts +++ b/tests/integration/workout-batch-create.test.ts @@ -356,8 +356,16 @@ describe("POST /api/workouts/batch (real Postgres)", () => { it("preserves same-source workouts and surfaces exact re-posts as duplicates", async () => { const { POST } = await import("@/app/api/workouts/batch/route"); + // Two sessions of one source in the same five-minute slot: different + // start and length, so two workouts. const body = { - workouts: [baseWorkout("hk-uuid-dup-1"), baseWorkout("hk-uuid-dup-2")], + workouts: [ + baseWorkout("hk-uuid-dup-1"), + baseWorkout("hk-uuid-dup-2", { + startedAt: "2026-05-14T06:32:00.000Z", + endedAt: "2026-05-14T06:50:00.000Z", + }), + ], }; const first = await POST(makeRequest(body)); @@ -393,6 +401,31 @@ describe("POST /api/workouts/batch (real Postgres)", () => { expect(stored).toBe(2); }); + it("stores a session re-sent under a new id once, the way the list shows it", async () => { + // HealthKit can hand the same session over twice with two external ids. + // The list collapses it to one workout; the write keeps one row and + // reports the other as a duplicate instead of storing what no read shows. + const { POST } = await import("@/app/api/workouts/batch/route"); + const res = await POST( + makeRequest({ + workouts: [ + baseWorkout("hk-uuid-resend-1"), + baseWorkout("hk-uuid-resend-2"), + ], + }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { + data: { inserted: number; duplicates: number }; + }; + expect(json.data.inserted).toBe(1); + expect(json.data.duplicates).toBe(1); + const stored = await getPrismaClient().workout.count({ + where: { userId: TEST_USER_ID }, + }); + expect(stored).toBe(1); + }); + it("replays a cached response when the same Idempotency-Key is reused", async () => { const { POST } = await import("@/app/api/workouts/batch/route"); diff --git a/tests/integration/workout-batch-race.test.ts b/tests/integration/workout-batch-race.test.ts index bc5a9361c..d55aed62b 100644 --- a/tests/integration/workout-batch-race.test.ts +++ b/tests/integration/workout-batch-race.test.ts @@ -182,9 +182,9 @@ describe("POST /api/workouts/batch — concurrent-write race", () => { const { POST } = await import("@/app/api/workouts/batch/route"); // Batch A — a cross-source pair one minute apart plus one shared key. - // The v1.4.42 `dedupeWorkoutBatch` picker collapses the Apple Health + - // Manual pair (same activity, within the 90 s window) to one - // surviving row, leaving 2 effective rows per batch. + // The workout picker collapses the Apple Health + Manual pair (same + // activity, same five-minute slot) to one surviving row, leaving 2 + // effective rows per batch. const batchA: WorkoutFixture[] = [ workout("hk-uuid-only-a-0", 1), workout("hk-uuid-only-a-1", 2, "MANUAL"), From feb80582330edc24acf591853c290164a1dcdfb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 04:37:44 +0200 Subject: [PATCH 10/11] fix(medications): tally the cadence chips from the compliance ledger `/api/medications/[id]/cadence` computed its adherence chip from a private copy of the compliance ledger that ignored pause eras, courses and auto-missed doses, and read only the chart's window of intakes. A medication paused for a week read 77 % there and 100 % on the detail page. The chips now tally `trailingComplianceLedger`, the ledger the compliance payload builds, over the same window, with the medication's pauses and courses; the copy is removed and a guard freezes the set of files that build dose rows. --- .../dose-ledger-single-source-guard.test.ts | 55 ++++++ src/app/api/medications/[id]/cadence/route.ts | 39 ++-- src/lib/analytics/compliance/display.ts | 13 +- src/lib/analytics/compliance/ledger.ts | 35 ++++ .../scheduling/__tests__/compliance.test.ts | 154 ++++++++------- src/lib/medications/scheduling/compliance.ts | 180 +++--------------- 6 files changed, 225 insertions(+), 251 deletions(-) create mode 100644 src/__tests__/dose-ledger-single-source-guard.test.ts diff --git a/src/__tests__/dose-ledger-single-source-guard.test.ts b/src/__tests__/dose-ledger-single-source-guard.test.ts new file mode 100644 index 000000000..aff5974b4 --- /dev/null +++ b/src/__tests__/dose-ledger-single-source-guard.test.ts @@ -0,0 +1,55 @@ +/** + * Structural guard: medication compliance is tallied from one ledger. + * + * The cadence chips carried a private copy of the ledger build that + * `lib/analytics/compliance/ledger.ts` owns, minus the pause eras and the + * courses, so `/api/medications/[id]/cadence` could report 77 % for a month + * the detail page called 100 %. The chips now tally the shared ledger. + * + * `reconstructDoseHistory` is the step that turns minted bands and intakes + * into dose rows. The set of files that call it is frozen: the ledger, the + * cadence timeline (the chart and the streaks) and the dose-history view. A + * new caller is a new ledger and has to be added here, next to a reason. + * Comments are stripped first; the count floor proves the matcher reads. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +const CALLERS: Record = { + "lib/analytics/compliance/ledger.ts": + "The compliance ledger every rate and the chips tally.", + "lib/medications/scheduling/cadence.ts": + "The cadence timeline the chart draws and the streaks walk.", + "app/api/medications/[id]/dose-history/route.ts": + "The dose-history view, row by row over the same bands.", +}; + +const CALL = /\breconstructDoseHistory\s*\(/; + +function stripComments(text: string): string { + return text.replace(/\/\*[\s\S]*?\*\//g, "").replace(/^\s*\/\/.*$/gm, ""); +} + +describe("one compliance ledger", () => { + it("the dose rows are built only where this file says", () => { + const callers = walkSourceFiles(SRC, { floor: 3000 }) + .filter( + (file) => + !file.startsWith("generated/") && + !file.includes("__tests__/") && + !/\.test\.tsx?$/.test(file) && + file !== "lib/medications/scheduling/dose-history.ts", + ) + .filter((file) => + CALL.test(stripComments(readFileSync(join(SRC, file), "utf8"))), + ) + .sort(); + expect(callers.length).toBeGreaterThanOrEqual(3); + expect(callers).toEqual(Object.keys(CALLERS).sort()); + }); +}); diff --git a/src/app/api/medications/[id]/cadence/route.ts b/src/app/api/medications/[id]/cadence/route.ts index 2660a2773..96daeb9b1 100644 --- a/src/app/api/medications/[id]/cadence/route.ts +++ b/src/app/api/medications/[id]/cadence/route.ts @@ -24,10 +24,12 @@ import { apiError, apiSuccess, returnAllZodIssues } from "@/lib/api-response"; import { buildCadenceTimeline, computeNextDose, - type CadenceEngineContext, } from "@/lib/medications/scheduling/cadence"; import { complianceChips } from "@/lib/medications/scheduling/compliance"; import { + buildComplianceMedicationContext, + COMPLIANCE_LEDGER_WINDOW_DAYS, + COURSES_COMPLIANCE_SELECT, lastNonSkippedTakenAt, SCHEDULE_COMPLIANCE_SELECT, } from "@/lib/analytics/compliance"; @@ -58,6 +60,11 @@ export const GET = apiHandler( schedules: { select: SCHEDULE_COMPLIANCE_SELECT }, // v1.16.3 — archived schedule eras for era-aware chips/tallies. scheduleRevisions: { orderBy: { validFrom: "asc" } }, + // The pause eras and courses, so the chips tally the ledger the + // detail page's rate reads: a paused day or a gap between two + // courses expects nothing. + pauseEras: { select: { pausedAt: true, resumedAt: true } }, + courses: COURSES_COMPLIANCE_SELECT, }, }); if (!med) { @@ -75,11 +82,13 @@ export const GET = apiHandler( const windowDays = parsed.data.days ?? 30; const asOf = new Date(); - // Pull intake events covering the window + a small buffer so a dose - // logged at the edge can still pair with its slot. - const bufferMs = 24 * 60 * 60 * 1000; + // The events the compliance ledger reads (its full trailing year plus a + // day): the chips tally the same rows as the detail page, and a rolling + // cadence re-anchors on the latest intake even when it predates the + // chart's window. const from = new Date( - asOf.getTime() - windowDays * 24 * 60 * 60 * 1000 - bufferMs, + asOf.getTime() - + (COMPLIANCE_LEDGER_WINDOW_DAYS + 1) * 24 * 60 * 60 * 1000, ); const events = await prisma.medicationIntakeEvent.findMany({ where: { @@ -93,6 +102,8 @@ export const GET = apiHandler( scheduledFor: true, takenAt: true, skipped: true, + // A dose the auto-miss job closed counts as missed, as in the rate. + autoMissed: true, // v1.15.20 — pinned takes bind by anchor in the chip tally so the // chips agree with the % + the history on a "zugeordnet" dose. attributionSource: true, @@ -121,16 +132,11 @@ export const GET = apiHandler( // legacy weekday walker (e.g. `FREQ=WEEKLY;BYDAY=MO` expands daily, // PRN counts as a real cadence) and disagrees with the dashboard. // Mirrors `analytics/compliance.ts`'s context construction. - const lastIntakeAt = lastNonSkippedTakenAt(events); - const engineCtx: CadenceEngineContext = { - startsOn: med.startsOn, - endsOn: med.endsOn, - oneShot: med.oneShot, - createdAt: med.createdAt, - lastIntakeAt, - timeZone: userTz, - scheduleRevisions: med.scheduleRevisions, - }; + const engineCtx = buildComplianceMedicationContext( + med, + lastNonSkippedTakenAt(events), + userTz, + ); // v1.39.1 (#1033) — intake tracking off: no chip, no next dose. const liveSchedules = dueSchedules(med); @@ -146,11 +152,10 @@ export const GET = apiHandler( const chips = complianceChips( liveSchedules, events, + engineCtx, asOf, windowDays, anchor, - userTz, - engineCtx, ); const next = computeNextDose( liveSchedules, diff --git a/src/lib/analytics/compliance/display.ts b/src/lib/analytics/compliance/display.ts index ac2f8cea2..7073eab46 100644 --- a/src/lib/analytics/compliance/display.ts +++ b/src/lib/analytics/compliance/display.ts @@ -29,9 +29,9 @@ import { } from "./windows"; import { COMPLIANCE_LEDGER_WINDOW_DAYS, - buildComplianceLedgerRows, tallyComplianceFromLedger, - tallyLedgerRows, + trailingComplianceLedger, + trailingComplianceTally, } from "./ledger"; import { buildCurrentCycle, type CurrentCycle } from "./cycle"; import { rollingIntakeInstants } from "./adapters"; @@ -564,9 +564,7 @@ export function buildMedicationComplianceBundle( : ledgerPeriodStart; const hasSchedules = schedules.length > 0; - const ledgerRows = hasSchedules - ? buildComplianceLedgerRows(events, schedules, ctx, ledgerFrom, now, now) - : []; + const ledgerRows = trailingComplianceLedger(events, schedules, ctx, now); // ONE cadence timeline over the full ledger window. Each per-window // streak below walks only its own trailing `effectiveDays`, so sharing @@ -607,10 +605,7 @@ export function buildMedicationComplianceBundle( 1, Math.ceil((now.getTime() - effectiveStart.getTime()) / ONE_DAY_MS), ); - const tally = tallyLedgerRows(ledgerRows, { - from: periodStart, - to: now, - }); + const tally = trailingComplianceTally(ledgerRows, days, now); const { current: streak } = streaksFromTimeline( timeline, now, diff --git a/src/lib/analytics/compliance/ledger.ts b/src/lib/analytics/compliance/ledger.ts index baffb1123..230b3dfb0 100644 --- a/src/lib/analytics/compliance/ledger.ts +++ b/src/lib/analytics/compliance/ledger.ts @@ -306,6 +306,41 @@ export function tallyLedgerRows( */ export const COMPLIANCE_LEDGER_WINDOW_DAYS = 365; +/** + * The ledger over the trailing {@link COMPLIANCE_LEDGER_WINDOW_DAYS} ending at + * `now`: the one set of dose rows every per-medication compliance figure is + * tallied from. A slot before the medication's creation stays in it only when + * a recorded dose claims it (#1028). + */ +export function trailingComplianceLedger( + events: IntakeEvent[], + schedules: ComplianceSchedule[], + ctx: ComplianceMedicationContext, + now: Date, +): DoseHistoryRow[] { + if (schedules.length === 0) return []; + const from = new Date( + now.getTime() - COMPLIANCE_LEDGER_WINDOW_DAYS * 24 * 60 * 60 * 1000, + ); + return buildComplianceLedgerRows(events, schedules, ctx, from, now, now); +} + +/** + * The tally of the trailing `days` ending at `now`, carved out of + * {@link trailingComplianceLedger}: the same rows, so the same window reads + * the same rate on every surface. + */ +export function trailingComplianceTally( + ledger: DoseHistoryRow[], + days: number, + now: Date, +): LedgerComplianceTally { + return tallyLedgerRows(ledger, { + from: new Date(now.getTime() - days * 24 * 60 * 60 * 1000), + to: now, + }); +} + /** * One day of the cadence-aware compliance series: the day's compliance % * over the doses the schedule actually expected that day, plus the raw diff --git a/src/lib/medications/scheduling/__tests__/compliance.test.ts b/src/lib/medications/scheduling/__tests__/compliance.test.ts index 6291df5ad..dae3cd298 100644 --- a/src/lib/medications/scheduling/__tests__/compliance.test.ts +++ b/src/lib/medications/scheduling/__tests__/compliance.test.ts @@ -8,16 +8,29 @@ import { describe, expect, it } from "vitest"; import { complianceChips, streaksFromTimeline } from "../compliance"; import { buildCadenceTimeline } from "../cadence"; -import type { - CadenceEngineContext, - IntakeEventLike, - ScheduleLike, -} from "../cadence"; +import type { IntakeEventLike, ScheduleLike } from "../cadence"; +import type { ComplianceMedicationContext } from "@/lib/analytics/compliance/types"; +import { buildMedicationComplianceBundle } from "@/lib/analytics/compliance/display"; function d(iso: string): Date { return new Date(iso); } +/** The medication context the chips read; UTC unless a test says otherwise. */ +function ctx( + over?: Partial, +): ComplianceMedicationContext { + return { + startsOn: null, + endsOn: null, + oneShot: false, + createdAt: d("2025-05-01T00:00:00Z"), + lastIntakeAt: null, + timeZone: "UTC", + ...over, + }; +} + const DAILY_8AM: ScheduleLike = { windowStart: "08:00", windowEnd: "09:00", @@ -27,7 +40,7 @@ const DAILY_8AM: ScheduleLike = { describe("complianceChips", () => { it("returns null adherence when no doses are expected in the window", () => { const NOW = d("2025-06-10T12:00:00"); - const result = complianceChips([], [], NOW, 30); + const result = complianceChips([], [], ctx(), NOW, 30); expect(result.adherenceRate).toBeNull(); expect(result.missedLast30).toBe(0); }); @@ -44,14 +57,14 @@ describe("complianceChips", () => { } // 5-day window: 5 expected, 5 taken, 0 missed (today's slot is upcoming // because NOW is 12:00 and window is 08:00-09:00... actually past today) - const result = complianceChips([DAILY_8AM], events, NOW, 5); + const result = complianceChips([DAILY_8AM], events, ctx(), NOW, 5); expect(result.adherenceRate).toBeGreaterThanOrEqual(80); expect(result.missedLast30).toBeLessThanOrEqual(1); }); it("computes 0% adherence when no events match any past slot", () => { const NOW = d("2025-06-10T12:00:00"); - const result = complianceChips([DAILY_8AM], [], NOW, 5); + const result = complianceChips([DAILY_8AM], [], ctx(), NOW, 5); expect(result.adherenceRate).toBe(0); expect(result.missedLast30).toBeGreaterThan(0); }); @@ -60,25 +73,25 @@ describe("complianceChips", () => { const NOW = d("2025-06-10T12:00:00"); // Window: NOW - 3d = 2025-06-07T12:00 → past slots: 08, 09, 10 // Provide one taken + one skipped + one taken for the three slots so - // the denominator excludes the skipped one. + // the denominator excludes the skipped one. A skip names its slot. const events: IntakeEventLike[] = [ { - scheduledFor: d("2025-06-08T08:30:00"), + scheduledFor: d("2025-06-08T08:00:00"), takenAt: d("2025-06-08T08:35:00"), skipped: false, }, { - scheduledFor: d("2025-06-09T08:30:00"), + scheduledFor: d("2025-06-09T08:00:00"), takenAt: null, skipped: true, }, { - scheduledFor: d("2025-06-10T08:30:00"), + scheduledFor: d("2025-06-10T08:00:00"), takenAt: d("2025-06-10T08:35:00"), skipped: false, }, ]; - const result = complianceChips([DAILY_8AM], events, NOW, 3); + const result = complianceChips([DAILY_8AM], events, ctx(), NOW, 3); // 2 taken / (2 taken + 0 missed) = 100, skipped excluded. expect(result.adherenceRate).toBe(100); }); @@ -97,7 +110,7 @@ describe("complianceChips", () => { skipped: false, }); } - const result = complianceChips([DAILY_8AM], events, NOW, 14); + const result = complianceChips([DAILY_8AM], events, ctx(), NOW, 14); expect(result.currentStreak).toBeGreaterThanOrEqual(4); }); @@ -132,7 +145,7 @@ describe("complianceChips", () => { skipped: false, }, ]; - const result = complianceChips([DAILY_8AM], events, NOW, 7); + const result = complianceChips([DAILY_8AM], events, ctx(), NOW, 7); // Streak ending at today = day 9 + day 10 = 2 (day 8 missed broke it). expect(result.currentStreak).toBeLessThanOrEqual(2); // Longest streak in window = max(3 from 5..7, 2 from 9..10) = 3. @@ -158,7 +171,7 @@ describe("complianceChips", () => { skipped: false, }, ]; - const result = complianceChips([weekly], events, NOW, 14); + const result = complianceChips([weekly], events, ctx(), NOW, 14); expect(result.adherenceRate).toBe(100); expect(result.missedLast30).toBe(0); }); @@ -170,7 +183,7 @@ describe("complianceChips", () => { daysOfWeek: "1", // Monday only }; const NOW = d("2025-06-10T12:00:00"); // Tuesday - const result = complianceChips([weekly], [], NOW, 14); + const result = complianceChips([weekly], [], ctx(), NOW, 14); expect(result.adherenceRate).toBe(0); expect(result.missedLast30).toBeGreaterThan(0); }); @@ -199,18 +212,16 @@ describe("complianceChips", () => { const berlin = complianceChips( [sched], events, + ctx({ timeZone: "Europe/Berlin" }), NOW, 7, - undefined, - "Europe/Berlin", ); const tokyo = complianceChips( [sched], events, + ctx({ timeZone: "Asia/Tokyo" }), NOW, 7, - undefined, - "Asia/Tokyo", ); // Both must compute a finite adherence rate (the schedule has @@ -226,23 +237,10 @@ describe("complianceChips", () => { }); describe("complianceChips — canonical-engine delegation (v1.7.0 SB-SCHED-2)", () => { - function engineCtx( - over?: Partial, - ): CadenceEngineContext { - return { - startsOn: null, - endsOn: null, - oneShot: false, - createdAt: d("2025-05-01T00:00:00Z"), - lastIntakeAt: null, - timeZone: "Europe/Berlin", - ...over, - }; - } + const engineCtx = (over?: Partial) => + ctx({ timeZone: "Europe/Berlin", ...over }); - it("RRULE weekly Monday: legacy walker counts every day missed, engine counts only Mondays", () => { - // `daysOfWeek = null` reads as every-day to the legacy walker, so an - // RRULE-weekly schedule over-counts expected doses without the engine. + it("RRULE weekly Monday: only the Mondays in the window are expected", () => { const rruleWeekly: ScheduleLike = { windowStart: "08:00", windowEnd: "09:00", @@ -250,25 +248,15 @@ describe("complianceChips — canonical-engine delegation (v1.7.0 SB-SCHED-2)", rrule: "FREQ=WEEKLY;BYDAY=MO", timesOfDay: ["08:00"], }; - const NOW = d("2025-06-10T12:00:00Z"); // Tuesday + // Saturday: both Mondays in the window are past a weekly dose's late tail. + const NOW = d("2025-06-14T12:00:00Z"); - // Legacy path (no engineCtx) expands daily → many missed days. - const legacy = complianceChips([rruleWeekly], [], NOW, 14); - // Canonical path (engineCtx) expands only Mondays in the window. - const engine = complianceChips( - [rruleWeekly], - [], - NOW, - 14, - undefined, - "Europe/Berlin", - engineCtx(), - ); + const engine = complianceChips([rruleWeekly], [], engineCtx(), NOW, 14); - // The engine path expects far fewer doses (only the Mondays in a - // 14-day window: 2) than the daily legacy expansion (~13). - expect(engine.missedLast30).toBeLessThan(legacy.missedLast30); - expect(engine.missedLast30).toBeLessThanOrEqual(2); + // Two Mondays in a 14-day window, both missed; `daysOfWeek = null` + // must not read as every day. + expect(engine.missedLast30).toBe(2); + expect(engine.adherenceRate).toBe(0); }); it("PRN: engine short-circuits to zero expected doses (no adherence, no missed)", () => { @@ -281,15 +269,7 @@ describe("complianceChips — canonical-engine delegation (v1.7.0 SB-SCHED-2)", }; const NOW = d("2025-06-10T12:00:00Z"); - const engine = complianceChips( - [prn], - [], - NOW, - 30, - undefined, - "Europe/Berlin", - engineCtx(), - ); + const engine = complianceChips([prn], [], engineCtx(), NOW, 30); // PRN is as-needed — never projected, reminded, or counted in // compliance-expected. No expected slots → null adherence, zero missed. @@ -315,20 +295,16 @@ describe("complianceChips — canonical-engine delegation (v1.7.0 SB-SCHED-2)", const engine = complianceChips( [cyclic], [], + engineCtx({ startsOn: d("2025-05-01T00:00:00Z") }), NOW, 14, - undefined, - "Europe/Berlin", - engineCtx({ startsOn: d("2025-05-01T00:00:00Z") }), ); const everyDay = complianceChips( [{ ...cyclic, scheduleType: "SCHEDULED" }], [], + engineCtx({ startsOn: d("2025-05-01T00:00:00Z") }), NOW, 14, - undefined, - "Europe/Berlin", - engineCtx({ startsOn: d("2025-05-01T00:00:00Z") }), ); // The cyclic gate must drop the off-week's doses, so fewer missed @@ -445,3 +421,43 @@ describe("streaksFromTimeline — DST-correct day walk", () => { expect(current).toBe(7); }); }); + +describe("complianceChips — the detail page's rate", () => { + it("drops a pause from the chips exactly as the compliance payload does", () => { + // Daily 08:00, paused from 10 days ago to 3 days ago, every other dose + // taken. The chips used to tally the paused week as seven missed doses + // (77 %) while the detail page, reading the compliance ledger, said 100. + const NOW = d("2025-06-30T12:00:00Z"); + const day = (n: number) => new Date(NOW.getTime() - n * 86_400_000); + const pausedAt = day(10); + const resumedAt = day(3); + const events: IntakeEventLike[] = []; + for (let n = 1; n <= 30; n++) { + const slot = new Date(day(n).setUTCHours(8, 0, 0, 0)); + if (slot >= pausedAt && slot < resumedAt) continue; + events.push({ scheduledFor: slot, takenAt: slot, skipped: false }); + } + const daily: ScheduleLike = { + windowStart: "08:00", + windowEnd: "09:00", + daysOfWeek: null, + timesOfDay: ["08:00"], + }; + const context = ctx({ + createdAt: d("2025-01-01T00:00:00Z"), + pauseEras: [{ pausedAt, resumedAt }], + }); + + const chips = complianceChips([daily], events, context, NOW, 30); + const bundle = buildMedicationComplianceBundle( + events, + [daily], + context, + NOW, + ); + + expect(chips.missedLast30).toBe(0); + expect(chips.adherenceRate).toBe(100); + expect(chips.adherenceRate).toBe(bundle.compliance30.rate); + }); +}); diff --git a/src/lib/medications/scheduling/compliance.ts b/src/lib/medications/scheduling/compliance.ts index 8a8157db6..2eff7aee1 100644 --- a/src/lib/medications/scheduling/compliance.ts +++ b/src/lib/medications/scheduling/compliance.ts @@ -6,33 +6,28 @@ * rate, current streak (days), longest streak (days), missed in last * 30 days. Each chip is monochrome — maintainer note: no gamified badges. * - * Distinct from `src/lib/analytics/compliance.ts.calculateCompliance`: - * that helper computes against expected counts (schedules-per-day × - * days) and is used by the existing /api/medications/[id]/compliance - * route. This module computes against the pair-matched timeline so - * the chips and the cadence chart agree on every single dose. + * Adherence and missed doses come from the compliance ledger the + * /api/medications/[id]/compliance route reads, so the chips and the + * detail page's rate agree; the streaks walk the pair-matched timeline + * the cadence chart draws. * - * v1.4.25 W21 Fix-O — accepts an optional IANA `timeZone` argument - * forwarded into the timeline + streak math so a user in Tokyo gets - * the same chip values as a user in Berlin even when the host's - * system time is set differently. Omitting the argument keeps the - * legacy system-local behaviour the W19e tests pin. + * The timeline and streak math run in the context's IANA zone, so a user + * in Tokyo gets the same chip values as a user in Berlin whatever the + * host's system time is. */ -import type { SlotBand } from "./attribution"; -import { type BandMinterMedication } from "./band-minter"; -import { buildBandsForSchedulesWithEras } from "./schedule-eras"; import { buildCadenceTimeline, startOfLocalDay, - type CadenceEngineContext, type IntakeEventLike, type PairedDose, type ScheduleLike, } from "./cadence"; -import { reconstructDoseHistory, type HistoryIntake } from "./dose-history"; -import { type CanonicalSchedule, type RecurrenceContext } from "./recurrence"; -import { normaliseDoseWindows } from "./worker-helpers"; +import { + trailingComplianceLedger, + trailingComplianceTally, +} from "@/lib/analytics/compliance/ledger"; +import type { ComplianceMedicationContext } from "@/lib/analytics/compliance/types"; import { userDayKey } from "@/lib/tz/resolver"; export interface ComplianceChips { @@ -146,134 +141,19 @@ export function streaksFromTimeline( } /** - * v1.15.18 — band-ledger tally counts for the detail-page chips. - * - * When the route threads an engine context (RRULE / rolling / cyclic / - * one-shot / PRN / legacy) the adherence rate + missed count come from the - * SAME unified dose-history ledger the compliance % and the history view - * read — so the chips can never contradict either. The streak still walks - * the cadence timeline (its day-grain "every dose taken or skipped" rule is - * orthogonal to per-dose attribution). Returns null when there is no engine - * context (a pure-math caller) — the caller then falls back to the legacy - * timeline tally. + * The four chips: adherence and missed doses from the compliance ledger the + * detail page's rate reads (`trailingComplianceTally`, with the medication's + * pause eras and courses), the streaks from the cadence timeline (a + * day-grain "every dose taken or skipped" rule, orthogonal to per-dose + * attribution). */ -function ledgerChipCounts( - schedules: ScheduleLike[], - events: IntakeEventLike[], - asOf: Date, - windowDays: number, - timeZone: string | undefined, - engineCtx: CadenceEngineContext | undefined, -): { taken: number; missed: number } | null { - if (!engineCtx) return null; - const DAY_MS = 24 * 60 * 60 * 1000; - const from = new Date(asOf.getTime() - windowDays * DAY_MS); - const userTz = engineCtx.timeZone || timeZone || "UTC"; - - const medication: BandMinterMedication = { - id: "chips-tally", - startsOn: engineCtx.startsOn, - endsOn: engineCtx.endsOn, - oneShot: engineCtx.oneShot, - createdAt: engineCtx.createdAt, - }; - const recurrenceCtx: RecurrenceContext = { - medication: { - id: "chips-tally", - startsOn: engineCtx.startsOn, - endsOn: engineCtx.endsOn, - oneShot: engineCtx.oneShot, - createdAt: engineCtx.createdAt, - }, - timeZone: userTz, - lastIntakeAt: engineCtx.lastIntakeAt, - // v1.40 (#1024) — a medication: `startsOn` is a course start. - rollingAnchor: "courseStart", - }; - const canonicalSchedules: CanonicalSchedule[] = schedules.map((s, i) => { - const base: CanonicalSchedule = { - id: s.id ?? `chips-${i}`, - rrule: s.rrule ?? null, - rollingIntervalDays: s.rollingIntervalDays ?? null, - timesOfDay: s.timesOfDay ?? [], - daysOfWeek: s.daysOfWeek ?? null, - windowStart: s.windowStart, - windowEnd: s.windowEnd, - reminderGraceMinutes: s.reminderGraceMinutes ?? null, - scheduleType: s.scheduleType ?? "SCHEDULED", - cyclicOnWeeks: s.cyclicOnWeeks ?? null, - cyclicOffWeeks: s.cyclicOffWeeks ?? null, - doseWindows: normaliseDoseWindows(s.doseWindows), - }; - // Surface windowStart as the single time-of-day for a legacy daily row - // so the band minter's cadence gate mints its daily band (see the - // analytics/compliance tally for the same normalisation). - if ( - base.timesOfDay.length === 0 && - base.rrule === null && - base.rollingIntervalDays === null && - base.scheduleType !== "PRN" && - !engineCtx.oneShot - ) { - return { ...base, timesOfDay: [base.windowStart] }; - } - return base; - }); - - const intakeInstants = events - .filter((e) => !e.skipped && e.takenAt !== null && e.takenAt <= asOf) - .map((e) => e.takenAt as Date) - .sort((a, b) => a.getTime() - b.getTime()); - - const groups = buildBandsForSchedulesWithEras({ - medication, - schedules: canonicalSchedules, - revisions: engineCtx.scheduleRevisions ?? [], - ctx: recurrenceCtx, - userTz, - range: { from, to: asOf }, - now: asOf, - intakeInstants, - }); - const bands: SlotBand[] = []; - for (const g of groups) { - if (g.hasExpectedSlots) bands.push(...g.bands); - } - - const intakes: HistoryIntake[] = events - .filter((e) => e.scheduledFor >= from && e.scheduledFor <= asOf) - .map((e) => ({ - scheduledFor: e.scheduledFor, - takenAt: e.takenAt, - skipped: e.skipped, - autoMissed: e.autoMissed ?? false, - // v1.15.20 — pinned takes bind by anchor (taken-late, slot served). - pinned: e.attributionSource === "USER_PIN", - })); - - const rows = reconstructDoseHistory( - bands, - intakes, - asOf, - engineCtx.createdAt, - ); - let taken = 0; - let missed = 0; - for (const row of rows) { - if (row.status === "taken_on_time" || row.status === "taken_late") taken++; - else if (row.status === "missed") missed++; - } - return { taken, missed }; -} - export function complianceChips( schedules: ScheduleLike[], events: IntakeEventLike[], + ctx: ComplianceMedicationContext, asOf: Date, windowDays = 30, anchor?: Date, - timeZone?: string, - engineCtx?: CadenceEngineContext, ): ComplianceChips { const timeline = buildCadenceTimeline( schedules, @@ -281,33 +161,21 @@ export function complianceChips( asOf, windowDays, anchor, - timeZone, - engineCtx, + ctx.timeZone, + ctx, ); - // v1.15.18 — adherence + missed come from the unified band ledger when an - // engine context is present (so the chips agree with the % and the history - // view); the legacy timeline tally is the fallback for pure-math callers. - const ledger = ledgerChipCounts( - schedules, - events, - asOf, + const { taken, missed } = trailingComplianceTally( + trailingComplianceLedger(events, schedules, ctx, asOf), windowDays, - timeZone, - engineCtx, + asOf, ); - const taken = ledger - ? ledger.taken - : timeline.filter((d) => d.status === "taken").length; - const missed = ledger - ? ledger.missed - : timeline.filter((d) => d.status === "missed").length; const denom = taken + missed; const adherenceRate = denom === 0 ? null : Math.round((taken / denom) * 100); const { current, longest } = streaksFromTimeline( timeline, asOf, windowDays, - timeZone, + ctx.timeZone, ); return { adherenceRate, From 5720b70e8e4f4b710fce76528061b255d4202ef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 05:33:21 +0200 Subject: [PATCH 11/11] test(guards): drop the removed chip ledger from the course-anchor builders The cadence chips no longer build a recurrence context of their own; they tally the compliance ledger, whose adapter is already on the list. --- src/__tests__/recurrence-course-anchor-guard.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/__tests__/recurrence-course-anchor-guard.test.ts b/src/__tests__/recurrence-course-anchor-guard.test.ts index b6c5c1e69..d41f7bb95 100644 --- a/src/__tests__/recurrence-course-anchor-guard.test.ts +++ b/src/__tests__/recurrence-course-anchor-guard.test.ts @@ -31,7 +31,6 @@ const ROOT = resolve(SRC, ".."); const MEDICATION_BUILDERS = [ "src/lib/analytics/compliance/adapters.ts", "src/lib/medications/scheduling/cadence.ts", - "src/lib/medications/scheduling/compliance.ts", "src/lib/medications/scheduling/worker-helpers.ts", ]; @@ -91,7 +90,7 @@ function builders(): Builder[] { describe("the rolling course anchor is opt-in", () => { it("finds the builders it judges (no vacuous pass)", () => { const found = builders(); - expect(found.length).toBeGreaterThanOrEqual(6); + expect(found.length).toBeGreaterThanOrEqual(5); // The matcher sees an annotated literal and a typed return. expect([ ...`const c: RecurrenceContext = { a: 1 };\nfunction f(): RecurrenceContext {\n return {};\n}`.matchAll(