diff --git a/src/__tests__/api/analytics/since-cap.test.ts b/src/__tests__/api/analytics/since-cap.test.ts index 221f1342e..1b3cfbf16 100644 --- a/src/__tests__/api/analytics/since-cap.test.ts +++ b/src/__tests__/api/analytics/since-cap.test.ts @@ -118,7 +118,7 @@ vi.mock("@/lib/analytics/correlations-fast-path", () => ({ })); vi.mock("@/lib/analytics/summaries-slice", () => ({ - computeSummariesSlice: vi.fn().mockResolvedValue({ + readSummariesSlice: vi.fn().mockResolvedValue({ summaries: {}, bmi: null, bpInTargetPct: null, @@ -176,7 +176,7 @@ beforeEach(async () => { await import("@/lib/analytics/score/reader"); const { computeCorrelationHypothesesFastPath } = await import("@/lib/analytics/correlations-fast-path"); - const { computeSummariesSlice } = + const { readSummariesSlice } = await import("@/lib/analytics/summaries-slice"); (ensureUserRollupsFresh as ReturnType).mockResolvedValue( @@ -196,7 +196,7 @@ beforeEach(async () => { ( computeCorrelationHypothesesFastPath as ReturnType ).mockResolvedValue({ hypotheses: [] }); - (computeSummariesSlice as ReturnType).mockResolvedValue({ + (readSummariesSlice as ReturnType).mockResolvedValue({ summaries: {}, bmi: null, lastSeenByType: {}, @@ -206,7 +206,7 @@ beforeEach(async () => { describe("GET /api/analytics — per-type live walk retired (v1.4.49.1)", () => { // v1.4.49.1 — the W-SINCE 425-day `since` cap was a defense-in-depth // bound on the now-deleted 15-way per-type live walk. The default - // slice routes through `computeSummariesSlice` exclusively, which + // slice routes through `readSummariesSlice` exclusively, which // reads `measurement_rollups` DAY buckets + a 90-day narrow // `$queryRaw` directly — no chunked findMany against `measurements` // at all on the default critical path. The two assertions that pinned @@ -222,7 +222,7 @@ describe("GET /api/analytics — per-type live walk retired (v1.4.49.1)", () => ); expect(response.status).toBe(200); - // The slim slice resolves through `computeSummariesSlice` only; + // The slim slice resolves through `readSummariesSlice` only; // the per-type findMany loop is skipped entirely so the rollup- // path semantics are unaffected by the `since` cap. const findMany = prisma.measurement.findMany as ReturnType; diff --git a/src/__tests__/correlation-single-implementation-guard.test.ts b/src/__tests__/correlation-single-implementation-guard.test.ts new file mode 100644 index 000000000..f9697ab84 --- /dev/null +++ b/src/__tests__/correlation-single-implementation-guard.test.ts @@ -0,0 +1,93 @@ +/** + * Structural guard: one Pearson correlation, one bar. + * + * Three implementations used to answer "how correlated are these two + * series": the engine's `pearson` (exact p-value, `lib/insights/ + * correlations.ts`), its gated wrapper `significantPearsonCorrelation` + * (n >= 20 and p < 0.05), and an ungated `pearsonCorrelation` that banded a + * coefficient from |r| alone above five pairs. The ungated one fed the + * /insights envelope and the AI feature payload, so five days of noise + * sloping the right way reached a reader labelled "strong" there while every + * other surface refused the same five points. It is gone; every surface goes + * through the wrapper, and the wrapper through the engine. + * + * The matcher flags a function whose name says Pearson or correlation + * coefficient, declared anywhere but the engine and the wrapper's file, and + * any mention of the deleted name. It is a tripwire: a coefficient computed + * inline under another name slips it. Comments are stripped first; the + * planted cases prove it fires. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +const HOMES = new Set([ + "lib/insights/correlations.ts", + "lib/analytics/correlations.ts", +]); + +const PEARSON_DEFINITION = + /(?:function\s+|const\s+)\w*(?:pearson|correlationCoefficient)\w*\s*(?:=|\()/i; +const DELETED = /(? + !p.startsWith("generated/") && + !p.includes("__tests__/") && + !/\.test\.tsx?$/.test(p), + ); +} + +describe("one Pearson correlation", () => { + it("the matchers fire on a second implementation", () => { + expect(PEARSON_DEFINITION.test("export function pearsonR(xs, ys) {")).toBe( + true, + ); + expect(PEARSON_DEFINITION.test("const quickPearson = (a, b) =>")).toBe( + true, + ); + expect(DELETED.test("pearsonCorrelation(pairs)")).toBe(true); + expect(DELETED.test("significantPearsonCorrelation(pairs)")).toBe(false); + }); + + it("both homes still hold their function", () => { + const engine = readFileSync( + join(SRC, "lib/insights/correlations.ts"), + "utf8", + ); + const wrapper = readFileSync( + join(SRC, "lib/analytics/correlations.ts"), + "utf8", + ); + expect(engine).toMatch(/export function pearson\s*\(/); + expect(wrapper).toMatch( + /export function significantPearsonCorrelation\s*\(/, + ); + }); + + it("no other file defines a Pearson coefficient or calls the deleted one", () => { + // The homes match the definition matcher, so a matcher gone blind fails here. + const definers = sourceFiles().filter((file) => + PEARSON_DEFINITION.test( + stripComments(readFileSync(join(SRC, file), "utf8")), + ), + ); + expect(definers.length).toBeGreaterThanOrEqual(2); + + const offenders = sourceFiles().filter((file) => { + const code = stripComments(readFileSync(join(SRC, file), "utf8")); + if (DELETED.test(code)) return true; + return !HOMES.has(file) && PEARSON_DEFINITION.test(code); + }); + expect(offenders).toEqual([]); + }); +}); diff --git a/src/__tests__/dashboard-summary-read-plan-guard.test.ts b/src/__tests__/dashboard-summary-read-plan-guard.test.ts index 44c9707d8..b1b9a14e1 100644 --- a/src/__tests__/dashboard-summary-read-plan-guard.test.ts +++ b/src/__tests__/dashboard-summary-read-plan-guard.test.ts @@ -97,10 +97,9 @@ describe("dashboard summary reads — the route keeps using them", () => { expect(route).toMatch( /readSparkBuckets\(\s*prisma,\s*userId,\s*measurementTypes,/, ); - // No second copy of the rollup read can drift out of the guarded - // module. (`FROM measurements` still appears in the route's own - // streak-day aggregate, which is a bounded 365-row read and not part - // of this finding.) + expect(route).toMatch(/readActivityDayKeys\(\s*prisma,\s*userId,/); + // No second copy of a read can drift out of the guarded module. expect(route.includes("FROM measurement_rollups")).toBe(false); + expect(route.includes("FROM measurements")).toBe(false); }); }); diff --git a/src/__tests__/dense-measurement-read-guard.test.ts b/src/__tests__/dense-measurement-read-guard.test.ts index eb46135e0..75c083e73 100644 --- a/src/__tests__/dense-measurement-read-guard.test.ts +++ b/src/__tests__/dense-measurement-read-guard.test.ts @@ -133,6 +133,8 @@ const ALLOWED_SQL: Record = { "Selects account ids; `measurements` is read inside `EXISTS`, one row per account at most.", "lib/rollups/measurement-coverage.ts::1": "Selects from `unnest` of the caller's type list; `measurements` is read inside `EXISTS`, one row per type at most.", + "lib/dashboard/summary-reads.ts::1": + "Selects from `generate_series` of the streak window's local days; `measurements` is read inside `EXISTS`, one row per day at most.", }; interface UntakenRead { diff --git a/src/__tests__/dose-ledger-single-source-guard.test.ts b/src/__tests__/dose-ledger-single-source-guard.test.ts new file mode 100644 index 000000000..aff5974b4 --- /dev/null +++ b/src/__tests__/dose-ledger-single-source-guard.test.ts @@ -0,0 +1,55 @@ +/** + * Structural guard: medication compliance is tallied from one ledger. + * + * The cadence chips carried a private copy of the ledger build that + * `lib/analytics/compliance/ledger.ts` owns, minus the pause eras and the + * courses, so `/api/medications/[id]/cadence` could report 77 % for a month + * the detail page called 100 %. The chips now tally the shared ledger. + * + * `reconstructDoseHistory` is the step that turns minted bands and intakes + * into dose rows. The set of files that call it is frozen: the ledger, the + * cadence timeline (the chart and the streaks) and the dose-history view. A + * new caller is a new ledger and has to be added here, next to a reason. + * Comments are stripped first; the count floor proves the matcher reads. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +const CALLERS: Record = { + "lib/analytics/compliance/ledger.ts": + "The compliance ledger every rate and the chips tally.", + "lib/medications/scheduling/cadence.ts": + "The cadence timeline the chart draws and the streaks walk.", + "app/api/medications/[id]/dose-history/route.ts": + "The dose-history view, row by row over the same bands.", +}; + +const CALL = /\breconstructDoseHistory\s*\(/; + +function stripComments(text: string): string { + return text.replace(/\/\*[\s\S]*?\*\//g, "").replace(/^\s*\/\/.*$/gm, ""); +} + +describe("one compliance ledger", () => { + it("the dose rows are built only where this file says", () => { + const callers = walkSourceFiles(SRC, { floor: 3000 }) + .filter( + (file) => + !file.startsWith("generated/") && + !file.includes("__tests__/") && + !/\.test\.tsx?$/.test(file) && + file !== "lib/medications/scheduling/dose-history.ts", + ) + .filter((file) => + CALL.test(stripComments(readFileSync(join(SRC, file), "utf8"))), + ) + .sort(); + expect(callers.length).toBeGreaterThanOrEqual(3); + expect(callers).toEqual(Object.keys(CALLERS).sort()); + }); +}); diff --git a/src/__tests__/latest-per-type-read-guard.test.ts b/src/__tests__/latest-per-type-read-guard.test.ts new file mode 100644 index 000000000..a0e7eb796 --- /dev/null +++ b/src/__tests__/latest-per-type-read-guard.test.ts @@ -0,0 +1,82 @@ +/** + * Structural guard: the latest reading per measurement type has two readers, + * and the summaries slice one cache cell. + * + * Postgres 16 has no skip scan, so `DISTINCT ON (type)` over `measurements` + * sorts every row it can see to keep one per type. Five copies of that shape + * sat in the dashboard, analytics, insights and targets reads; each one now + * goes through `readLatestCanonical` (the ladder-canonical reading of the + * latest day) or `readLatestEver` (the latest raw reading), both driven from + * the type list with one index probe per type. + * + * The summaries slice was computed inside three cells (the tile-strip route, + * the thick analytics body, the dashboard snapshot). `readSummariesSlice` is + * the one cell; `computeSummariesSlice` is its builder and is called nowhere + * else in production code. + * + * Comments are stripped before matching, and the planted cases prove each + * matcher finds what it claims to. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +function stripComments(text: string): string { + return text + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/^\s*\/\/.*$/gm, "") + .replace(/^\s*--.*$/gm, ""); +} + +/** `DISTINCT ON (type)` or `DISTINCT ON (m."type")`, the type alone. */ +const LATEST_PER_TYPE = /DISTINCT\s+ON\s*\(\s*(?:\w+\s*\.\s*)?"?type"?\s*\)/i; + +const SLICE_BUILDER_CALL = /\bcomputeSummariesSlice\s*\(/; + +function productionFiles(): string[] { + return walkSourceFiles(SRC, { floor: 3000 }).filter( + (file) => + !file.startsWith("generated/") && + !file.includes("__tests__/") && + !/\.test\.tsx?$/.test(file), + ); +} + +describe("latest-per-type reads", () => { + it("the matchers find their shapes", () => { + expect( + LATEST_PER_TYPE.test(`SELECT DISTINCT ON (m."type") m."value"`), + ).toBe(true); + expect(LATEST_PER_TYPE.test(`SELECT DISTINCT ON ("type") value`)).toBe( + true, + ); + expect( + LATEST_PER_TYPE.test(`SELECT DISTINCT ON ("type", "bucket_start")`), + ).toBe(false); + expect( + SLICE_BUILDER_CALL.test(`await computeSummariesSlice(user.id)`), + ).toBe(true); + }); + + it("no production read keeps one row per type with DISTINCT ON", () => { + const offenders = productionFiles().filter((file) => + LATEST_PER_TYPE.test( + stripComments(readFileSync(join(SRC, file), "utf8")), + ), + ); + expect(offenders).toEqual([]); + }); + + it("the summaries slice is built only behind its cache cell", () => { + const callers = productionFiles().filter((file) => + SLICE_BUILDER_CALL.test( + stripComments(readFileSync(join(SRC, file), "utf8")), + ), + ); + expect(callers).toEqual(["lib/analytics/summaries-slice.ts"]); + }); +}); diff --git a/src/__tests__/naive-timestamp-zone-shift-guard.test.ts b/src/__tests__/naive-timestamp-zone-shift-guard.test.ts new file mode 100644 index 000000000..cfb111b5a --- /dev/null +++ b/src/__tests__/naive-timestamp-zone-shift-guard.test.ts @@ -0,0 +1,71 @@ +/** + * Structural guard: a naive UTC timestamp is read as UTC before it is + * shifted into a person's zone. + * + * `measured_at`, `scheduled_for`, `taken_at` and the other `*_at` instants + * are `timestamp` columns holding UTC wall-clock time. On that type, + * `col AT TIME ZONE tz` means "this is local time in tz" and returns the + * matching instant, so the value moves by the offset in the wrong direction: + * in Berlin a 23:30 UTC reading lands on the day before instead of the day + * after. The right form is `(col AT TIME ZONE 'UTC') AT TIME ZONE tz`. The + * dashboard streak and the medication-compliance backfill both keyed days + * with the single form, while their comments called it identical to + * `userDayKey`. + * + * The matcher flags a quoted `*_at` or `scheduled_for` column shifted + * straight into a bound zone (`${...}` or `$N`). A `timestamptz` column + * (`bucket_start`) is out of its reach by name. Comments are stripped first; + * the planted cases prove it fires. It is a tripwire, not a proof. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +const SINGLE_SHIFT = + /"(?:[a-z_]+_at|scheduled_for)"\s*\)?\s*AT\s+TIME\s+ZONE\s+(?:\$\{|\$\d)/i; + +const DOUBLE_SHIFT = + /AT\s+TIME\s+ZONE\s+'UTC'\s*\)?\s*AT\s+TIME\s+ZONE\s+(?:\$\{|\$\d)/i; + +function stripComments(text: string): string { + return text + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/^\s*\/\/.*$/gm, "") + .replace(/^\s*--.*$/gm, ""); +} + +describe("naive timestamp zone shifts", () => { + it("the matcher fires on the single shift and spares the double one", () => { + expect( + SINGLE_SHIFT.test(`to_char(m."measured_at" AT TIME ZONE \${tz}, 'D')`), + ).toBe(true); + expect(SINGLE_SHIFT.test(`("scheduled_for" AT TIME ZONE $3)`)).toBe(true); + expect( + SINGLE_SHIFT.test( + `("scheduled_for" AT TIME ZONE 'UTC') AT TIME ZONE \${tz}`, + ), + ).toBe(false); + expect(SINGLE_SHIFT.test(`r."bucket_start" AT TIME ZONE \${tz}`)).toBe( + false, + ); + }); + + it("no production read shifts a naive column once", () => { + const files = walkSourceFiles(SRC, { floor: 3000 }).filter( + (file) => + !file.startsWith("generated/") && + !file.includes("__tests__/") && + !/\.test\.tsx?$/.test(file), + ); + const code = (file: string) => + stripComments(readFileSync(join(SRC, file), "utf8")); + // The reads that shift correctly, so a sweep that reads nothing fails. + const doubleShifts = files.filter((file) => DOUBLE_SHIFT.test(code(file))); + expect(doubleShifts.length).toBeGreaterThanOrEqual(5); + expect(files.filter((file) => SINGLE_SHIFT.test(code(file)))).toEqual([]); + }); +}); diff --git a/src/__tests__/recurrence-course-anchor-guard.test.ts b/src/__tests__/recurrence-course-anchor-guard.test.ts index b6c5c1e69..d41f7bb95 100644 --- a/src/__tests__/recurrence-course-anchor-guard.test.ts +++ b/src/__tests__/recurrence-course-anchor-guard.test.ts @@ -31,7 +31,6 @@ const ROOT = resolve(SRC, ".."); const MEDICATION_BUILDERS = [ "src/lib/analytics/compliance/adapters.ts", "src/lib/medications/scheduling/cadence.ts", - "src/lib/medications/scheduling/compliance.ts", "src/lib/medications/scheduling/worker-helpers.ts", ]; @@ -91,7 +90,7 @@ function builders(): Builder[] { describe("the rolling course anchor is opt-in", () => { it("finds the builders it judges (no vacuous pass)", () => { const found = builders(); - expect(found.length).toBeGreaterThanOrEqual(6); + expect(found.length).toBeGreaterThanOrEqual(5); // The matcher sees an annotated literal and a typed return. expect([ ...`const c: RecurrenceContext = { a: 1 };\nfunction f(): RecurrenceContext {\n return {};\n}`.matchAll( diff --git a/src/__tests__/spread-single-implementation-guard.test.ts b/src/__tests__/spread-single-implementation-guard.test.ts new file mode 100644 index 000000000..46b5dfa5a --- /dev/null +++ b/src/__tests__/spread-single-implementation-guard.test.ts @@ -0,0 +1,85 @@ +/** + * Structural guard: a standard deviation is computed in `lib/stats/spread`, + * with its kind named at the call site. + * + * About a dozen hand-rolled spreads each picked population (÷ n) or sample + * (÷ n − 1) without saying so, and the two surfaces a person compares picked + * differently: the series API reported the population SD of a blood-pressure + * series, the MCP and AI payload the sample SD of the same readings, about + * 22 % apart at three readings. The reading spread both report is now + * `READING_SPREAD`. + * + * The matcher flags a squared deviation from a mean in a file that also takes + * a square root. The files below compute one for a test statistic or a fit, + * never as a reported spread, and say so. The doctor report's glucose panel + * composes its SD from per-bucket sums of squares (sample, the same rule as + * `glucoseSD`) and is out of the matcher's reach by shape. Comments are + * stripped first; the planted case proves the matcher fires. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +const ALLOWED: Record = { + "lib/stats/spread.ts": "The implementation.", + "lib/insights/correlations.ts": + "The correlation engine's centred sums and the Welch t-test variances.", + "lib/insights/mood-tag-influence.ts": "Welch t-test variances.", + "lib/insights/mood-pattern-cards.ts": "A pooled variance for a t statistic.", + "lib/insights/derived/trajectory.ts": + "The residual standard error of a regression fit (÷ n − 2).", +}; + +const SQUARED_DEVIATION = + /\(\s*[\w.[\]]+\s*-\s*[\w.[\]]+\s*\)\s*\*\*\s*2|\(\s*([\w.[\]]+)\s*-\s*([\w.[\]]+)\s*\)\s*\*\s*\(\s*\1\s*-\s*\2\s*\)/; + +function stripComments(text: string): string { + return text.replace(/\/\*[\s\S]*?\*\//g, "").replace(/^\s*\/\/.*$/gm, ""); +} + +function computesSpread(code: string): boolean { + return SQUARED_DEVIATION.test(code) && code.includes("Math.sqrt("); +} + +function read(file: string): string { + return stripComments(readFileSync(join(SRC, file), "utf8")); +} + +describe("one standard deviation", () => { + it("the matcher fires on a hand-rolled spread", () => { + expect( + computesSpread( + "const v = xs.reduce((s, x) => s + (x - m) ** 2, 0) / n; Math.sqrt(v);", + ), + ).toBe(true); + expect( + computesSpread("acc += (v - mean) * (v - mean); Math.sqrt(acc / n);"), + ).toBe(true); + }); + + it("no file outside the list computes its own", () => { + const offenders = walkSourceFiles(SRC, { floor: 3000 }) + .filter( + (file) => + !file.startsWith("generated/") && + !file.includes("__tests__/") && + !/\.test\.tsx?$/.test(file), + ) + .filter((file) => computesSpread(read(file)) && !(file in ALLOWED)); + expect(offenders).toEqual([]); + }); + + it("the series API and the AI and MCP payload report the same spread", () => { + expect(read("app/api/measurements/series/route.ts")).toMatch( + /standardDeviation\(\s*values,\s*READING_SPREAD\s*\)/, + ); + const features = read("lib/insights/features.ts"); + expect( + features.match(/roundedSpread\(\s*vals,\s*READING_SPREAD\s*\)/g), + ).toHaveLength(2); + }); +}); diff --git a/src/__tests__/unguarded-correlation-guard.test.ts b/src/__tests__/unguarded-correlation-guard.test.ts deleted file mode 100644 index 96f0993d4..000000000 --- a/src/__tests__/unguarded-correlation-guard.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Structural guard on the unguarded Pearson helper. - * - * `pearsonCorrelation` bands a coefficient from |r| alone above a floor of - * five pairs, with no significance test. Its guarded twin - * `significantPearsonCorrelation` sits directly beneath it in the same file - * and exists because of exactly one failure: five days of noise sloping the - * right way reaching a reader labelled "strong". The comment above the twin - * says so in as many words. - * - * The twin was written once and adopted one surface at a time, which is how - * the mood page kept the naive call for four minor lines after the surfaces - * either side of it had moved. Nothing failed when it was skipped, because - * nothing was watching. This is what watches: a new caller of the unguarded - * helper has to be added to the list below, in a diff a reviewer reads, next - * to a written reason. - * - * It is a tripwire, not a proof. It cannot say the remaining entries are - * right — only that the set has not grown without someone editing this file. - * The two entries it carries today are named as open work, not as blessings. - */ -import { describe, it, expect } from "vitest"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - -import { walkSourceFiles } from "./helpers/source-files"; - -const SRC = join(process.cwd(), "src"); - -/** - * Files that may still call `pearsonCorrelation` directly, and why. - * - * Neither entry is settled. Both are the same migration the user-facing - * surfaces already made, still owed by two payload builders whose consumers - * pin the n >= 5 shape; moving them changes what those payloads contain and - * belongs in its own change with its own tests, not smuggled into this one. - */ -const ALLOWED: Record = { - "app/api/insights/comprehensive/route.ts": - "Five correlations on the /insights envelope (weight x BP, mood x BP, " + - "mood x weight, mood x pulse, BP-medication continuity). Open: they " + - "carry the same small-n exposure the mood page had, and the page's " + - "cards read `n` straight off the envelope.", - "lib/insights/features.ts": - "Eight cross-metric coefficients on the AI feature payload. Open: the " + - "prompt reads them as findings, so the same significance bar applies; " + - "the snapshot shape and its budget tests move with it.", -}; - -/** Every non-test, non-generated source file under `src/`. */ -function sourceFiles(): string[] { - return walkSourceFiles(SRC, { floor: 3000 }) - .filter((p) => !p.startsWith("generated/")) - .filter((p) => !p.includes("__tests__")) - .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx")) - .sort(); -} - -function read(rel: string): string { - return readFileSync(join(SRC, rel), "utf8"); -} - -/** - * A call to the unguarded helper. Whitespace-tolerant and anchored on a word - * boundary so `significantPearsonCorrelation(` cannot match: a matcher that - * quietly matches nothing is the failure mode this repository has shipped - * before, so the count is asserted below as well. - */ -const CALL = /(? { - it("finds callers at all, so an empty result cannot read as a pass", () => { - const definition = read("lib/analytics/correlations.ts"); - expect(definition).toMatch(/export function pearsonCorrelation\s*\(/); - expect( - definition, - "the guarded twin must exist for this guard to mean anything", - ).toMatch(/export function significantPearsonCorrelation\s*\(/); - - const callers = sourceFiles().filter( - (rel) => rel !== "lib/analytics/correlations.ts" && CALL.test(read(rel)), - ); - expect(callers.length).toBeGreaterThan(0); - }); - - it("names every caller outside the allowlist", () => { - const callers = sourceFiles().filter( - (rel) => rel !== "lib/analytics/correlations.ts" && CALL.test(read(rel)), - ); - - expect( - callers.filter((rel) => !(rel in ALLOWED)), - "A new caller of the unguarded `pearsonCorrelation`. A surface a " + - "person reads must use `significantPearsonCorrelation` (n >= 20 " + - "AND p < 0.05). If this one genuinely cannot, add it to ALLOWED " + - "in this file with the reason, so the exception is reviewed rather " + - "than assumed.", - ).toEqual([]); - }); - - it("keeps the allowlist honest by refusing stale entries", () => { - const callers = new Set( - sourceFiles().filter( - (rel) => - rel !== "lib/analytics/correlations.ts" && CALL.test(read(rel)), - ), - ); - - // An entry whose file no longer calls the helper is a licence nobody - // needs any more; leaving it standing is how the next one gets waved in. - expect( - Object.keys(ALLOWED).filter((rel) => !callers.has(rel)), - "this file is on the allowlist but no longer calls the unguarded " + - "helper — drop the entry", - ).toEqual([]); - }); -}); diff --git a/src/__tests__/workout-canonical-identity-guard.test.ts b/src/__tests__/workout-canonical-identity-guard.test.ts index 73fb493fb..416faa7e5 100644 --- a/src/__tests__/workout-canonical-identity-guard.test.ts +++ b/src/__tests__/workout-canonical-identity-guard.test.ts @@ -31,7 +31,7 @@ const SRC = join(process.cwd(), "src"); const PICKER = "lib/measurements/pick-canonical-workout-rows.ts"; /** - * Every production call site of the read-time picker, frozen. + * Every production call site of the picker, frozen. * * Adding a surface that collapses workouts is a deliberate act: it must fetch * the identity fields or it silently disagrees with every other surface. Adding @@ -48,6 +48,18 @@ const IDENTITY_OWNER: Record = { // block's data. Recording the owner here rather than assuming the call site // owns its own select is the whole point of this map. "lib/ai/coach/snapshot-blocks/workouts-block.ts": "lib/ai/coach/snapshot.ts", + // The write path: the batch route runs the same picker over the rows it is + // about to insert, so a duplicate inside one batch is dropped by the rule + // the list later applies. + "app/api/workouts/batch/route.ts": "app/api/workouts/batch/route.ts", +}; + +/** + * How a call site proves its rows carry the duration. A read selects the + * column; the batch route maps it off the row it computed for the insert. + */ +const IDENTITY_PROOF: Record = { + "app/api/workouts/batch/route.ts": "durationSec: p.row.durationSec", }; const EXPECTED_CALL_SITES = Object.keys(IDENTITY_OWNER).sort(); @@ -65,10 +77,8 @@ function read(rel: string): string { } /** - * Lines that actually invoke the picker. Comment lines are excluded because the - * write-time sibling (`lib/workouts/canonical-rows.ts`) names the read-time - * picker in its docblock to explain how the two layer, and that mention is - * documentation rather than a call. + * Lines that actually invoke the picker. Comment lines are excluded: a + * docblock that names the picker is documentation rather than a call. */ function invokesPicker(source: string): boolean { return source @@ -104,7 +114,9 @@ describe("every call site fetches the session identity", () => { // rows came from. Coarse in the safe direction — it can only fail to catch // a second unrelated select in the same file, never pass a caller that // fetches no duration at all. - expect(read(IDENTITY_OWNER[rel])).toContain("durationSec: true"); + expect(read(IDENTITY_OWNER[rel])).toContain( + IDENTITY_PROOF[rel] ?? "durationSec: true", + ); }); }); diff --git a/src/app/api/analytics/route.ts b/src/app/api/analytics/route.ts index de8e909bd..6c5f611de 100644 --- a/src/app/api/analytics/route.ts +++ b/src/app/api/analytics/route.ts @@ -6,7 +6,7 @@ import { analyticsQuerySchema } from "@/lib/validations/analytics"; import { NO_STORE_BUT_BFCACHE } from "@/lib/http/cache-headers"; import { cachedSwr, caches, type ServerCache } from "@/lib/cache/server-cache"; import { summarize, type DataPoint } from "@/lib/analytics/trends"; -import { computeSummariesSlice } from "@/lib/analytics/summaries-slice"; +import { readSummariesSlice } from "@/lib/analytics/summaries-slice"; import { resolveEffectiveBpTargets } from "@/lib/analytics/effective-range"; import { bpTargetsEqual, getBpTargets } from "@/lib/analytics/bp-targets"; import { DEFAULT_TIMEZONE } from "@/lib/tz/resolver"; @@ -103,14 +103,7 @@ export const GET = apiHandler(async (request?: Request) => { // tile strip's next mount should repaint from the prior summaries // instantly while one background recompute refreshes them, instead // of paying the 2-SQL-pass rebuild inline. - const slim = await cachedSwr( - caches.analytics as ServerCache< - Awaited> - >, - `${user.id}|summaries`, - () => computeSummariesSlice(user.id), - annotate, - ); + const slim = await readSummariesSlice(user.id, { serveStale: true }); // Drop the types whose module the user turned off, from the SAME map // the snapshot builder gates on (`SUMMARY_TYPE_MODULE`, via // `gateSummariesByModules`). With `NEXT_PUBLIC_DASHBOARD_SNAPSHOT=false` @@ -282,7 +275,7 @@ async function buildAnalyticsResponse(user: AuthedUser, locale: Locale) { // release; previously only the live walk produced it) // - `avg30LastYear` populated for any type whose WMY tier carries // the year-ago window - // - `lastSeenByType` from the `DISTINCT ON (type)` latest read + // - `lastSeenByType` from the per-type latest read // // `anomalyCount` and `slope90` left this wire in v1.37.19: the slim // path could never compute the former (it always reported 0) and no @@ -292,7 +285,7 @@ async function buildAnalyticsResponse(user: AuthedUser, locale: Locale) { // `avg30LastYear` on types with no WMY coverage, which matches the // pre-fix behaviour for tenants whose year-ago window happened to // fall outside the 425-day floor. - const slim = await computeSummariesSlice(user.id); + const slim = await readSummariesSlice(user.id); // Same module gate as the slim slice and the snapshot builder — the // default slice feeds the dashboard too (it is the thick half of the // legacy two-cell path), so it cannot be the one feed that leaks a diff --git a/src/app/api/dashboard/summary/route.ts b/src/app/api/dashboard/summary/route.ts index 55a3d77c3..5fe712a16 100644 --- a/src/app/api/dashboard/summary/route.ts +++ b/src/app/api/dashboard/summary/route.ts @@ -78,6 +78,7 @@ import { cachedSwr, caches, type ServerCache } from "@/lib/cache/server-cache"; import { buildMedsTodayBlock } from "@/lib/dashboard/meds-today"; import { computeStreak } from "@/lib/dashboard/streak"; import { + readActivityDayKeys, readLatestEver, readSparkBuckets, } from "@/lib/dashboard/summary-reads"; @@ -329,15 +330,6 @@ function trendOf(values: number[]): MetricCard["trend"] { return delta > 0 ? "up" : "down"; } -/** v1.4.38 W-F — distinct activity day-keys from the streak window. - * The route only needs YYYY-MM-DD strings, so the aggregate runs - * `date_trunc('day', m.measured_at AT TIME ZONE $userTz)` and returns - * the day-keys directly. At most 365 rows; vs. the legacy `findMany` - * which could return tens of thousands. */ -interface ActivityDayRow { - day_key: string; -} - export const GET = apiHandler(async () => { // v1.37.0 — MANAGE-level read: computed over the whole record, with no // provider anywhere on the path. @@ -532,27 +524,8 @@ async function buildDashboardSummary( select: { takenAt: true, scheduledFor: true }, }), ), - // v1.4.38 W-F — replaces the legacy 365-day `measurement.findMany` - // that pulled every raw row in the window only to collapse them to - // day-keys in JS. The SQL `date_trunc` returns at most 365 rows - // even on accounts with millions of step samples. The `userTz` - // conversion is folded server-side so the day boundary matches - // `userDayKey`'s output (it formats the timestamp in the user's - // tz before slicing YYYY-MM-DD). - time( - "streakDays", - () => - prisma.$queryRaw` - SELECT DISTINCT - to_char( - (m."measured_at" AT TIME ZONE ${userTz}), - 'YYYY-MM-DD' - ) AS day_key - FROM measurements m - WHERE m."user_id" = ${userId} - AND m."measured_at" >= ${streakWindowStart} - AND m."deleted_at" IS NULL - `, + time("streakDays", () => + readActivityDayKeys(prisma, userId, userTz, streakWindowStart, now), ), // v1.11.4 — raw per-stage SLEEP_DURATION rows for the night-total // tile. `SLEEP_DURATION` is stored one row per STAGE per night @@ -635,13 +608,9 @@ async function buildDashboardSummary( } // v1.4.38 W-F — activity-day set assembled from the bounded streak - // queries. The measurement side already arrives as YYYY-MM-DD keys - // from the SQL `date_trunc + AT TIME ZONE` aggregate, byte-identical - // to what `userDayKey(measuredAt, userTz)` would produce in JS. - const activityDays = new Set(); - for (const row of measurementStreakDays) { - if (row.day_key) activityDays.add(row.day_key); - } + // queries. The measurement side arrives as local YYYY-MM-DD keys, the + // same keys `userDayKey(measuredAt, userTz)` produces for the intakes. + const activityDays = new Set(measurementStreakDays); for (const e of streakActivity) { activityDays.add(userDayKey(e.takenAt ?? e.scheduledFor, userTz)); } diff --git a/src/app/api/insights/comprehensive/__tests__/route.test.ts b/src/app/api/insights/comprehensive/__tests__/route.test.ts index 2e9758306..2f1a185ab 100644 --- a/src/app/api/insights/comprehensive/__tests__/route.test.ts +++ b/src/app/api/insights/comprehensive/__tests__/route.test.ts @@ -569,16 +569,10 @@ describe("GET /api/insights/comprehensive — envelope shape", () => { expect(body.data.moodSummary?.count).toBe(2); }); - it("derives weight × BP scatter from daily-bucketed series", async () => { - vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); - // Five day-aligned pairs (n >= 5 unlocks pearsonCorrelation). - const days = [ - "2026-05-01", - "2026-05-02", - "2026-05-03", - "2026-05-04", - "2026-05-05", - ]; + function mockWeightBpDays(count: number) { + const days = Array.from({ length: count }, (_, i) => + new Date(Date.UTC(2026, 4, 1 + i)).toISOString().slice(0, 10), + ); (buildComprehensiveAggregate as ReturnType).mockResolvedValue( { summaries: {}, @@ -592,22 +586,41 @@ describe("GET /api/insights/comprehensive — envelope shape", () => { })), }, firstMeasurementAt: new Date(), - totalMeasurements: 10, + totalMeasurements: count * 2, }, ); + } + + type WeightBpBody = { + data: { + scatterData: Array<{ weight: number; sysBP: number }>; + weightBpCorrelation: { r: number; n: number } | null; + }; + }; + + it("derives weight × BP scatter from daily-bucketed series", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + mockWeightBpDays(25); const res = await callGet(makeReq()); - const body = (await res.json()) as { - data: { - scatterData: Array<{ weight: number; sysBP: number }>; - weightBpCorrelation: { r: number; n: number } | null; - }; - }; - expect(body.data.scatterData).toHaveLength(5); + const body = (await res.json()) as WeightBpBody; + expect(body.data.scatterData).toHaveLength(25); expect(body.data.scatterData[0]).toEqual({ weight: 80, sysBP: 120 }); - // Perfect linear relationship → r === 1. + // Perfect linear relationship over 25 days → r === 1. expect(body.data.weightBpCorrelation?.r).toBe(1); - expect(body.data.weightBpCorrelation?.n).toBe(5); + expect(body.data.weightBpCorrelation?.n).toBe(25); + }); + + it("reports no correlation below the significance bar every surface uses", async () => { + // Five perfectly aligned days: the scatter shows them, but five points + // cannot carry a correlation (n >= 20 and p < 0.05). + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + mockWeightBpDays(5); + + const res = await callGet(makeReq()); + const body = (await res.json()) as WeightBpBody; + expect(body.data.scatterData).toHaveLength(5); + expect(body.data.weightBpCorrelation).toBeNull(); }); }); @@ -849,13 +862,10 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing ( // tier in R12. Both read the UTC `dailyByType` / UTC `scheduledFor` // slice, so their output must be invariant to the caller's timezone. // Run the SAME fixture under UTC then America/New_York and deep-equal. - const days = [ - "2026-05-01", - "2026-05-02", - "2026-05-03", - "2026-05-04", - "2026-05-05", - ]; + // 25 days: a correlation has to clear n >= 20 and p < 0.05 to exist. + const days = Array.from({ length: 25 }, (_, i) => + new Date(Date.UTC(2026, 4, 1 + i)).toISOString().slice(0, 10), + ); const aggregate = { summaries: {}, bpRawRows: { sys: [], dia: [] }, @@ -896,20 +906,20 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing ( scheduleRevisions: [], pauseEras: [], }; - // Intake taken on the first three days, logged as skipped on the last - // two → a non-degenerate continuity series. The skips used to be absent + // Intake taken on the first fifteen days, logged as skipped on the + // last ten → a non-degenerate continuity series. The skips used to be absent // rows: a day with no record read as a measured 0 % continuity, which is // the fabrication the route no longer makes, and a fixture that relies on // it would be pinning the defect rather than the timezone question this - // test asks. Recording the two days as skips keeps five paired days AND + // test asks. Recording the days as skips keeps every day paired AND // keeps every one of them a thing the person actually logged. const intakeEvents = days.map((d, i) => ({ medicationId: "med-bp", userId: "user-comp-1", deletedAt: null, scheduledFor: new Date(`${d}T07:00:00Z`), - takenAt: i < 3 ? new Date(`${d}T07:05:00Z`) : null, - skipped: i >= 3, + takenAt: i < 15 ? new Date(`${d}T07:05:00Z`) : null, + skipped: i >= 15, })); function primeMocks() { @@ -948,7 +958,8 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing ( ); // And the guard is meaningful only if these joins actually produced // output to compare. - expect(utcBody.data.scatterData).toHaveLength(5); + expect(utcBody.data.scatterData).toHaveLength(25); + expect(utcBody.data.weightBpCorrelation).not.toBeNull(); expect(utcBody.data.bpMedicationCorrelation).not.toBeNull(); }); }); diff --git a/src/app/api/insights/comprehensive/route.ts b/src/app/api/insights/comprehensive/route.ts index f15b92687..6d3adac3a 100644 --- a/src/app/api/insights/comprehensive/route.ts +++ b/src/app/api/insights/comprehensive/route.ts @@ -12,7 +12,7 @@ import { generateAlerts, } from "@/lib/analytics/classifications"; import { - pearsonCorrelation, + significantPearsonCorrelation, type PairedPoint, } from "@/lib/analytics/correlations"; import { @@ -285,7 +285,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) { const weightDaily = aggregate.dailyByType.WEIGHT ?? []; const sysDaily = aggregate.dailyByType.BLOOD_PRESSURE_SYS ?? []; const weightBpPairs: PairedPoint[] = joinDailyByDay(weightDaily, sysDaily); - const weightBpCorrelation = pearsonCorrelation(weightBpPairs); + const weightBpCorrelation = significantPearsonCorrelation(weightBpPairs); const scatterData = weightBpPairs.map((p) => ({ weight: p.a, sysBP: p.b, @@ -308,11 +308,11 @@ export async function buildComprehensiveResponse(user: AuthedUser) { ); const moodBpPairs = pairMoodWithDaily(dailyMoodEntries, sysDailyLocal); - const moodBpCorrelation = pearsonCorrelation(moodBpPairs); + const moodBpCorrelation = significantPearsonCorrelation(moodBpPairs); const moodBpScatterData = moodBpPairs.map((p) => ({ mood: p.a, sysBP: p.b })); const moodWeightPairs = pairMoodWithDaily(dailyMoodEntries, weightDailyLocal); - const moodWeightCorrelation = pearsonCorrelation(moodWeightPairs); + const moodWeightCorrelation = significantPearsonCorrelation(moodWeightPairs); const moodWeightScatterData = moodWeightPairs.map((p) => ({ mood: p.a, weight: p.b, @@ -374,7 +374,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) { .sort(([a], [b]) => a.localeCompare(b)) .map(([day, stats]) => ({ day, value: stats.sum / stats.count })); const moodPulsePairs = pairMoodWithDaily(dailyMoodEntries, restingPulseDaily); - const moodPulseCorrelation = pearsonCorrelation(moodPulsePairs); + const moodPulseCorrelation = significantPearsonCorrelation(moodPulsePairs); const moodPulseScatterData = moodPulsePairs.map((p) => ({ mood: p.a, pulse: p.b, @@ -558,7 +558,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) { }); } - const corr = pearsonCorrelation(pairs); + const corr = significantPearsonCorrelation(pairs); if (corr) { bpMedicationCorrelation = { ...corr, @@ -667,7 +667,7 @@ function deriveDailyLocalMeans( /** * Daily-key inner join between two daily-aggregated series. Returns - * `PairedPoint` shapes compatible with `pearsonCorrelation`. + * `PairedPoint` shapes compatible with `significantPearsonCorrelation`. * * Inputs are sorted ASC by `day` (the aggregator's ORDER BY * guarantees that). The walk is O(n + m) merge-join style instead of diff --git a/src/app/api/insights/targets/__tests__/route.test.ts b/src/app/api/insights/targets/__tests__/route.test.ts index 9338f928d..d83596dc3 100644 --- a/src/app/api/insights/targets/__tests__/route.test.ts +++ b/src/app/api/insights/targets/__tests__/route.test.ts @@ -170,9 +170,9 @@ beforeEach(() => { ( prisma.moodEntryRollup.findMany as ReturnType ).mockResolvedValue([]); - // v1.28.25 — the latest-ever-per-type read is a raw `DISTINCT ON` - // (Prisma's `distinct` dedups client-side after pulling every row). (prisma.$queryRawUnsafe as ReturnType).mockResolvedValue([]); + // The latest-ever-per-type read (`readLatestEver`). + (prisma.$queryRaw as ReturnType).mockResolvedValue([]); vi.mocked(readGlucoseTargetSummaries).mockResolvedValue([]); vi.mocked(readRestingPulseProxy).mockResolvedValue([]); }); @@ -321,35 +321,30 @@ describe("GET /api/insights/targets — mood-rollup tier swap", () => { expect(moodTarget).toBeUndefined(); }); - it("reads latest-ever-per-type via raw DISTINCT ON with the one-year floor", async () => { - // Audit High finding 3 established the 365-day floor; v1.28.25 moves - // the read onto a real Postgres `DISTINCT ON` (Prisma's `distinct` - // dedups in the driver AFTER pulling every row in the window, which - // on a dense-type tenant shipped a year of raw rows to Node just to - // keep seven). Pin the raw query: DISTINCT ON, per-type descending - // walk, user id + floored date as bound parameters. + it("reads latest-ever-per-type one index probe per type, floored at one year", async () => { + // Audit High finding 3 established the 365-day floor. The read is the + // shared `readLatestEver`: driven from the type list, one LIMIT 1 probe + // per type, so a dense type never sorts a year of raw rows to keep one. await callGet(makeReq()); - const rawCalls = (prisma.$queryRawUnsafe as ReturnType).mock - .calls; - const distinctCall = rawCalls.find( - (c) => typeof c[0] === "string" && c[0].includes("DISTINCT ON"), + const rawCalls = (prisma.$queryRaw as ReturnType).mock.calls; + const latestCall = rawCalls.find( + (c) => + Array.isArray(c[0]) && + (c[0] as string[]).join("?").includes("CROSS JOIN LATERAL"), ); - expect(distinctCall).toBeDefined(); - const [sql, boundUserId, flooredAt] = distinctCall as [ - string, - string, - Date, - ]; - expect(sql).toContain(`DISTINCT ON (m."type")`); - expect(sql).toContain(`ORDER BY m."type" ASC, m."measured_at" DESC`); - // Parameter-bound, never spliced: user id + date ride $1/$2. - expect(sql).toContain("$1"); - expect(sql).toContain("$2"); - expect(boundUserId).toBe("user-targets-1"); + expect(latestCall).toBeDefined(); + const [strings, ...values] = latestCall as [string[], ...unknown[]]; + expect(strings.join("?")).not.toContain("DISTINCT ON"); + expect(values).toContain("user-targets-1"); + const floor = values.find( + (v): v is { values: unknown[] } => + typeof v === "object" && v !== null && !Array.isArray(v), + ); + const flooredAt = floor?.values[0]; expect(flooredAt).toBeInstanceOf(Date); const oneYearMs = 365 * 24 * 60 * 60 * 1000; - const ageMs = Date.now() - flooredAt.getTime(); + const ageMs = Date.now() - (flooredAt as Date).getTime(); // Floor is somewhere around one year ago — give a ±1-hour window // for clock drift between the route's `Date.now()` and the test // assertion. @@ -454,7 +449,7 @@ describe("GET /api/insights/targets — mood-rollup tier swap", () => { timeZone, ), ); - (prisma.$queryRawUnsafe as ReturnType).mockResolvedValue([ + (prisma.$queryRaw as ReturnType).mockResolvedValue([ { type: "WEIGHT", value: 70 }, { type: "BLOOD_PRESSURE_SYS", value: 125 }, { type: "BLOOD_PRESSURE_DIA", value: 75 }, diff --git a/src/app/api/measurements/series/route.ts b/src/app/api/measurements/series/route.ts index b3abf5504..45bb64bb1 100644 --- a/src/app/api/measurements/series/route.ts +++ b/src/app/api/measurements/series/route.ts @@ -54,6 +54,7 @@ import { import { resolveSeriesReferenceLines } from "@/lib/measurements/series-reference-lines"; import { getServerTranslator } from "@/lib/i18n/server-translator"; import { resolveJobLocale } from "@/lib/i18n/job-locale"; +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; /** * v1.11.4 — sleep is read row-per-stage (not from the day rollup), so its @@ -209,10 +210,8 @@ interface SeriesPoint { function stdDev(values: number[]): number { if (values.length < 2) return 0; - const mean = values.reduce((sum, v) => sum + v, 0) / values.length; - const variance = - values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length; - return Math.round(Math.sqrt(variance) * 100) / 100; + const sd = standardDeviation(values, READING_SPREAD) ?? 0; + return Math.round(sd * 100) / 100; } export const GET = apiHandler(async (request: NextRequest) => { diff --git a/src/app/api/medications/[id]/cadence/route.ts b/src/app/api/medications/[id]/cadence/route.ts index 2660a2773..96daeb9b1 100644 --- a/src/app/api/medications/[id]/cadence/route.ts +++ b/src/app/api/medications/[id]/cadence/route.ts @@ -24,10 +24,12 @@ import { apiError, apiSuccess, returnAllZodIssues } from "@/lib/api-response"; import { buildCadenceTimeline, computeNextDose, - type CadenceEngineContext, } from "@/lib/medications/scheduling/cadence"; import { complianceChips } from "@/lib/medications/scheduling/compliance"; import { + buildComplianceMedicationContext, + COMPLIANCE_LEDGER_WINDOW_DAYS, + COURSES_COMPLIANCE_SELECT, lastNonSkippedTakenAt, SCHEDULE_COMPLIANCE_SELECT, } from "@/lib/analytics/compliance"; @@ -58,6 +60,11 @@ export const GET = apiHandler( schedules: { select: SCHEDULE_COMPLIANCE_SELECT }, // v1.16.3 — archived schedule eras for era-aware chips/tallies. scheduleRevisions: { orderBy: { validFrom: "asc" } }, + // The pause eras and courses, so the chips tally the ledger the + // detail page's rate reads: a paused day or a gap between two + // courses expects nothing. + pauseEras: { select: { pausedAt: true, resumedAt: true } }, + courses: COURSES_COMPLIANCE_SELECT, }, }); if (!med) { @@ -75,11 +82,13 @@ export const GET = apiHandler( const windowDays = parsed.data.days ?? 30; const asOf = new Date(); - // Pull intake events covering the window + a small buffer so a dose - // logged at the edge can still pair with its slot. - const bufferMs = 24 * 60 * 60 * 1000; + // The events the compliance ledger reads (its full trailing year plus a + // day): the chips tally the same rows as the detail page, and a rolling + // cadence re-anchors on the latest intake even when it predates the + // chart's window. const from = new Date( - asOf.getTime() - windowDays * 24 * 60 * 60 * 1000 - bufferMs, + asOf.getTime() - + (COMPLIANCE_LEDGER_WINDOW_DAYS + 1) * 24 * 60 * 60 * 1000, ); const events = await prisma.medicationIntakeEvent.findMany({ where: { @@ -93,6 +102,8 @@ export const GET = apiHandler( scheduledFor: true, takenAt: true, skipped: true, + // A dose the auto-miss job closed counts as missed, as in the rate. + autoMissed: true, // v1.15.20 — pinned takes bind by anchor in the chip tally so the // chips agree with the % + the history on a "zugeordnet" dose. attributionSource: true, @@ -121,16 +132,11 @@ export const GET = apiHandler( // legacy weekday walker (e.g. `FREQ=WEEKLY;BYDAY=MO` expands daily, // PRN counts as a real cadence) and disagrees with the dashboard. // Mirrors `analytics/compliance.ts`'s context construction. - const lastIntakeAt = lastNonSkippedTakenAt(events); - const engineCtx: CadenceEngineContext = { - startsOn: med.startsOn, - endsOn: med.endsOn, - oneShot: med.oneShot, - createdAt: med.createdAt, - lastIntakeAt, - timeZone: userTz, - scheduleRevisions: med.scheduleRevisions, - }; + const engineCtx = buildComplianceMedicationContext( + med, + lastNonSkippedTakenAt(events), + userTz, + ); // v1.39.1 (#1033) — intake tracking off: no chip, no next dose. const liveSchedules = dueSchedules(med); @@ -146,11 +152,10 @@ export const GET = apiHandler( const chips = complianceChips( liveSchedules, events, + engineCtx, asOf, windowDays, anchor, - userTz, - engineCtx, ); const next = computeNextDose( liveSchedules, diff --git a/src/app/api/workouts/batch/route.ts b/src/app/api/workouts/batch/route.ts index 6d3e30d85..db2d1b1c7 100644 --- a/src/app/api/workouts/batch/route.ts +++ b/src/app/api/workouts/batch/route.ts @@ -91,7 +91,7 @@ import { UNSTABLE_EXTERNAL_ID_REASON, type UnstableExternalIdShape, } from "@/lib/validations/external-id"; -import { dedupeWorkoutBatch } from "@/lib/workouts/canonical-rows"; +import { pickCanonicalWorkoutRows } from "@/lib/measurements/pick-canonical-workout-rows"; import { WORKOUTS_WRITE_SCOPE } from "@/lib/workouts/scopes"; import { EXTERNAL_SOURCE } from "@/lib/measurements/external-source"; import { encryptRouteGeometry } from "@/lib/workouts/route-geometry-cipher"; @@ -494,41 +494,29 @@ async function postBatch(request: NextRequest): Promise { }); } - // v1.4.42 W5 — write-time cross-source dedup. The v1.5 iOS app - // drains a HealthKit observer queue that frequently carries the - // SAME logical workout from two paired sensors (Apple Watch + - // Withings ScanWatch). Without this pass, both rows persist (their - // `externalId`s differ → the `(userId, source, externalId)` unique - // index never fires) and the read-time canonical-picker has to - // carry the duplicate forward on every dashboard render. - // - // The helper is pure (no Prisma, no user lookups). It groups by - // `(userId, activityType, startedAt ± 90 s)`, prefers the canonical - // source ladder (APPLE_HEALTH > WITHINGS > MANUAL > IMPORT), and - // breaks ties on calories > earliest createdAt > input order. Rows - // dropped here surface to the iOS client as `duplicate` so the - // sync cursor advances past them identically to the externalId - // case below. - const canonicalPrepared = dedupeWorkoutBatch( + // Write-time cross-source dedup through the one workout picker the reads + // use, with the same source ladder: one logical workout from two sources + // in one batch (or one HealthKit session re-sent under a new id) lands + // once. Rows the picker hides surface to the client as `duplicate`, so + // the sync cursor advances past them like the externalId case below. + const canonicalPrepared = pickCanonicalWorkoutRows( prepared.map((p) => ({ + index: p.index, userId: p.row.userId, - activityType: p.row.sportType, + sportType: p.row.sportType, startedAt: p.row.startedAt as Date, + durationSec: p.row.durationSec, // Zod's `measurementSourceEnum.optional().default("MANUAL")` // guarantees `source` is set post-parse, but Prisma's - // `WorkoutCreateManyInput` types it optional. Default through - // the same fallback the validator uses so the picker's source - // ladder still resolves on the rare row where TypeScript - // can't prove the union. + // `WorkoutCreateManyInput` types it optional. source: (p.row.source ?? "MANUAL") as MeasurementSource, - caloriesKcal: p.row.totalEnergyKcal ?? null, - // No createdAt yet — the row is pre-insert; the tie-breaker - // falls through to input order via the `index` field below. - index: p.index, + avgHeartRate: p.row.avgHeartRate ?? null, + maxHeartRate: p.row.maxHeartRate ?? null, + metadata: p.row.metadata ?? null, })), userPriority?.sourcePriorityJson ?? null, ); - const survivingIndices = new Set(canonicalPrepared.map((r) => r.index ?? -1)); + const survivingIndices = new Set(canonicalPrepared.map((r) => r.index)); const droppedByWriteDedup: number[] = []; for (const p of prepared) { if (!survivingIndices.has(p.index)) { diff --git a/src/lib/analytics/__tests__/correlations.test.ts b/src/lib/analytics/__tests__/correlations.test.ts index 1e91db9f4..26be92f55 100644 --- a/src/lib/analytics/__tests__/correlations.test.ts +++ b/src/lib/analytics/__tests__/correlations.test.ts @@ -2,7 +2,6 @@ import { describe, it, expect } from "vitest"; import type { DataPoint } from "../trends"; import { pairByTimestamp, - pearsonCorrelation, significantPearsonCorrelation, } from "../correlations"; import type { PairedPoint } from "../correlations"; @@ -52,62 +51,6 @@ describe("pairByTimestamp", () => { }); }); -describe("pearsonCorrelation", () => { - it("returns null for fewer than minPairs", () => { - const pairs = [ - { a: 1, b: 2, date: new Date() }, - { a: 2, b: 4, date: new Date() }, - ]; - expect(pearsonCorrelation(pairs)).toBeNull(); - }); - - it("detects perfect positive correlation", () => { - const pairs = Array.from({ length: 10 }, (_, i) => ({ - a: i, - b: i * 2, - date: new Date(), - })); - const result = pearsonCorrelation(pairs)!; - expect(result.r).toBe(1); - expect(result.strength).toBe("stark"); - expect(result.n).toBe(10); - }); - - it("detects perfect negative correlation", () => { - const pairs = Array.from({ length: 10 }, (_, i) => ({ - a: i, - b: 100 - i * 2, - date: new Date(), - })); - const result = pearsonCorrelation(pairs)!; - expect(result.r).toBe(-1); - expect(result.strength).toBe("stark"); - }); - - it("detects no correlation for random-like data", () => { - const pairs = [ - { a: 1, b: 5, date: new Date() }, - { a: 2, b: 3, date: new Date() }, - { a: 3, b: 7, date: new Date() }, - { a: 4, b: 2, date: new Date() }, - { a: 5, b: 6, date: new Date() }, - ]; - const result = pearsonCorrelation(pairs)!; - expect(Math.abs(result.r)).toBeLessThan(0.4); - }); - - it("returns keine for constant values", () => { - const pairs = Array.from({ length: 5 }, () => ({ - a: 5, - b: 5, - date: new Date(), - })); - const result = pearsonCorrelation(pairs)!; - expect(result.r).toBe(0); - expect(result.strength).toBe("keine"); - }); -}); - describe("significantPearsonCorrelation (M-CS3 — n>=20 AND p<0.05)", () => { const mk = (a: number, b: number): PairedPoint => ({ a, @@ -116,14 +59,9 @@ describe("significantPearsonCorrelation (M-CS3 — n>=20 AND p<0.05)", () => { }); it("suppresses a 5-pair r≈0.7 dataset (below the n>=20 floor)", () => { - // Five points with a strong-looking slope — exactly the small-n fluke the - // plain `pearsonCorrelation` would surface as a "stark" correlation. + // Five points with a strong-looking slope: the small-n fluke an + // ungated band would call a "stark" correlation. const pairs = [mk(1, 1), mk(2, 2), mk(3, 2), mk(4, 4), mk(5, 3)]; - // Sanity-check the legacy path WOULD have surfaced it (non-null, n=5). - const legacy = pearsonCorrelation(pairs); - expect(legacy).not.toBeNull(); - expect(legacy!.n).toBe(5); - // The gated path refuses it — too few pairs. expect(significantPearsonCorrelation(pairs)).toBeNull(); }); diff --git a/src/lib/analytics/__tests__/mood-series.test.ts b/src/lib/analytics/__tests__/mood-series.test.ts index 07ec24220..e29667876 100644 --- a/src/lib/analytics/__tests__/mood-series.test.ts +++ b/src/lib/analytics/__tests__/mood-series.test.ts @@ -83,6 +83,19 @@ describe("buildMoodDailySeries", () => { expect(series.summary.count).toBe(2); }); + it("bounds the live walk to the window the rollup read covers", async () => { + await buildMoodDailySeries("user-1"); + const [rollupUser, since] = vi.mocked(readMoodDayRollups).mock.calls[0] as [ + string, + Date, + ]; + expect(rollupUser).toBe("user-1"); + const args = vi.mocked(prisma.moodEntry.findMany).mock.calls[0]?.[0] as { + where: { moodLoggedAt?: { gte: Date } }; + }; + expect(args.where.moodLoggedAt?.gte).toEqual(since); + }); + it("returns an empty series for a user with no rollups and no raw entries", async () => { const series = await buildMoodDailySeries("user-1"); expect(series.entries).toEqual([]); diff --git a/src/lib/analytics/__tests__/summaries-slice.test.ts b/src/lib/analytics/__tests__/summaries-slice.test.ts index a503f58e0..de194fba8 100644 --- a/src/lib/analytics/__tests__/summaries-slice.test.ts +++ b/src/lib/analytics/__tests__/summaries-slice.test.ts @@ -66,7 +66,8 @@ vi.mock("@/lib/analytics/source-rank-sql", () => { import { prisma } from "@/lib/db"; import { canonicalMeasurementsCte } from "@/lib/analytics/source-rank-sql"; import { startOfUtcDay } from "@/lib/tz/start-of-utc-day"; -import { computeSummariesSlice } from "../summaries-slice"; +import { caches, __resetAllCachesForTests } from "@/lib/cache/server-cache"; +import { computeSummariesSlice, readSummariesSlice } from "../summaries-slice"; const RAW = prisma.$queryRaw as unknown as ReturnType; const UNSAFE = prisma.$queryRawUnsafe as unknown as ReturnType; @@ -686,3 +687,37 @@ describe("computeSummariesSlice", () => { }); }); }); + +describe("readSummariesSlice — the one cache cell", () => { + beforeEach(() => { + __resetAllCachesForTests(); + UNSAFE.mockResolvedValue([]); + }); + + it("builds the slice once for every consumer inside the fresh TTL", async () => { + const first = await readSummariesSlice("user-cell", { serveStale: true }); + const builds = RAW.mock.calls.length; + expect(builds).toBeGreaterThan(0); + + const composed = await readSummariesSlice("user-cell"); + const snapshot = await readSummariesSlice("user-cell", { + coverage: new Map(), + }); + + expect(composed).toBe(first); + expect(snapshot).toBe(first); + expect(RAW.mock.calls.length).toBe(builds); + }); + + it("serves stale only to the edge reader; a composing reader rebuilds", async () => { + const before = await readSummariesSlice("user-stale"); + caches.analytics.markStaleByPrefix("user-stale|"); + + const edge = await readSummariesSlice("user-stale", { serveStale: true }); + expect(edge).toBe(before); + + caches.analytics.markStaleByPrefix("user-stale|"); + const composed = await readSummariesSlice("user-stale"); + expect(composed).not.toBe(before); + }); +}); diff --git a/src/lib/analytics/compliance/display.ts b/src/lib/analytics/compliance/display.ts index ac2f8cea2..7073eab46 100644 --- a/src/lib/analytics/compliance/display.ts +++ b/src/lib/analytics/compliance/display.ts @@ -29,9 +29,9 @@ import { } from "./windows"; import { COMPLIANCE_LEDGER_WINDOW_DAYS, - buildComplianceLedgerRows, tallyComplianceFromLedger, - tallyLedgerRows, + trailingComplianceLedger, + trailingComplianceTally, } from "./ledger"; import { buildCurrentCycle, type CurrentCycle } from "./cycle"; import { rollingIntakeInstants } from "./adapters"; @@ -564,9 +564,7 @@ export function buildMedicationComplianceBundle( : ledgerPeriodStart; const hasSchedules = schedules.length > 0; - const ledgerRows = hasSchedules - ? buildComplianceLedgerRows(events, schedules, ctx, ledgerFrom, now, now) - : []; + const ledgerRows = trailingComplianceLedger(events, schedules, ctx, now); // ONE cadence timeline over the full ledger window. Each per-window // streak below walks only its own trailing `effectiveDays`, so sharing @@ -607,10 +605,7 @@ export function buildMedicationComplianceBundle( 1, Math.ceil((now.getTime() - effectiveStart.getTime()) / ONE_DAY_MS), ); - const tally = tallyLedgerRows(ledgerRows, { - from: periodStart, - to: now, - }); + const tally = trailingComplianceTally(ledgerRows, days, now); const { current: streak } = streaksFromTimeline( timeline, now, diff --git a/src/lib/analytics/compliance/ledger.ts b/src/lib/analytics/compliance/ledger.ts index baffb1123..230b3dfb0 100644 --- a/src/lib/analytics/compliance/ledger.ts +++ b/src/lib/analytics/compliance/ledger.ts @@ -306,6 +306,41 @@ export function tallyLedgerRows( */ export const COMPLIANCE_LEDGER_WINDOW_DAYS = 365; +/** + * The ledger over the trailing {@link COMPLIANCE_LEDGER_WINDOW_DAYS} ending at + * `now`: the one set of dose rows every per-medication compliance figure is + * tallied from. A slot before the medication's creation stays in it only when + * a recorded dose claims it (#1028). + */ +export function trailingComplianceLedger( + events: IntakeEvent[], + schedules: ComplianceSchedule[], + ctx: ComplianceMedicationContext, + now: Date, +): DoseHistoryRow[] { + if (schedules.length === 0) return []; + const from = new Date( + now.getTime() - COMPLIANCE_LEDGER_WINDOW_DAYS * 24 * 60 * 60 * 1000, + ); + return buildComplianceLedgerRows(events, schedules, ctx, from, now, now); +} + +/** + * The tally of the trailing `days` ending at `now`, carved out of + * {@link trailingComplianceLedger}: the same rows, so the same window reads + * the same rate on every surface. + */ +export function trailingComplianceTally( + ledger: DoseHistoryRow[], + days: number, + now: Date, +): LedgerComplianceTally { + return tallyLedgerRows(ledger, { + from: new Date(now.getTime() - days * 24 * 60 * 60 * 1000), + to: now, + }); +} + /** * One day of the cadence-aware compliance series: the day's compliance % * over the doses the schedule actually expected that day, plus the raw diff --git a/src/lib/analytics/correlations.ts b/src/lib/analytics/correlations.ts index 10270f1f5..6065af944 100644 --- a/src/lib/analytics/correlations.ts +++ b/src/lib/analytics/correlations.ts @@ -72,54 +72,18 @@ export function pairByTimestamp( } /** - * Pearson correlation coefficient. - * Returns null if fewer than minPairs data points. - */ -export function pearsonCorrelation( - pairs: PairedPoint[], - minPairs: number = 5, -): CorrelationResult | null { - if (pairs.length < minPairs) return null; - - const n = pairs.length; - const sumA = pairs.reduce((s, p) => s + p.a, 0); - const sumB = pairs.reduce((s, p) => s + p.b, 0); - const sumAB = pairs.reduce((s, p) => s + p.a * p.b, 0); - const sumAA = pairs.reduce((s, p) => s + p.a * p.a, 0); - const sumBB = pairs.reduce((s, p) => s + p.b * p.b, 0); - - const denominator = Math.sqrt( - (n * sumAA - sumA * sumA) * (n * sumBB - sumB * sumB), - ); - - if (denominator === 0) return { r: 0, strength: "keine", n }; - - const r = Math.round(((n * sumAB - sumA * sumB) / denominator) * 1000) / 1000; - const absR = Math.abs(r); - - let strength: CorrelationResult["strength"]; - if (absR >= 0.7) strength = "stark"; - else if (absR >= 0.4) strength = "moderat"; - else if (absR >= 0.2) strength = "schwach"; - else strength = "keine"; - - return { r, strength, n }; -} - -/** - * v1.2.5 (M-CS3) — significance-gated Pearson for USER-FACING surfaces. + * v1.2.5 (M-CS3) — significance-gated Pearson, the one correlation every + * surface reports. * - * The plain `pearsonCorrelation` above assigns a strength band from |r| - * alone with no significance test and a default floor of only 5 pairs — - * so a 5-point r ≈ 0.7 fluke surfaces as a "stark" correlation. Every - * surfaced insight-card / status-snapshot correlation must instead clear - * the same rigorous bar the `/insights` correlation engine enforces: + * A strength band read from |r| alone, above a floor of five pairs, let a + * 5-point r ≈ 0.7 fluke surface as a "stark" correlation. Every reported + * correlation clears the bar the `/insights` correlation engine enforces: * * - n >= 20 paired points (the engine's `MIN_PAIRED_N`), AND * - a two-sided Student-t p-value < 0.05 (`MAX_P_VALUE`). * * Below either bar this returns `null` so the caller suppresses the - * correlation exactly as it already does for the < minPairs case — no + * correlation — no * call-site shape change. The returned `{ r, strength, n }` keeps the * legacy shape (German strength band) so the snapshot byte-shape is * unchanged on the happy path. diff --git a/src/lib/analytics/glucose-metrics.ts b/src/lib/analytics/glucose-metrics.ts index a0f566735..06eca2cee 100644 --- a/src/lib/analytics/glucose-metrics.ts +++ b/src/lib/analytics/glucose-metrics.ts @@ -40,6 +40,7 @@ */ import { GLUCOSE_TIR_CUT_POINTS } from "@/lib/glucose"; +import { standardDeviation } from "@/lib/stats/spread"; // ── Inputs ─────────────────────────────────────────────── @@ -236,15 +237,7 @@ function mean(values: number[]): number { * Returns null for n < 2 (undefined sample SD). */ export function glucoseSD(values: number[]): number | null { - const n = values.length; - if (n < 2) return null; - const m = mean(values); - let ss = 0; - for (const v of values) { - const d = v - m; - ss += d * d; - } - return Math.sqrt(ss / (n - 1)); + return standardDeviation(values, "sample"); } /** diff --git a/src/lib/analytics/mood-prognosis/features.ts b/src/lib/analytics/mood-prognosis/features.ts index 467b71236..53f012998 100644 --- a/src/lib/analytics/mood-prognosis/features.ts +++ b/src/lib/analytics/mood-prognosis/features.ts @@ -68,6 +68,7 @@ import { LINKED_FEATURE_METRICS, } from "./feature-keys"; import { occurrencesSuffice } from "./thresholds"; +import { standardDeviation } from "@/lib/stats/spread"; /** * Most features the fit may carry. @@ -444,10 +445,7 @@ export function buildFeatureMatrix( const standardisation = selected.map((key, column) => { const values = rawRows.map((row) => row[column]); const mean = values.reduce((sum, v) => sum + v, 0) / values.length; - const variance = - values.reduce((sum, v) => sum + (v - mean) * (v - mean), 0) / - values.length; - return { key, mean, sd: Math.sqrt(variance) }; + return { key, mean, sd: standardDeviation(values, "population") ?? 0 }; }); // A column with no spread inside the complete-case set carries no diff --git a/src/lib/analytics/mood-series.ts b/src/lib/analytics/mood-series.ts index 676eb62e4..8e57d64e3 100644 --- a/src/lib/analytics/mood-series.ts +++ b/src/lib/analytics/mood-series.ts @@ -116,11 +116,12 @@ export async function buildMoodDailySeries( return { entries, summary, entryCount, source: "rollup" }; } - // Coverage fallback — no rollup rows yet. Run the legacy live walk ONCE; - // the warm-up above mints the rollups for the next request. + // Coverage fallback — no rollup rows yet. Run the live walk ONCE over the + // same window the rollup read covers; the warm-up above mints the rollups + // for the next request. const moodEntries = await client.moodEntry.findMany({ // v1.7.0 sync — exclude tombstoned rows from the fallback. - where: { userId, deletedAt: null }, + where: { userId, deletedAt: null, moodLoggedAt: { gte: since } }, orderBy: { moodLoggedAt: "asc" }, select: { date: true, score: true }, }); diff --git a/src/lib/analytics/score/__tests__/config-change-is-not-a-health-event.test.ts b/src/lib/analytics/score/__tests__/config-change-is-not-a-health-event.test.ts index 2d46fa5c1..8e1511cb5 100644 --- a/src/lib/analytics/score/__tests__/config-change-is-not-a-health-event.test.ts +++ b/src/lib/analytics/score/__tests__/config-change-is-not-a-health-event.test.ts @@ -189,7 +189,7 @@ const fakeDb = vi.hoisted(() => { vi.mock("@/lib/db", () => ({ prisma: fakeDb })); // ── Everything around the score, faked at its own seam ────────────── -const computeSummariesSlice = vi.fn(); +const readSummariesSlice = vi.fn(); const probeRollupCoverage = vi.fn(); const isFullyCovered = vi.fn(); const readMoodDayRollups = vi.fn(); @@ -205,7 +205,7 @@ const loadIntradayPulse = vi.fn(); const readDayMeanSeries = vi.fn(); vi.mock("@/lib/analytics/summaries-slice", () => ({ - computeSummariesSlice: (...a: unknown[]) => computeSummariesSlice(...a), + readSummariesSlice: (...a: unknown[]) => readSummariesSlice(...a), })); vi.mock("@/lib/rollups/measurement-coverage", () => ({ probeRollupCoverage: (...a: unknown[]) => probeRollupCoverage(...a), @@ -381,7 +381,7 @@ beforeEach(() => { vi.clearAllMocks(); __resetAllCachesForTests(); dismissedKeys.clear(); - computeSummariesSlice.mockResolvedValue({ + readSummariesSlice.mockResolvedValue({ summaries: {}, lastSeenByType: {}, bmi: null, diff --git a/src/lib/analytics/score/activity.ts b/src/lib/analytics/score/activity.ts index 4601eb786..326363dfd 100644 --- a/src/lib/analytics/score/activity.ts +++ b/src/lib/analytics/score/activity.ts @@ -9,7 +9,7 @@ import { insufficientPillar, mean, okPillar, - standardDeviation, + readingSpread, } from "./shared"; export const ACTIVITY_WINDOW_DAYS = 28; @@ -74,7 +74,7 @@ export function computeActivityPillar( const score = clampScore((observed / plateau) * 100); const noiseFloor = Math.max( 1, - Math.round(((0.5 * standardDeviation(values)) / plateau) * 100), + Math.round(((0.5 * readingSpread(values)) / plateau) * 100), ); const latestDay = [...points].sort((a, b) => b.day.localeCompare(a.day))[0]; diff --git a/src/lib/analytics/score/glycaemia.ts b/src/lib/analytics/score/glycaemia.ts index b62462a02..dca0bb893 100644 --- a/src/lib/analytics/score/glycaemia.ts +++ b/src/lib/analytics/score/glycaemia.ts @@ -10,7 +10,7 @@ import { median, okPillar, scoreReferenceBand, - standardDeviation, + readingSpread, uniqueSources, withinWindow, } from "./shared"; @@ -126,7 +126,7 @@ export function computeGlycaemiaPillar( const observed = median(values); // The behavioural/device noise convention from the design is one half of // within-person SD, translated onto this pillar's proportional score scale. - const halfSd = standardDeviation(values) * 0.5; + const halfSd = readingSpread(values) * 0.5; const floorScore = scoreReferenceBand( Math.max(0, observed - halfSd), FASTING_GLUCOSE_REFERENCE.low, diff --git a/src/lib/analytics/score/shared.ts b/src/lib/analytics/score/shared.ts index e5dfbbdfa..1fa3bff2d 100644 --- a/src/lib/analytics/score/shared.ts +++ b/src/lib/analytics/score/shared.ts @@ -11,6 +11,7 @@ import type { } from "@/lib/insights/derived/types"; import type { PillarValue, ScoreBand, ScorePillarId } from "./types"; +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; export const DAY_MS = 86_400_000; @@ -88,11 +89,9 @@ export function median(values: readonly number[]): number { : ordered[middle]; } -export function standardDeviation(values: readonly number[]): number { +export function readingSpread(values: readonly number[]): number { if (values.length < 2) return 0; - const center = mean(values); - const variance = mean(values.map((value) => (value - center) ** 2)); - return Math.sqrt(variance); + return standardDeviation(values, READING_SPREAD) ?? 0; } export function uniqueSources(values: readonly string[]): string[] { diff --git a/src/lib/analytics/score/sleep.ts b/src/lib/analytics/score/sleep.ts index 086fd887c..bebc81462 100644 --- a/src/lib/analytics/score/sleep.ts +++ b/src/lib/analytics/score/sleep.ts @@ -13,7 +13,7 @@ import { insufficientPillar, mean, okPillar, - standardDeviation, + readingSpread, } from "./shared"; export const SLEEP_WINDOW_DAYS = 28; @@ -75,7 +75,7 @@ export function computeSleepPillar( : Math.round(sufficiency * 0.6 + regularity * 0.4); const noiseFloor = Math.max( 1, - Math.round(((0.5 * standardDeviation(asleepValues)) / need) * 100), + Math.round(((0.5 * readingSpread(asleepValues)) / need) * 100), ); const latest = nights.at(-1)!; diff --git a/src/lib/analytics/summaries-slice.ts b/src/lib/analytics/summaries-slice.ts index b1a5519a2..4bc962cfc 100644 --- a/src/lib/analytics/summaries-slice.ts +++ b/src/lib/analytics/summaries-slice.ts @@ -30,7 +30,7 @@ * windowed columns (`avg7`, `avg30`, and the slope tuples). The * heavy COUNT/MIN/MAX/AVG columns are NOT projected. * - * 3. **`DISTINCT ON (type)` latest read** — most-recent `(type, + * 3. **Latest read** (`readLatestCanonical`) — most-recent `(type, * value, measured_at)` triplet per type for the `latest` field * and the `lastSeenByType` freshness map. * @@ -77,6 +77,7 @@ import { import type { DataSummary } from "@/lib/analytics/trends"; import { measurementTypeEnum } from "@/lib/validations/measurement"; import { annotate } from "@/lib/logging/context"; +import { caches, type ServerCache } from "@/lib/cache/server-cache"; import { ensureUserRollupsFresh, readRollupBuckets, @@ -95,6 +96,7 @@ import { buildSourceRankCase, canonicalMeasurementsCte, } from "@/lib/analytics/source-rank-sql"; +import { readLatestCanonical } from "@/lib/measurements/latest-canonical"; import { isFullyCovered, probeRollupCoverage, @@ -223,19 +225,6 @@ interface WindowedAggregateRow extends NarrowAggregateRow { r2_30: number | null; } -interface LatestRow { - type: string; - value: number; - /** - * v1.4.34 IW-B — per-type freshness timestamp. The DISTINCT ON pass - * already orders by `measured_at DESC` to pick the latest row's - * value; surfacing `measured_at` from the same row costs zero extra - * round-trips and feeds the dashboard tile-strip's stale-data - * caption (driven via ``). - */ - measured_at: Date; -} - /** Same threshold/rounding contract as `trendSlope()` in `trends.ts`. */ function buildSlope( slope: number | null, @@ -373,6 +362,35 @@ export async function computeSummariesSlice( ); } +/** + * The one cache cell of the summaries slice. The tile-strip route, the thick + * analytics body and the dashboard snapshot all read the slice; each used to + * compute it inside its own cell, so one dashboard mount built it up to three + * times over the same rows. + * + * `serveStale` is for the caller that answers the client with the slice + * itself. The composing callers are stale-while-revalidate cells of their + * own and read fresh-only: a background refresh after a write must not + * compose the pre-write slice and cache it again as fresh. + */ +export async function readSummariesSlice( + userId: string, + options: { coverage?: RollupCoverageMap; serveStale?: boolean } = {}, +): Promise { + const cache = caches.analytics as ServerCache; + const key = summariesSliceCacheKey(userId); + const build = () => computeSummariesSlice(userId, options.coverage); + const { value, outcome } = options.serveStale + ? await cache.wrapSwr(key, build) + : await cache.wrap(key, build); + annotate({ meta: { "cache.summaries_slice.outcome": outcome } }); + return value; +} + +export function summariesSliceCacheKey(userId: string): string { + return `${userId}|summaries`; +} + /** * `latest` of an hourly-mean type (pulse) is the value of its latest local * day, not its latest single reading. @@ -559,15 +577,13 @@ async function computeFromRollups(userId: string): Promise { // v1.11.1 — resolve the user's source-priority ladders once, then build the // rank CASE expressions used to collapse overlapping sources to the canonical // reading per (type, day) inside each query. `"type"`/`"source"` variants for - // the unqualified rollup + inner-subquery columns, `m."…"` for the latest - // read's outer alias. + // the unqualified rollup + inner-subquery columns. const priorityJson = await loadUserSourcePriority(userId); const rankUnqualified = buildSourceRankCase( priorityJson, '"type"', '"source"', ); - const rankM = buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'); const rankPerSource = buildSourceRankCase( priorityJson, 'p."type"', @@ -618,22 +634,7 @@ async function computeFromRollups(userId: string): Promise { `, userId, ), - // v1.11.1 — the latest tile reflects the canonical source for the latest - // day, matching the chart: order by latest day first, then the ladder rank - // (canonical source wins), then the latest reading of that source. - prisma.$queryRawUnsafe( - ` - SELECT DISTINCT ON (m."type") - m."type"::text AS type, - m."value"::double precision AS value, - m."measured_at" AS measured_at - FROM measurements m - WHERE m."user_id" = $1 - AND m."deleted_at" IS NULL - ORDER BY m."type", date_trunc('day', m."measured_at") DESC, (${rankM}), m."measured_at" DESC - `, - userId, - ), + readLatestCanonical(userId, priorityJson), // v1.4.37.2 hotfix — the v1.4.35 implementation read EVERY DAY // rollup bucket for the user (`findMany` without a `bucketStart` // window) and then composed `count / min / max / mean` in JS. @@ -1055,7 +1056,6 @@ async function computeFromLiveAggregate( '"type"', '"source"', ); - const rankM = buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'); const [allTime, windowed, latests] = await Promise.all([ prisma.$queryRawUnsafe( @@ -1129,19 +1129,7 @@ async function computeFromLiveAggregate( `, userId, ), - prisma.$queryRawUnsafe( - ` - SELECT DISTINCT ON (m."type") - m."type"::text AS type, - m."value"::double precision AS value, - m."measured_at" AS measured_at - FROM measurements m - WHERE m."user_id" = $1 - AND m."deleted_at" IS NULL - ORDER BY m."type", date_trunc('day', m."measured_at") DESC, (${rankM}), m."measured_at" DESC - `, - userId, - ), + readLatestCanonical(userId, priorityJson), ]); const latestByType = new Map(); diff --git a/src/lib/analytics/trends.ts b/src/lib/analytics/trends.ts index 242f30bea..08b17d4d7 100644 --- a/src/lib/analytics/trends.ts +++ b/src/lib/analytics/trends.ts @@ -3,6 +3,8 @@ * All functions are pure and work on sorted arrays of {date, value} pairs. */ +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; + export interface DataPoint { date: Date; value: number; @@ -95,9 +97,7 @@ export function detectAnomalies(data: DataPoint[], threshold = 2.0): Anomaly[] { const values = data.map((p) => p.value); const mean = values.reduce((s, v) => s + v, 0) / values.length; - const variance = - values.reduce((s, v) => s + (v - mean) ** 2, 0) / values.length; - const stdDev = Math.sqrt(variance); + const stdDev = standardDeviation(values, READING_SPREAD) ?? 0; if (stdDev === 0) return []; diff --git a/src/lib/dashboard/__tests__/snapshot.test.ts b/src/lib/dashboard/__tests__/snapshot.test.ts index 03c7e4fab..53daaec70 100644 --- a/src/lib/dashboard/__tests__/snapshot.test.ts +++ b/src/lib/dashboard/__tests__/snapshot.test.ts @@ -13,7 +13,7 @@ */ import { describe, it, expect, vi, beforeEach } from "vitest"; -const computeSummariesSlice = vi.fn(); +const readSummariesSlice = vi.fn(); const probeRollupCoverage = vi.fn(); const isFullyCovered = vi.fn(); const readMoodDayRollups = vi.fn(); @@ -24,7 +24,7 @@ const buildMedsTodayBlock = vi.fn(); const computeUserHealthScoreMock = vi.fn(); vi.mock("@/lib/analytics/summaries-slice", () => ({ - computeSummariesSlice: (...a: unknown[]) => computeSummariesSlice(...a), + readSummariesSlice: (...a: unknown[]) => readSummariesSlice(...a), })); vi.mock("@/lib/rollups/measurement-coverage", () => ({ probeRollupCoverage: (...a: unknown[]) => probeRollupCoverage(...a), @@ -210,7 +210,7 @@ function baseUser( beforeEach(() => { vi.clearAllMocks(); - computeSummariesSlice.mockResolvedValue({ + readSummariesSlice.mockResolvedValue({ summaries: { WEIGHT: { ...emptySummary, count: 3, latest: 80 } }, lastSeenByType: { WEIGHT: { lastSeenAt: new Date().toISOString() }, @@ -301,7 +301,31 @@ describe("buildDashboardSnapshot — envelope shape", () => { expect(probeRollupCoverage).toHaveBeenCalledTimes(1); // The already-probed map is passed as the slice's second argument so // it reuses it instead of probing again. - expect(computeSummariesSlice).toHaveBeenCalledWith("user-1", coverageMap); + expect(readSummariesSlice).toHaveBeenCalledWith("user-1", { + coverage: coverageMap, + }); + }); +}); + +describe("buildDashboardSnapshot — the shared summaries cell", () => { + it("folds the water tile into a copy, never into the cached slice", async () => { + probeRollupCoverage.mockResolvedValue(new Map([["WEIGHT", false]])); + isFullyCovered.mockReturnValue(false); + const cached = { + summaries: { WEIGHT: { ...emptySummary, count: 3, latest: 80 } }, + lastSeenByType: { WEIGHT: { lastSeenAt: new Date().toISOString() } }, + bmi: null, + }; + readSummariesSlice.mockResolvedValue(cached); + fakePrisma.nutrientIntakeDay.findMany.mockResolvedValue([ + { day: new Date().toISOString().slice(0, 10), amount: 1500 }, + ]); + + const snap = await buildDashboardSnapshot(fakePrisma, baseUser()); + + expect(snap.tiles.summaries.NUTRIENT_WATER).toBeDefined(); + expect(Object.keys(cached.summaries)).toEqual(["WEIGHT"]); + expect(Object.keys(cached.lastSeenByType)).toEqual(["WEIGHT"]); }); }); @@ -891,7 +915,7 @@ describe("buildDashboardSnapshot — metricStates (iOS cold-launch seed)", () => it("keys each entry by the iOS MetricKind raw value (incl. the non-obvious raws)", async () => { const now = new Date().toISOString(); - computeSummariesSlice.mockResolvedValue({ + readSummariesSlice.mockResolvedValue({ summaries: { WEIGHT: { ...emptySummary, count: 1, latest: 80 }, OXYGEN_SATURATION: { ...emptySummary, count: 1, latest: 98 }, @@ -948,7 +972,7 @@ describe("buildDashboardSnapshot — metricStates (iOS cold-launch seed)", () => }); it("omits metrics with no latest value or no timestamp", async () => { - computeSummariesSlice.mockResolvedValue({ + readSummariesSlice.mockResolvedValue({ summaries: { WEIGHT: { ...emptySummary, count: 0, latest: null }, PULSE: { ...emptySummary, count: 1, latest: 60 }, @@ -1162,7 +1186,7 @@ describe("buildDashboardSnapshot — module gating (v1.18.0)", () => { priorMonth: { pct: 60 }, priorYear: { pct: 50 }, }); - computeSummariesSlice.mockResolvedValue({ + readSummariesSlice.mockResolvedValue({ summaries: { WEIGHT: { ...emptySummary, count: 3, latest: 80 }, SLEEP_DURATION: { ...emptySummary, count: 5, latest: 420 }, diff --git a/src/lib/dashboard/snapshot.ts b/src/lib/dashboard/snapshot.ts index 4d404906e..bcd5054c5 100644 --- a/src/lib/dashboard/snapshot.ts +++ b/src/lib/dashboard/snapshot.ts @@ -35,7 +35,7 @@ * The builder NEVER POSTs `/api/insights/generate`. */ import type { PrismaClient, MeasurementType } from "@/generated/prisma/client"; -import { computeSummariesSlice } from "@/lib/analytics/summaries-slice"; +import { readSummariesSlice } from "@/lib/analytics/summaries-slice"; import type { SleepSourceDiscrepancy } from "@/lib/analytics/sleep-night"; import { getUnitForType } from "@/lib/validations/measurement"; import { @@ -1013,7 +1013,7 @@ async function liftBriefing( /** * v1.7.0 — derive the per-metric latest-reading block from the slim * summaries slice already fetched for `tiles.summaries`. ZERO extra DB - * round-trips: the slim slice's `DISTINCT ON (type)` read already + * round-trips: the slim slice's per-type latest read already * carries the latest `value` (`summaries[type].latest`) and the * matching timestamp (`lastSeenByType[type].lastSeenAt`); the unit is * the static canonical-unit lookup. This keeps the snapshot pool-safe @@ -1329,7 +1329,7 @@ export async function buildDashboardSnapshot( ] = await Promise.all([ // A5 — reuse the coverage map already probed above so the slice // doesn't re-run the identical `probeRollupCoverage` query. - time("summaries", () => computeSummariesSlice(user.id, coverage)), + time("summaries", () => readSummariesSlice(user.id, { coverage })), time("mood", () => buildMoodBlock(prisma, user.id)), // v1.34 — the thick slice awaits the shared `modulesPromise` (the same // pattern the score-ring task uses) so the hero ring's health score can @@ -1372,29 +1372,30 @@ export async function buildDashboardSnapshot( // `NUTRIENT_WATER` key BEFORE the module gate below strips it (like // every other summary type) when `nutrients` is off. Not a real // `MeasurementType` — see the widened `SUMMARY_TYPE_MODULE` note. - if (nutrientWater) { - const lastMs = new Date(nutrientWater.lastSeenAt).getTime(); - const daysAgo = Math.floor((nowMs - lastMs) / (24 * 60 * 60 * 1000)); - slimRaw.summaries = { - ...slimRaw.summaries, - NUTRIENT_WATER: nutrientWater.summary, - }; - slimRaw.lastSeenByType = { - ...slimRaw.lastSeenByType, - NUTRIENT_WATER: { lastSeenAt: nutrientWater.lastSeenAt, daysAgo }, - }; - } + // The slice is the shared cache cell's value: fold into copies, never into + // the cached object itself. + const slimSummaries = nutrientWater + ? { ...slimRaw.summaries, NUTRIENT_WATER: nutrientWater.summary } + : slimRaw.summaries; + const slimLastSeen = nutrientWater + ? { + ...slimRaw.lastSeenByType, + NUTRIENT_WATER: { + lastSeenAt: nutrientWater.lastSeenAt, + daysAgo: Math.floor( + (nowMs - new Date(nutrientWater.lastSeenAt).getTime()) / + (24 * 60 * 60 * 1000), + ), + }, + } + : slimRaw.lastSeenByType; // v1.18.0 — strip disabled-module data before it leaves the server. // Mood is blanked when the mood module is off; sleep / glucose summary // types are dropped from the slim slice; the glucose clinical panel + // per-context block are cleared when glucose is off; the layout + // catalogue hide every disabled-module widget. - const slim = gateSummariesByModules( - slimRaw.summaries, - slimRaw.lastSeenByType, - modules, - ); + const slim = gateSummariesByModules(slimSummaries, slimLastSeen, modules); const mood = modules.mood === false ? { summary: null, entries: [] as DashboardSnapshotMoodEntry[] } diff --git a/src/lib/dashboard/summary-reads.ts b/src/lib/dashboard/summary-reads.ts index 15fc47786..2ca4df790 100644 --- a/src/lib/dashboard/summary-reads.ts +++ b/src/lib/dashboard/summary-reads.ts @@ -15,7 +15,7 @@ * an equality on the leading type column, which turns a full pass over * the account into one bounded lookup per type. */ -import type { PrismaClient } from "@/generated/prisma/client"; +import { Prisma, type PrismaClient } from "@/generated/prisma/client"; import type { MeasurementType } from "@/generated/prisma/client"; /** v1.4.38 W-F — the single most recent reading per measurement @@ -29,7 +29,8 @@ import type { MeasurementType } from "@/generated/prisma/client"; * was still in the database. The all-time aggregate (`groupBy` on * `measurements`) already proves the row exists; the row count is * still bounded by `|measurementTypes|`, now because the query is - * driven FROM that list (v1.38.5) rather than grouped down to it. */ + * driven FROM that list (v1.38.5) rather than grouped down to it. + * `since` drops a type whose latest reading is older than it. */ export interface LatestEverRow { type: MeasurementType; value: number; @@ -90,7 +91,11 @@ export function readLatestEver( db: PrismaClient, userId: string, measurementTypes: readonly MeasurementType[], + since: Date | null = null, ): Promise { + const floor = since + ? Prisma.sql`AND m."measured_at" >= ${since}` + : Prisma.empty; return db.$queryRaw` SELECT mt."type" AS type, @@ -105,6 +110,7 @@ export function readLatestEver( WHERE m."user_id" = ${userId} AND m."type" = mt."type" AND m."deleted_at" IS NULL + ${floor} ORDER BY m."measured_at" DESC LIMIT 1 ) l @@ -159,3 +165,46 @@ export function readSparkBuckets( ORDER BY type, bucket_start ASC `; } + +/** + * The local days in `[since, now]` that hold at least one live reading, as + * `YYYY-MM-DD` keys in `timeZone`: the measurement half of the streak. + * + * One existence probe per local day against `(user_id, measured_at)`, each + * stopping at its first row, instead of a `DISTINCT` over a year of readings + * (tens of thousands of rows on an account with a watch). The DAY rollups + * cannot answer this: their buckets are UTC days, and the streak counts the + * person's own days. + * + * `measured_at` is a naive UTC timestamp, so a local day's bounds are its + * local midnights read in `timeZone` and taken back to UTC. The read this + * replaced shifted the column with a single `AT TIME ZONE`, which treats the + * UTC value as local time and moves it the wrong way: in Berlin a reading at + * 23:30 UTC landed on the day before instead of the day after. + */ +export async function readActivityDayKeys( + db: PrismaClient, + userId: string, + timeZone: string, + since: Date, + now: Date, +): Promise { + const rows = await db.$queryRaw>` + SELECT to_char(d, 'YYYY-MM-DD') AS day_key + FROM generate_series( + (${since}::timestamptz AT TIME ZONE ${timeZone})::date, + (${now}::timestamptz AT TIME ZONE ${timeZone})::date, + INTERVAL '1 day' + ) AS d + WHERE EXISTS ( + SELECT 1 + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."deleted_at" IS NULL + AND m."measured_at" >= (d::timestamp AT TIME ZONE ${timeZone}) AT TIME ZONE 'UTC' + AND m."measured_at" < ((d + INTERVAL '1 day')::timestamp AT TIME ZONE ${timeZone}) AT TIME ZONE 'UTC' + ) + ORDER BY d + `; + return rows.map((row) => row.day_key); +} diff --git a/src/lib/day/__tests__/notable-history-cache.test.ts b/src/lib/day/__tests__/notable-history-cache.test.ts new file mode 100644 index 000000000..dd050aea9 --- /dev/null +++ b/src/lib/day/__tests__/notable-history-cache.test.ts @@ -0,0 +1,121 @@ +/** + * The day view's comparison history is read once per record, zone, ladder + * and readable type set, and each day takes its own slice of it. + * + * Paging from one day to the next used to read the same two years again. + * The slice has to give exactly what a live read of the window would: the + * extremes depend on where the series starts. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const readLocalDailyStats = vi.fn(); +vi.mock("@/lib/day/daily-stats", () => ({ + readLocalDailyStats: (...a: unknown[]) => readLocalDailyStats(...a), +})); +vi.mock("@/lib/db", () => ({ + prisma: { $queryRaw: vi.fn(async () => []) }, +})); + +import type { MeasurementType } from "@/generated/prisma/enums"; +import { __resetAllCachesForTests } from "@/lib/cache/server-cache"; +import { shiftDateKey, userDayKey } from "@/lib/tz/format"; + +import { + NOTABLE_LOOKBACK_DAYS, + extremesInSeries, + loadNotableRange, +} from "../notable"; + +const TZ = "Europe/Berlin"; +const TODAY = userDayKey(new Date(), TZ); +const SPIKE = shiftDateKey(TODAY, -20); +const FIRST = shiftDateKey(TODAY, -900); + +/** Systolic 120 every day from FIRST to TODAY, 150 on SPIKE. */ +function value(day: string): number { + return day === SPIKE ? 150 : 120; +} + +beforeEach(() => { + __resetAllCachesForTests(); + readLocalDailyStats.mockReset(); + readLocalDailyStats.mockImplementation( + async (args: { types: MeasurementType[]; from: Date; to: Date }) => { + const days = new Map(); + const end = userDayKey(args.to, TZ); + for ( + let day = userDayKey(args.from, TZ); + day < end; + day = shiftDateKey(day, 1) + ) { + if (day >= FIRST && day <= TODAY) days.set(day, value(day)); + } + return new Map( + args.types + .filter((t) => t === "BLOOD_PRESSURE_SYS") + .map((t) => [t, days]), + ); + }, + ); +}); + +function load(day: string, extra: { floor?: Date } = {}) { + return loadNotableRange({ + userId: "user-1", + from: day, + to: day, + tz: TZ, + priorityJson: null, + typeVisible: (t) => t !== "BLOOD_GLUCOSE", + gaps: false, + extremeTypes: ["BLOOD_PRESSURE_SYS"], + firstValueTypes: [], + ...extra, + }); +} + +/** What a live read of the window gives. */ +function expected(day: string) { + const series: Array<[string, number]> = []; + for ( + let d = shiftDateKey(day, -NOTABLE_LOOKBACK_DAYS); + d <= day; + d = shiftDateKey(d, 1) + ) { + if (d >= FIRST) series.push([d, value(d)]); + } + return extremesInSeries("BLOOD_PRESSURE_SYS", series, day); +} + +describe("notable history cache", () => { + it("reads the history once while paging through neighbouring days", async () => { + const spike = await load(SPIKE); + const before = await load(shiftDateKey(SPIKE, -1)); + const after = await load(shiftDateKey(SPIKE, 1)); + + expect(readLocalDailyStats).toHaveBeenCalledTimes(1); + expect(spike).toEqual(expected(SPIKE)); + expect(spike.map((n) => n.kind)).toEqual(["extremeHigh"]); + expect(before).toEqual(expected(shiftDateKey(SPIKE, -1))); + expect(after).toEqual(expected(shiftDateKey(SPIKE, 1))); + }); + + it("never reads a type the modules switch off", async () => { + await load(SPIKE); + const [args] = readLocalDailyStats.mock.calls[0] as [ + { types: MeasurementType[] }, + ]; + expect(args.types).not.toContain("BLOOD_GLUCOSE"); + expect(args.types).toContain("BLOOD_PRESSURE_SYS"); + }); + + it("reads live from a lookback floor inside the window", async () => { + const floor = new Date(Date.now() - 60 * 86_400_000); + await load(SPIKE, { floor }); + await load(SPIKE, { floor }); + + expect(readLocalDailyStats).toHaveBeenCalledTimes(2); + const [args] = readLocalDailyStats.mock.calls[0] as [{ from: Date }]; + expect(args.from).toEqual(floor); + }); +}); diff --git a/src/lib/day/notable.ts b/src/lib/day/notable.ts index 527c8d60d..4f275680a 100644 --- a/src/lib/day/notable.ts +++ b/src/lib/day/notable.ts @@ -25,10 +25,13 @@ * thousands of background samples, which says little as an extreme and * costs a scan of every one of them over two years. */ +import { createHash } from "node:crypto"; + import type { MeasurementType } from "@/generated/prisma/enums"; import type { DayNotable } from "@/lib/day/contract"; import { readLocalDailyStats, type DailyStats } from "@/lib/day/daily-stats"; +import { caches, type ServerCache } from "@/lib/cache/server-cache"; import { prisma } from "@/lib/db"; import { startOfLocalDayKey } from "@/lib/tz/local-day"; import { daysBetweenDateKeys, shiftDateKey, userDayKey } from "@/lib/tz/format"; @@ -184,6 +187,95 @@ export async function readFirstReadings( return out; } +/** Cached history starts on a block boundary, so neighbouring days share it. */ +const HISTORY_BLOCK_DAYS = 90; +const HISTORY_EPOCH = "1970-01-01"; +/** Measurement writes sweep the cell; the TTL only bounds an idle one. */ +const HISTORY_TTL_MS = 15 * 60_000; + +/** + * The daily series behind the extremes, for the local days `[from, to]`. + * + * Every day view compares its day with the two years before it, so paging + * from one day to the next read the same two years again. The history is + * cached per record, zone, source ladder and readable type set, from a block + * boundary at or before `from` up to today, and each caller takes its own + * slice: a day's value does not depend on the window around it, and a slice + * cut at local day keys is exactly the window the live read would see. A + * type the record's modules switch off is never read. A lookback floor + * inside the window, or a window that reaches past today, reads live. + */ +async function readExtremeHistory(args: { + userId: string; + readable: readonly MeasurementType[]; + from: string; + to: string; + tz: string; + priorityJson: unknown; + floor: Date | null; +}): Promise { + const fromInstant = startOfLocalDayKey(args.from, args.tz); + const today = userDayKey(new Date(), args.tz); + const live = (from: Date) => + readLocalDailyStats({ + userId: args.userId, + types: args.readable, + from, + to: startOfLocalDayKey(shiftDateKey(args.to, 1), args.tz), + tz: args.tz, + priorityJson: args.priorityJson, + }); + if (args.floor && args.floor > fromInstant) return live(args.floor); + if (args.to > today) return live(fromInstant); + + const offset = daysBetweenDateKeys(HISTORY_EPOCH, args.from); + const blockStart = shiftDateKey( + HISTORY_EPOCH, + Math.floor(offset / HISTORY_BLOCK_DAYS) * HISTORY_BLOCK_DAYS, + ); + const ladder = createHash("sha256") + .update(JSON.stringify(args.priorityJson ?? null)) + .digest("base64url") + .slice(0, 16); + const key = [ + args.userId, + "notable-history", + args.tz, + ladder, + [...args.readable].sort().join(","), + blockStart, + today, + ].join("|"); + const cache = caches.analytics as ServerCache; + const { value } = await cache.wrap( + key, + () => + readLocalDailyStats({ + userId: args.userId, + types: args.readable, + from: startOfLocalDayKey(blockStart, args.tz), + to: startOfLocalDayKey(shiftDateKey(today, 1), args.tz), + tz: args.tz, + priorityJson: args.priorityJson, + }), + HISTORY_TTL_MS, + ); + return sliceDays(value, args.from, args.to); +} + +/** The days of `stats` inside `[from, to]`, a copy. */ +function sliceDays(stats: DailyStats, from: string, to: string): DailyStats { + const out: DailyStats = new Map(); + for (const [type, days] of stats) { + const kept = new Map(); + for (const [day, value] of days) { + if (day >= from && day <= to) kept.set(day, value); + } + out.set(type, kept); + } + return out; +} + /** * The observations for a window of local days `[from, to]`. * @@ -212,20 +304,18 @@ export async function loadNotableRange(args: { (t) => args.typeVisible(t) && (wanted === null || wanted.has(t)), ); const firstTypes = (args.firstValueTypes ?? types).filter(args.typeVisible); - let historyFrom = startOfLocalDayKey( - shiftDateKey(args.from, -NOTABLE_LOOKBACK_DAYS), - args.tz, - ); - if (args.floor && args.floor > historyFrom) historyFrom = args.floor; const [stats, firsts] = await Promise.all([ - readLocalDailyStats({ - userId: args.userId, - types, - from: historyFrom, - to: startOfLocalDayKey(shiftDateKey(args.to, 1), args.tz), - tz: args.tz, - priorityJson: args.priorityJson, - }), + types.length === 0 + ? new Map() + : readExtremeHistory({ + userId: args.userId, + readable: NOTABLE_EXTREME_TYPES.filter(args.typeVisible), + from: shiftDateKey(args.from, -NOTABLE_LOOKBACK_DAYS), + to: args.to, + tz: args.tz, + priorityJson: args.priorityJson, + floor: args.floor ?? null, + }), readFirstReadings(args.userId, firstTypes), ]); diff --git a/src/lib/insights/__tests__/features.test.ts b/src/lib/insights/__tests__/features.test.ts index f43d56adf..0f582b50c 100644 --- a/src/lib/insights/__tests__/features.test.ts +++ b/src/lib/insights/__tests__/features.test.ts @@ -682,3 +682,24 @@ describe("extractFeatures — a day of pulse is the mean of its hours' means", ( expect(f.historicalComparison?.pulse?.current7dAvg).toBe(71.25); }); }); + +describe("extractFeatures — blood-pressure spread", () => { + it("reports the same population spread the series API shows", async () => { + // 120 / 130 / 140 over the last three days: population SD 8.165, the + // sample SD (the old rule here) would be 10. + const day = 24 * 60 * 60 * 1000; + const rows = [120, 130, 140].map((value, i) => ({ + type: "BLOOD_PRESSURE_SYS", + value, + measuredAt: new Date(Date.now() - (i + 1) * day), + sleepStage: null, + source: "MANUAL", + deviceType: null, + })); + prismaMock.measurement.findMany.mockResolvedValue(rows); + + const features = await extractFeatures("user-1", false); + + expect(features.bloodPressure?.sdSys30).toBe(8.2); + }); +}); diff --git a/src/lib/insights/__tests__/symptom-series-chronic.test.ts b/src/lib/insights/__tests__/symptom-series-chronic.test.ts new file mode 100644 index 000000000..c949b3380 --- /dev/null +++ b/src/lib/insights/__tests__/symptom-series-chronic.test.ts @@ -0,0 +1,78 @@ +/** + * A chronic condition does not zero-fill the symptom-severity series. + * + * The series fills healthy days with 0 across an episode's span. A chronic + * condition has no end, so its span covered the whole window and the engine + * correlated a near-constant run of zeros. Only episodes that can put the + * account into Rest Mode carry a span; the fake store below honours the + * lifecycle filter the way Postgres would. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +type Episode = { + id: string; + lifecycle: string; + onsetAt: Date; + resolvedAt: Date | null; +}; +const store: { episodes: Episode[] } = { episodes: [] }; + +vi.mock("@/lib/db", () => ({ + prisma: { + illnessEpisode: { + findMany: vi.fn( + async (args: { where: { lifecycle?: { not: string } } }) => + store.episodes.filter( + (e) => + args.where.lifecycle === undefined || + e.lifecycle !== args.where.lifecycle.not, + ), + ), + }, + illnessDayLog: { findMany: vi.fn(async () => []) }, + }, +})); + +import { fetchSymptomSeries } from "../correlation-channel-series"; + +const DAY = 24 * 60 * 60 * 1000; +const since = new Date(Date.now() - 60 * DAY); + +beforeEach(() => { + store.episodes = []; +}); + +describe("fetchSymptomSeries", () => { + it("has no series for a chronic condition alone", async () => { + store.episodes = [ + { + id: "chronic", + lifecycle: "CHRONIC_ONGOING", + onsetAt: new Date(Date.now() - 900 * DAY), + resolvedAt: null, + }, + ]; + const series = await fetchSymptomSeries("user-1", "UTC", since); + expect(series.points).toEqual([]); + }); + + it("fills only the acute episode's span beside a chronic one", async () => { + store.episodes = [ + { + id: "chronic", + lifecycle: "CHRONIC_ONGOING", + onsetAt: new Date(Date.now() - 900 * DAY), + resolvedAt: null, + }, + { + id: "cold", + lifecycle: "ACUTE", + onsetAt: new Date(Date.now() - 20 * DAY), + resolvedAt: new Date(Date.now() - 14 * DAY), + }, + ]; + const series = await fetchSymptomSeries("user-1", "UTC", since); + expect(series.points.length).toBeGreaterThanOrEqual(6); + expect(series.points.length).toBeLessThanOrEqual(8); + }); +}); diff --git a/src/lib/insights/comprehensive-aggregator.ts b/src/lib/insights/comprehensive-aggregator.ts index 63f22ec82..d6b27a377 100644 --- a/src/lib/insights/comprehensive-aggregator.ts +++ b/src/lib/insights/comprehensive-aggregator.ts @@ -114,6 +114,7 @@ import { type AccumulatorBucketRow, } from "@/lib/rollups/measurement-read"; import { windowWeighting } from "@/lib/measurements/day-statistic"; +import { readLatestCanonical } from "@/lib/measurements/latest-canonical"; import { SESSION_DAY_FRAME, dayWeightedRows, @@ -198,11 +199,9 @@ interface NarrowAggregateRow { // they no longer scan the raw measurements partition on the warm path. } -/** Latest raw value per type (DISTINCT ON over the 90-day window). */ -interface LatestRow { - type: string; - value: number; - measured_at: Date; +/** The latest canonical reading per type is read only inside the 90-day window. */ +function latestFloor(): Date { + return new Date(Date.now() - 90 * 24 * 60 * 60 * 1000); } /** @@ -426,8 +425,7 @@ async function buildFromRollups( // (`type: { in: [...] }`) plus per-sub-query wall-clock timings. // v1.11.1 — collapse overlapping sources to the ladder-canonical reading per // (type, day) before aggregating. `rankUnqualified` ranks the inner - // canonical-source subquery; `rankM` ranks the latest read's `m.`-aliased - // columns. The window_stats CTE + the main narrow both read from the + // canonical-source subquery. The window_stats CTE + the main narrow both read from the // canonical-source-filtered set so a dual-source vital isn't blended. const priorityJson = await loadUserSourcePriority(userId); const rankUnqualified = buildSourceRankCase( @@ -435,7 +433,6 @@ async function buildFromRollups( '"type"', '"source"', ); - const rankM = buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'); const [narrows, latests, dayBuckets, bpRawRows, firstRows] = await Promise.all([ @@ -484,20 +481,7 @@ async function buildFromRollups( ), ), timeSubquery(timings, "latests", () => - prisma.$queryRawUnsafe( - ` - SELECT DISTINCT ON (m."type") - m."type"::text AS type, - m."value"::double precision AS value, - m."measured_at" AS measured_at - FROM measurements m - WHERE m."user_id" = $1 - AND m."measured_at" >= (NOW() AT TIME ZONE 'UTC') - INTERVAL '90 days' - AND m."deleted_at" IS NULL - ORDER BY m."type", date_trunc('day', m."measured_at") DESC, (${rankM}), m."measured_at" DESC - `, - userId, - ), + readLatestCanonical(userId, priorityJson, latestFloor()), ), // v1.4.35 — DAY rollup buckets for the trailing 90 days, every // type. Drives both the per-type `count / min / max / mean` @@ -676,7 +660,6 @@ async function buildFromLiveAggregate( '"type"', '"source"', ); - const rankM = buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'); const [aggregates, latests, dayBuckets, bpRawRows] = await Promise.all([ timeSubquery(timings, "heavy", () => @@ -760,20 +743,7 @@ async function buildFromLiveAggregate( ), ), timeSubquery(timings, "latests", () => - prisma.$queryRawUnsafe( - ` - SELECT DISTINCT ON (m."type") - m."type"::text AS type, - m."value"::double precision AS value, - m."measured_at" AS measured_at - FROM measurements m - WHERE m."user_id" = $1 - AND m."measured_at" >= (NOW() AT TIME ZONE 'UTC') - INTERVAL '90 days' - AND m."deleted_at" IS NULL - ORDER BY m."type", date_trunc('day', m."measured_at") DESC, (${rankM}), m."measured_at" DESC - `, - userId, - ), + readLatestCanonical(userId, priorityJson, latestFloor()), ), // Buckets may exist for some types even when the COUNT probe came // back zero (race window between the probe and a sibling diff --git a/src/lib/insights/correlation-channel-series.ts b/src/lib/insights/correlation-channel-series.ts index 57635b873..d383307fe 100644 --- a/src/lib/insights/correlation-channel-series.ts +++ b/src/lib/insights/correlation-channel-series.ts @@ -20,6 +20,7 @@ * fabricating a constant. */ import { prisma } from "@/lib/db"; +import { REST_MODE_EPISODE_WHERE } from "@/lib/illness/rest-mode"; import { buildComplianceMedicationContext, COURSES_COMPLIANCE_SELECT, @@ -152,9 +153,9 @@ export async function fetchComplianceSeries( * v1.21.0 (FDREXTEND) — build the user's SYMPTOM_SEVERITY daily series in the * `outcome` role (callers that need the behaviour role re-tag the returned * series — the points are role-invariant). Reads every in-window illness episode - * + its day-logs; the builder zero-fills healthy days ONLY across real episode - * spans, so a user with no episodes yields an empty series that degrades to - * absent. + * that is not a chronic condition + its day-logs; the builder zero-fills healthy + * days ONLY across those spans, so a user with no such episode yields an empty + * series that degrades to absent. */ export async function fetchSymptomSeries( userId: string, @@ -165,11 +166,15 @@ export async function fetchSymptomSeries( const episodes = await prisma.illnessEpisode.findMany({ // An episode overlaps the window when it onset before `now` and either is // ongoing or resolved at/after the window start. + // A chronic condition has no end, so its span would zero-fill every day + // of the window and hand the engine a near-constant series. Only the + // episodes that can put the account into Rest Mode carry a span. where: { userId, deletedAt: null, onsetAt: { lte: now }, OR: [{ resolvedAt: null }, { resolvedAt: { gte: since } }], + ...REST_MODE_EPISODE_WHERE, }, select: { id: true, onsetAt: true, resolvedAt: true }, }); diff --git a/src/lib/insights/features.ts b/src/lib/insights/features.ts index 091714530..ee6c70f76 100644 --- a/src/lib/insights/features.ts +++ b/src/lib/insights/features.ts @@ -30,9 +30,14 @@ import { import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; import { getBpTargets } from "@/lib/analytics/bp-targets"; import { isBpReadingInTarget } from "@/lib/analytics/bp-in-target"; +import { + READING_SPREAD, + standardDeviation, + type VarianceKind, +} from "@/lib/stats/spread"; import { pairByTimestamp, - pearsonCorrelation, + significantPearsonCorrelation, type CorrelationResult, } from "@/lib/analytics/correlations"; import { getMedicationCategories } from "@/lib/medication-category"; @@ -894,12 +899,17 @@ export class FeaturesPayloadTooLargeError extends Error { } } -export function stdDev(values: number[]): number | null { +/** + * A spread rounded to the payload's one decimal, `null` below two values: a + * single reading has no spread to report, not a spread of zero. + */ +export function roundedSpread( + values: readonly number[], + kind: VarianceKind, +): number | null { if (values.length < 2) return null; - const mean = values.reduce((s, v) => s + v, 0) / values.length; - const variance = - values.reduce((s, v) => s + (v - mean) ** 2, 0) / (values.length - 1); - return Math.round(Math.sqrt(variance) * 10) / 10; + const sd = standardDeviation(values, kind); + return sd === null ? null : Math.round(sd * 10) / 10; } export function toDataPoints( @@ -1468,14 +1478,14 @@ export async function extractFeatures( const vals = sysData .filter((m) => m.measuredAt.getTime() >= fromMs) .map((m) => m.value); - return stdDev(vals); + return roundedSpread(vals, READING_SPREAD); })(), sdDia30: (() => { const fromMs = now - 30 * 24 * 60 * 60 * 1000; const vals = diaData .filter((m) => m.measuredAt.getTime() >= fromMs) .map((m) => m.value); - return stdDev(vals); + return roundedSpread(vals, READING_SPREAD); })(), pulsePressure30: (() => { const avgSys = sysSummary?.avg30 ?? null; @@ -1656,7 +1666,7 @@ export async function extractFeatures( })); const computeCorr = (a: DataPoint[], b: DataPoint[]) => - pearsonCorrelation(pairByTimestamp(a, b)); + significantPearsonCorrelation(pairByTimestamp(a, b)); const sleepPoints = toDataPoints(sleepData); diff --git a/src/lib/insights/metric-signal.ts b/src/lib/insights/metric-signal.ts index 1dbf261fd..8f943b256 100644 --- a/src/lib/insights/metric-signal.ts +++ b/src/lib/insights/metric-signal.ts @@ -25,6 +25,7 @@ */ import type { GradedSeries } from "@/lib/insights/graded-series"; import type { NormRange } from "@/lib/insights/derived/norms"; +import { standardDeviation } from "@/lib/stats/spread"; /** Favourable-direction framing — mirrors the metric-status registry. */ export type MetricDirection = "higher-better" | "lower-better" | "target-band"; @@ -96,16 +97,6 @@ function meanOf(values: number[]): number | null { return sum / values.length; } -/** Sample standard deviation, or null with fewer than two points. */ -function sampleSd(values: number[]): number | null { - if (values.length < 2) return null; - const m = meanOf(values); - if (m === null) return null; - let acc = 0; - for (const v of values) acc += (v - m) ** 2; - return Math.sqrt(acc / (values.length - 1)); -} - /** Placement of a value against a coarse normal range. */ export function placeInBand( value: number, @@ -169,7 +160,8 @@ export function buildMetricSignal( } const baseline = meanOf(baselineMeans); - const spread = sampleSd(baselineMeans); + // The spread decides the verdict, so it is the sample estimate. + const spread = standardDeviation(baselineMeans, "sample"); let delta: number | null = null; let deltaPct: number | null = null; diff --git a/src/lib/insights/mood-aggregates.ts b/src/lib/insights/mood-aggregates.ts index d8f4e1fa7..333e7a60d 100644 --- a/src/lib/insights/mood-aggregates.ts +++ b/src/lib/insights/mood-aggregates.ts @@ -428,9 +428,9 @@ export interface MoodMetricCorrelation { * * This routes through `significantPearsonCorrelation` — n >= 20 AND * p < 0.05 — for the same reason the weight, blood-pressure and - * mood-status surfaces do: the plain `pearsonCorrelation` bands from |r| - * alone above a floor of five pairs, so five days of noise sloping the - * right way arrive at the reader labelled "strong". A mood page is where + * mood-status surfaces do: a band read from |r| alone above a floor of + * five pairs lets five days of noise sloping the right way arrive at the + * reader labelled "strong". A mood page is where * that costs the most, because the reader is looking for something to * change about their week. */ diff --git a/src/lib/insights/mood-patterns.ts b/src/lib/insights/mood-patterns.ts index 7dc5fb218..289748095 100644 --- a/src/lib/insights/mood-patterns.ts +++ b/src/lib/insights/mood-patterns.ts @@ -20,6 +20,7 @@ import { import { round } from "@/lib/insights/status-shared"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; import { wallClockInTz } from "@/lib/tz/wall-clock"; +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; // --- In-target % over the last 30 daily points (lifted from mood-status) --- @@ -289,11 +290,11 @@ export function computeMoodStability( ): MoodStability | null { if (daily.length < STABILITY_MIN_DAYS) return null; - const values = daily.map((bucket) => bucket.value); - const avg = values.reduce((sum, v) => sum + v, 0) / values.length; - const variance = - values.reduce((sum, v) => sum + (v - avg) ** 2, 0) / values.length; - const stdDev = Math.sqrt(variance); + const stdDev = + standardDeviation( + daily.map((bucket) => bucket.value), + READING_SPREAD, + ) ?? 0; const normalised = Math.min(stdDev, STABILITY_SD_FULL_SCALE) / STABILITY_SD_FULL_SCALE; diff --git a/src/lib/insights/seasonal-adjust.ts b/src/lib/insights/seasonal-adjust.ts index 4db5e1c67..5be724265 100644 --- a/src/lib/insights/seasonal-adjust.ts +++ b/src/lib/insights/seasonal-adjust.ts @@ -29,6 +29,8 @@ * the regression runs over the days that exist and nothing is imputed. */ +import { variance as varianceOf } from "@/lib/stats/spread"; + /** Span (first to last paired day, inclusive) below which only the trend is removed. */ export const SEASONAL_MIN_SPAN_DAYS = 120; @@ -205,10 +207,11 @@ function densify( dense[days[i] - lo] = values[i]; sum += values[i]; } - const mean = sum / values.length; - let ss = 0; - for (const v of values) ss += (v - mean) ** 2; - return { dense, mean, variance: ss / values.length }; + return { + dense, + mean: sum / values.length, + variance: varianceOf(values, "population") ?? 0, + }; } /** diff --git a/src/lib/insights/signals-of-day.ts b/src/lib/insights/signals-of-day.ts index b522db563..3be2c7e0c 100644 --- a/src/lib/insights/signals-of-day.ts +++ b/src/lib/insights/signals-of-day.ts @@ -9,7 +9,11 @@ * so every existing call site keeps importing from there. */ import { trendSlope } from "@/lib/analytics/trends"; -import { avgInWindow, stdDev, toDataPoints } from "@/lib/insights/features"; +import { + avgInWindow, + roundedSpread, + toDataPoints, +} from "@/lib/insights/features"; import { SAME_HOURS_MIN_DAYS, SAME_HOURS_TYPES, @@ -177,7 +181,11 @@ function buildSignal( // spread and the anomaly stay over the readings. const avg7 = avgInWindow(records, now, 7, 0, day); const avg30 = avgInWindow(records, now, 30, 0, day); - const spread30 = stdDev(win30.map((rec) => rec.value)); + // The spread decides the verdict below, so it is the sample estimate. + const spread30 = roundedSpread( + win30.map((rec) => rec.value), + "sample", + ); const deltaVs7 = avg7 !== null ? shown(newest.value - avg7) : null; const deltaVs30 = avg30 !== null ? shown(newest.value - avg30) : null; const outsideNormalSwing = @@ -310,7 +318,10 @@ function buildSameHoursSignal( const avg30 = mean(earlier.map((d) => d.value)); const avg7 = mean(earlier.filter((d) => d.age <= 7).map((d) => d.value)); - const spread30 = stdDev(earlier.map((d) => d.value)); + const spread30 = roundedSpread( + earlier.map((d) => d.value), + "sample", + ); const latest = today.value; const outsideNormalSwing = avg30 !== null && spread30 !== null && spread30 > 0 diff --git a/src/lib/mcp/rich-reads.ts b/src/lib/mcp/rich-reads.ts index 4c7318286..53329ae9e 100644 --- a/src/lib/mcp/rich-reads.ts +++ b/src/lib/mcp/rich-reads.ts @@ -63,6 +63,7 @@ import { sanitizeValueText } from "@/lib/ai/coach/labs-snapshot"; import { scrubFenceMarkers } from "@/lib/ai/coach/data-fence"; import { encodeOffsetCursor } from "@/lib/mcp/pagination"; import type { CoachScopeWindow } from "@/lib/ai/coach/types"; +import { standardDeviation } from "@/lib/stats/spread"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -1158,11 +1159,9 @@ function mean(xs: number[]): number { return xs.reduce((s, x) => s + x, 0) / xs.length; } +/** The CUSUM threshold estimates the series' noise, so it is the sample SD. */ function stdDev(xs: number[]): number { - if (xs.length < 2) return 0; - const m = mean(xs); - const variance = xs.reduce((s, x) => s + (x - m) ** 2, 0) / (xs.length - 1); - return Math.sqrt(variance); + return standardDeviation(xs, "sample") ?? 0; } /** diff --git a/src/lib/measurements/latest-canonical.ts b/src/lib/measurements/latest-canonical.ts new file mode 100644 index 000000000..a8e7f3ec9 --- /dev/null +++ b/src/lib/measurements/latest-canonical.ts @@ -0,0 +1,66 @@ +import { prisma } from "@/lib/db"; +import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; +import { measurementTypeEnum } from "@/lib/validations/measurement"; + +export interface LatestCanonicalRow { + type: string; + value: number; + measured_at: Date; +} + +/** + * The latest canonical reading per measurement type: the latest UTC day + * first, then the source ladder (the canonical source wins), then the latest + * reading of that source, so a tile's headline matches its chart. `since` + * drops a type whose latest reading is older than it. + * + * Driven from the type list rather than a `DISTINCT ON (type)` over the + * account: Postgres 16 has no skip scan, so that shape sorted every live row + * of the account to keep one per type. Here each type costs one probe of + * `measurements_live_covering_idx` for its latest instant and one read of that + * single day. The closed day window matters to the planner as much as to the + * result: with only the lower bound it estimates a third of the type's rows + * per probe, and the summed estimate crosses `jit_above_cost`, whose + * compilation then costs more than the query itself. + */ +export function readLatestCanonical( + userId: string, + priorityJson: unknown, + since: Date | null = null, +): Promise { + const rank = buildSourceRankCase(priorityJson, 'm."type"', 'm."source"'); + return prisma.$queryRawUnsafe( + ` + SELECT + mt."type"::text AS type, + l.value AS value, + l.measured_at AS measured_at + FROM unnest($2::"measurement_type"[]) AS mt("type") + CROSS JOIN LATERAL ( + SELECT date_trunc('day', d."measured_at") AS day + FROM measurements d + WHERE d."user_id" = $1 + AND d."type" = mt."type" + AND d."deleted_at" IS NULL + ${since ? `AND d."measured_at" >= $3` : ""} + ORDER BY d."measured_at" DESC + LIMIT 1 + ) last_day + CROSS JOIN LATERAL ( + SELECT + m."value"::double precision AS value, + m."measured_at" AS measured_at + FROM measurements m + WHERE m."user_id" = $1 + AND m."type" = mt."type" + AND m."deleted_at" IS NULL + AND m."measured_at" >= last_day.day + AND m."measured_at" < last_day.day + INTERVAL '1 day' + ORDER BY (${rank}), m."measured_at" DESC, m."id" DESC + LIMIT 1 + ) l + ORDER BY mt."type" + `, + ...[userId, measurementTypeEnum.options, ...(since ? [since] : [])], + ); +} diff --git a/src/lib/measurements/pick-canonical-workout-rows.ts b/src/lib/measurements/pick-canonical-workout-rows.ts index cb97fd0ad..50fda8fdf 100644 --- a/src/lib/measurements/pick-canonical-workout-rows.ts +++ b/src/lib/measurements/pick-canonical-workout-rows.ts @@ -14,11 +14,9 @@ * logical run start within a minute of each other on both * sources; the rounding gives the cross-source matcher slack * without collapsing distinct back-to-back workouts. - * 2. For each bucket, walk the SOURCE priority ladder - * (`APPLE_HEALTH ≻ WITHINGS ≻ MANUAL ≻ IMPORT` per the existing - * source-priority validator's default) and pick the FIRST - * source that has any row in that bucket. Keep every row from - * the winning source; drop the rest. + * 2. For each bucket, walk the source ladder (`workoutSourceLadder`) + * and pick the FIRST source that has any row in that bucket. Keep + * every row from the winning source; drop the rest. * 3. Rows whose source isn't in the ladder pass through untouched * (fallback intent — never silently drop signal). * @@ -26,10 +24,10 @@ * (typically `ORDER BY startedAt ASC, id ASC` from the caller's * Prisma read). The output preserves input order. * - * Future v1.5.x tune: metric-aware picker per R-F open question #4 — - * route → Apple wins, HR zones → Withings wins. Out of scope for - * v1.4.30; the existing ladder ships first so the dashboard tile is - * unambiguous. + * This is the one workout picker. The reads use it, and so does the batch + * write route before it inserts: a write-time picker with its own window + * and a different default ladder kept or dropped a different row than the + * list then showed. */ import type { MeasurementSource } from "@/generated/prisma/client"; @@ -249,6 +247,17 @@ function enrichWinnersFromWhoop( return enriched; } +/** + * The source order for workouts: the `steps` metric's ladder, the + * canonical-activity bucket the Sources settings configure, falling back to + * its default when the user saved none or it does not parse. + */ +export function workoutSourceLadder( + userPriorityJson: unknown, +): readonly MeasurementSource[] { + return getSourceLadder(parseSourcePriority(userPriorityJson), "steps"); +} + /** * Pick canonical rows across sources for a user's workouts. * @@ -266,14 +275,7 @@ export function pickCanonicalWorkoutRows( ): T[] { if (rows.length === 0) return []; - // Workouts ride the `steps` metric's source ladder — the - // canonical-activity bucket. The user's iOS Settings → Sources UI - // configures the same ladder for the workout dashboard tile; - // unifying on "steps" keeps a single configuration surface rather - // than introducing a new `workouts` metric key. Future ladders for - // workout-specific tunes (route-vs-zones) ride a v1.5.x change. - const resolved = parseSourcePriority(userPriorityJson); - const sourceLadder = getSourceLadder(resolved, "steps"); + const sourceLadder = workoutSourceLadder(userPriorityJson); const buckets = new Map< string, diff --git a/src/lib/medications/scheduling/__tests__/compliance.test.ts b/src/lib/medications/scheduling/__tests__/compliance.test.ts index 6291df5ad..dae3cd298 100644 --- a/src/lib/medications/scheduling/__tests__/compliance.test.ts +++ b/src/lib/medications/scheduling/__tests__/compliance.test.ts @@ -8,16 +8,29 @@ import { describe, expect, it } from "vitest"; import { complianceChips, streaksFromTimeline } from "../compliance"; import { buildCadenceTimeline } from "../cadence"; -import type { - CadenceEngineContext, - IntakeEventLike, - ScheduleLike, -} from "../cadence"; +import type { IntakeEventLike, ScheduleLike } from "../cadence"; +import type { ComplianceMedicationContext } from "@/lib/analytics/compliance/types"; +import { buildMedicationComplianceBundle } from "@/lib/analytics/compliance/display"; function d(iso: string): Date { return new Date(iso); } +/** The medication context the chips read; UTC unless a test says otherwise. */ +function ctx( + over?: Partial, +): ComplianceMedicationContext { + return { + startsOn: null, + endsOn: null, + oneShot: false, + createdAt: d("2025-05-01T00:00:00Z"), + lastIntakeAt: null, + timeZone: "UTC", + ...over, + }; +} + const DAILY_8AM: ScheduleLike = { windowStart: "08:00", windowEnd: "09:00", @@ -27,7 +40,7 @@ const DAILY_8AM: ScheduleLike = { describe("complianceChips", () => { it("returns null adherence when no doses are expected in the window", () => { const NOW = d("2025-06-10T12:00:00"); - const result = complianceChips([], [], NOW, 30); + const result = complianceChips([], [], ctx(), NOW, 30); expect(result.adherenceRate).toBeNull(); expect(result.missedLast30).toBe(0); }); @@ -44,14 +57,14 @@ describe("complianceChips", () => { } // 5-day window: 5 expected, 5 taken, 0 missed (today's slot is upcoming // because NOW is 12:00 and window is 08:00-09:00... actually past today) - const result = complianceChips([DAILY_8AM], events, NOW, 5); + const result = complianceChips([DAILY_8AM], events, ctx(), NOW, 5); expect(result.adherenceRate).toBeGreaterThanOrEqual(80); expect(result.missedLast30).toBeLessThanOrEqual(1); }); it("computes 0% adherence when no events match any past slot", () => { const NOW = d("2025-06-10T12:00:00"); - const result = complianceChips([DAILY_8AM], [], NOW, 5); + const result = complianceChips([DAILY_8AM], [], ctx(), NOW, 5); expect(result.adherenceRate).toBe(0); expect(result.missedLast30).toBeGreaterThan(0); }); @@ -60,25 +73,25 @@ describe("complianceChips", () => { const NOW = d("2025-06-10T12:00:00"); // Window: NOW - 3d = 2025-06-07T12:00 → past slots: 08, 09, 10 // Provide one taken + one skipped + one taken for the three slots so - // the denominator excludes the skipped one. + // the denominator excludes the skipped one. A skip names its slot. const events: IntakeEventLike[] = [ { - scheduledFor: d("2025-06-08T08:30:00"), + scheduledFor: d("2025-06-08T08:00:00"), takenAt: d("2025-06-08T08:35:00"), skipped: false, }, { - scheduledFor: d("2025-06-09T08:30:00"), + scheduledFor: d("2025-06-09T08:00:00"), takenAt: null, skipped: true, }, { - scheduledFor: d("2025-06-10T08:30:00"), + scheduledFor: d("2025-06-10T08:00:00"), takenAt: d("2025-06-10T08:35:00"), skipped: false, }, ]; - const result = complianceChips([DAILY_8AM], events, NOW, 3); + const result = complianceChips([DAILY_8AM], events, ctx(), NOW, 3); // 2 taken / (2 taken + 0 missed) = 100, skipped excluded. expect(result.adherenceRate).toBe(100); }); @@ -97,7 +110,7 @@ describe("complianceChips", () => { skipped: false, }); } - const result = complianceChips([DAILY_8AM], events, NOW, 14); + const result = complianceChips([DAILY_8AM], events, ctx(), NOW, 14); expect(result.currentStreak).toBeGreaterThanOrEqual(4); }); @@ -132,7 +145,7 @@ describe("complianceChips", () => { skipped: false, }, ]; - const result = complianceChips([DAILY_8AM], events, NOW, 7); + const result = complianceChips([DAILY_8AM], events, ctx(), NOW, 7); // Streak ending at today = day 9 + day 10 = 2 (day 8 missed broke it). expect(result.currentStreak).toBeLessThanOrEqual(2); // Longest streak in window = max(3 from 5..7, 2 from 9..10) = 3. @@ -158,7 +171,7 @@ describe("complianceChips", () => { skipped: false, }, ]; - const result = complianceChips([weekly], events, NOW, 14); + const result = complianceChips([weekly], events, ctx(), NOW, 14); expect(result.adherenceRate).toBe(100); expect(result.missedLast30).toBe(0); }); @@ -170,7 +183,7 @@ describe("complianceChips", () => { daysOfWeek: "1", // Monday only }; const NOW = d("2025-06-10T12:00:00"); // Tuesday - const result = complianceChips([weekly], [], NOW, 14); + const result = complianceChips([weekly], [], ctx(), NOW, 14); expect(result.adherenceRate).toBe(0); expect(result.missedLast30).toBeGreaterThan(0); }); @@ -199,18 +212,16 @@ describe("complianceChips", () => { const berlin = complianceChips( [sched], events, + ctx({ timeZone: "Europe/Berlin" }), NOW, 7, - undefined, - "Europe/Berlin", ); const tokyo = complianceChips( [sched], events, + ctx({ timeZone: "Asia/Tokyo" }), NOW, 7, - undefined, - "Asia/Tokyo", ); // Both must compute a finite adherence rate (the schedule has @@ -226,23 +237,10 @@ describe("complianceChips", () => { }); describe("complianceChips — canonical-engine delegation (v1.7.0 SB-SCHED-2)", () => { - function engineCtx( - over?: Partial, - ): CadenceEngineContext { - return { - startsOn: null, - endsOn: null, - oneShot: false, - createdAt: d("2025-05-01T00:00:00Z"), - lastIntakeAt: null, - timeZone: "Europe/Berlin", - ...over, - }; - } + const engineCtx = (over?: Partial) => + ctx({ timeZone: "Europe/Berlin", ...over }); - it("RRULE weekly Monday: legacy walker counts every day missed, engine counts only Mondays", () => { - // `daysOfWeek = null` reads as every-day to the legacy walker, so an - // RRULE-weekly schedule over-counts expected doses without the engine. + it("RRULE weekly Monday: only the Mondays in the window are expected", () => { const rruleWeekly: ScheduleLike = { windowStart: "08:00", windowEnd: "09:00", @@ -250,25 +248,15 @@ describe("complianceChips — canonical-engine delegation (v1.7.0 SB-SCHED-2)", rrule: "FREQ=WEEKLY;BYDAY=MO", timesOfDay: ["08:00"], }; - const NOW = d("2025-06-10T12:00:00Z"); // Tuesday + // Saturday: both Mondays in the window are past a weekly dose's late tail. + const NOW = d("2025-06-14T12:00:00Z"); - // Legacy path (no engineCtx) expands daily → many missed days. - const legacy = complianceChips([rruleWeekly], [], NOW, 14); - // Canonical path (engineCtx) expands only Mondays in the window. - const engine = complianceChips( - [rruleWeekly], - [], - NOW, - 14, - undefined, - "Europe/Berlin", - engineCtx(), - ); + const engine = complianceChips([rruleWeekly], [], engineCtx(), NOW, 14); - // The engine path expects far fewer doses (only the Mondays in a - // 14-day window: 2) than the daily legacy expansion (~13). - expect(engine.missedLast30).toBeLessThan(legacy.missedLast30); - expect(engine.missedLast30).toBeLessThanOrEqual(2); + // Two Mondays in a 14-day window, both missed; `daysOfWeek = null` + // must not read as every day. + expect(engine.missedLast30).toBe(2); + expect(engine.adherenceRate).toBe(0); }); it("PRN: engine short-circuits to zero expected doses (no adherence, no missed)", () => { @@ -281,15 +269,7 @@ describe("complianceChips — canonical-engine delegation (v1.7.0 SB-SCHED-2)", }; const NOW = d("2025-06-10T12:00:00Z"); - const engine = complianceChips( - [prn], - [], - NOW, - 30, - undefined, - "Europe/Berlin", - engineCtx(), - ); + const engine = complianceChips([prn], [], engineCtx(), NOW, 30); // PRN is as-needed — never projected, reminded, or counted in // compliance-expected. No expected slots → null adherence, zero missed. @@ -315,20 +295,16 @@ describe("complianceChips — canonical-engine delegation (v1.7.0 SB-SCHED-2)", const engine = complianceChips( [cyclic], [], + engineCtx({ startsOn: d("2025-05-01T00:00:00Z") }), NOW, 14, - undefined, - "Europe/Berlin", - engineCtx({ startsOn: d("2025-05-01T00:00:00Z") }), ); const everyDay = complianceChips( [{ ...cyclic, scheduleType: "SCHEDULED" }], [], + engineCtx({ startsOn: d("2025-05-01T00:00:00Z") }), NOW, 14, - undefined, - "Europe/Berlin", - engineCtx({ startsOn: d("2025-05-01T00:00:00Z") }), ); // The cyclic gate must drop the off-week's doses, so fewer missed @@ -445,3 +421,43 @@ describe("streaksFromTimeline — DST-correct day walk", () => { expect(current).toBe(7); }); }); + +describe("complianceChips — the detail page's rate", () => { + it("drops a pause from the chips exactly as the compliance payload does", () => { + // Daily 08:00, paused from 10 days ago to 3 days ago, every other dose + // taken. The chips used to tally the paused week as seven missed doses + // (77 %) while the detail page, reading the compliance ledger, said 100. + const NOW = d("2025-06-30T12:00:00Z"); + const day = (n: number) => new Date(NOW.getTime() - n * 86_400_000); + const pausedAt = day(10); + const resumedAt = day(3); + const events: IntakeEventLike[] = []; + for (let n = 1; n <= 30; n++) { + const slot = new Date(day(n).setUTCHours(8, 0, 0, 0)); + if (slot >= pausedAt && slot < resumedAt) continue; + events.push({ scheduledFor: slot, takenAt: slot, skipped: false }); + } + const daily: ScheduleLike = { + windowStart: "08:00", + windowEnd: "09:00", + daysOfWeek: null, + timesOfDay: ["08:00"], + }; + const context = ctx({ + createdAt: d("2025-01-01T00:00:00Z"), + pauseEras: [{ pausedAt, resumedAt }], + }); + + const chips = complianceChips([daily], events, context, NOW, 30); + const bundle = buildMedicationComplianceBundle( + events, + [daily], + context, + NOW, + ); + + expect(chips.missedLast30).toBe(0); + expect(chips.adherenceRate).toBe(100); + expect(chips.adherenceRate).toBe(bundle.compliance30.rate); + }); +}); diff --git a/src/lib/medications/scheduling/compliance.ts b/src/lib/medications/scheduling/compliance.ts index 8a8157db6..2eff7aee1 100644 --- a/src/lib/medications/scheduling/compliance.ts +++ b/src/lib/medications/scheduling/compliance.ts @@ -6,33 +6,28 @@ * rate, current streak (days), longest streak (days), missed in last * 30 days. Each chip is monochrome — maintainer note: no gamified badges. * - * Distinct from `src/lib/analytics/compliance.ts.calculateCompliance`: - * that helper computes against expected counts (schedules-per-day × - * days) and is used by the existing /api/medications/[id]/compliance - * route. This module computes against the pair-matched timeline so - * the chips and the cadence chart agree on every single dose. + * Adherence and missed doses come from the compliance ledger the + * /api/medications/[id]/compliance route reads, so the chips and the + * detail page's rate agree; the streaks walk the pair-matched timeline + * the cadence chart draws. * - * v1.4.25 W21 Fix-O — accepts an optional IANA `timeZone` argument - * forwarded into the timeline + streak math so a user in Tokyo gets - * the same chip values as a user in Berlin even when the host's - * system time is set differently. Omitting the argument keeps the - * legacy system-local behaviour the W19e tests pin. + * The timeline and streak math run in the context's IANA zone, so a user + * in Tokyo gets the same chip values as a user in Berlin whatever the + * host's system time is. */ -import type { SlotBand } from "./attribution"; -import { type BandMinterMedication } from "./band-minter"; -import { buildBandsForSchedulesWithEras } from "./schedule-eras"; import { buildCadenceTimeline, startOfLocalDay, - type CadenceEngineContext, type IntakeEventLike, type PairedDose, type ScheduleLike, } from "./cadence"; -import { reconstructDoseHistory, type HistoryIntake } from "./dose-history"; -import { type CanonicalSchedule, type RecurrenceContext } from "./recurrence"; -import { normaliseDoseWindows } from "./worker-helpers"; +import { + trailingComplianceLedger, + trailingComplianceTally, +} from "@/lib/analytics/compliance/ledger"; +import type { ComplianceMedicationContext } from "@/lib/analytics/compliance/types"; import { userDayKey } from "@/lib/tz/resolver"; export interface ComplianceChips { @@ -146,134 +141,19 @@ export function streaksFromTimeline( } /** - * v1.15.18 — band-ledger tally counts for the detail-page chips. - * - * When the route threads an engine context (RRULE / rolling / cyclic / - * one-shot / PRN / legacy) the adherence rate + missed count come from the - * SAME unified dose-history ledger the compliance % and the history view - * read — so the chips can never contradict either. The streak still walks - * the cadence timeline (its day-grain "every dose taken or skipped" rule is - * orthogonal to per-dose attribution). Returns null when there is no engine - * context (a pure-math caller) — the caller then falls back to the legacy - * timeline tally. + * The four chips: adherence and missed doses from the compliance ledger the + * detail page's rate reads (`trailingComplianceTally`, with the medication's + * pause eras and courses), the streaks from the cadence timeline (a + * day-grain "every dose taken or skipped" rule, orthogonal to per-dose + * attribution). */ -function ledgerChipCounts( - schedules: ScheduleLike[], - events: IntakeEventLike[], - asOf: Date, - windowDays: number, - timeZone: string | undefined, - engineCtx: CadenceEngineContext | undefined, -): { taken: number; missed: number } | null { - if (!engineCtx) return null; - const DAY_MS = 24 * 60 * 60 * 1000; - const from = new Date(asOf.getTime() - windowDays * DAY_MS); - const userTz = engineCtx.timeZone || timeZone || "UTC"; - - const medication: BandMinterMedication = { - id: "chips-tally", - startsOn: engineCtx.startsOn, - endsOn: engineCtx.endsOn, - oneShot: engineCtx.oneShot, - createdAt: engineCtx.createdAt, - }; - const recurrenceCtx: RecurrenceContext = { - medication: { - id: "chips-tally", - startsOn: engineCtx.startsOn, - endsOn: engineCtx.endsOn, - oneShot: engineCtx.oneShot, - createdAt: engineCtx.createdAt, - }, - timeZone: userTz, - lastIntakeAt: engineCtx.lastIntakeAt, - // v1.40 (#1024) — a medication: `startsOn` is a course start. - rollingAnchor: "courseStart", - }; - const canonicalSchedules: CanonicalSchedule[] = schedules.map((s, i) => { - const base: CanonicalSchedule = { - id: s.id ?? `chips-${i}`, - rrule: s.rrule ?? null, - rollingIntervalDays: s.rollingIntervalDays ?? null, - timesOfDay: s.timesOfDay ?? [], - daysOfWeek: s.daysOfWeek ?? null, - windowStart: s.windowStart, - windowEnd: s.windowEnd, - reminderGraceMinutes: s.reminderGraceMinutes ?? null, - scheduleType: s.scheduleType ?? "SCHEDULED", - cyclicOnWeeks: s.cyclicOnWeeks ?? null, - cyclicOffWeeks: s.cyclicOffWeeks ?? null, - doseWindows: normaliseDoseWindows(s.doseWindows), - }; - // Surface windowStart as the single time-of-day for a legacy daily row - // so the band minter's cadence gate mints its daily band (see the - // analytics/compliance tally for the same normalisation). - if ( - base.timesOfDay.length === 0 && - base.rrule === null && - base.rollingIntervalDays === null && - base.scheduleType !== "PRN" && - !engineCtx.oneShot - ) { - return { ...base, timesOfDay: [base.windowStart] }; - } - return base; - }); - - const intakeInstants = events - .filter((e) => !e.skipped && e.takenAt !== null && e.takenAt <= asOf) - .map((e) => e.takenAt as Date) - .sort((a, b) => a.getTime() - b.getTime()); - - const groups = buildBandsForSchedulesWithEras({ - medication, - schedules: canonicalSchedules, - revisions: engineCtx.scheduleRevisions ?? [], - ctx: recurrenceCtx, - userTz, - range: { from, to: asOf }, - now: asOf, - intakeInstants, - }); - const bands: SlotBand[] = []; - for (const g of groups) { - if (g.hasExpectedSlots) bands.push(...g.bands); - } - - const intakes: HistoryIntake[] = events - .filter((e) => e.scheduledFor >= from && e.scheduledFor <= asOf) - .map((e) => ({ - scheduledFor: e.scheduledFor, - takenAt: e.takenAt, - skipped: e.skipped, - autoMissed: e.autoMissed ?? false, - // v1.15.20 — pinned takes bind by anchor (taken-late, slot served). - pinned: e.attributionSource === "USER_PIN", - })); - - const rows = reconstructDoseHistory( - bands, - intakes, - asOf, - engineCtx.createdAt, - ); - let taken = 0; - let missed = 0; - for (const row of rows) { - if (row.status === "taken_on_time" || row.status === "taken_late") taken++; - else if (row.status === "missed") missed++; - } - return { taken, missed }; -} - export function complianceChips( schedules: ScheduleLike[], events: IntakeEventLike[], + ctx: ComplianceMedicationContext, asOf: Date, windowDays = 30, anchor?: Date, - timeZone?: string, - engineCtx?: CadenceEngineContext, ): ComplianceChips { const timeline = buildCadenceTimeline( schedules, @@ -281,33 +161,21 @@ export function complianceChips( asOf, windowDays, anchor, - timeZone, - engineCtx, + ctx.timeZone, + ctx, ); - // v1.15.18 — adherence + missed come from the unified band ledger when an - // engine context is present (so the chips agree with the % and the history - // view); the legacy timeline tally is the fallback for pure-math callers. - const ledger = ledgerChipCounts( - schedules, - events, - asOf, + const { taken, missed } = trailingComplianceTally( + trailingComplianceLedger(events, schedules, ctx, asOf), windowDays, - timeZone, - engineCtx, + asOf, ); - const taken = ledger - ? ledger.taken - : timeline.filter((d) => d.status === "taken").length; - const missed = ledger - ? ledger.missed - : timeline.filter((d) => d.status === "missed").length; const denom = taken + missed; const adherenceRate = denom === 0 ? null : Math.round((taken / denom) * 100); const { current, longest } = streaksFromTimeline( timeline, asOf, windowDays, - timeZone, + ctx.timeZone, ); return { adherenceRate, diff --git a/src/lib/polar/__tests__/dedup.test.ts b/src/lib/polar/__tests__/dedup.test.ts index 0352b40be..21b379f01 100644 --- a/src/lib/polar/__tests__/dedup.test.ts +++ b/src/lib/polar/__tests__/dedup.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; -import { pickCanonicalWorkoutRows } from "@/lib/measurements/pick-canonical-workout-rows"; -import { DEFAULT_WORKOUT_SOURCE_PRIORITY } from "@/lib/sources/pick-canonical-workout"; +import { + pickCanonicalWorkoutRows, + workoutSourceLadder, +} from "@/lib/measurements/pick-canonical-workout-rows"; import { DEFAULT_SOURCE_PRIORITY } from "@/lib/validations/source-priority"; /** @@ -68,10 +70,10 @@ describe("POLAR workout dedup — read-time canonical picker", () => { }); describe("POLAR sits in the workout source ladders", () => { - it("is present in the workout default ladder, below WITHINGS, above STRAVA", () => { - const polar = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("POLAR"); - const withings = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("WITHINGS"); - const strava = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("STRAVA"); + it("is present in the workout ladder, below WITHINGS, above STRAVA", () => { + const polar = workoutSourceLadder(null).indexOf("POLAR"); + const withings = workoutSourceLadder(null).indexOf("WITHINGS"); + const strava = workoutSourceLadder(null).indexOf("STRAVA"); expect(polar).toBeGreaterThan(withings); expect(polar).toBeLessThan(strava); }); diff --git a/src/lib/rollups/__tests__/medication-compliance-rollups.test.ts b/src/lib/rollups/__tests__/medication-compliance-rollups.test.ts index 848cce605..13c7d970a 100644 --- a/src/lib/rollups/__tests__/medication-compliance-rollups.test.ts +++ b/src/lib/rollups/__tests__/medication-compliance-rollups.test.ts @@ -50,8 +50,6 @@ import { dayKeyForScheduledFor, recomputeMedicationComplianceForDay, recomputeMedicationComplianceForEvent, - readMedicationCompliance, - hasMedicationComplianceCoverage, recomputeUserMedicationCompliance, enqueueBootTimeMedicationComplianceBackfill, enqueueUserMedicationComplianceBackfill, @@ -317,107 +315,6 @@ describe("recomputeMedicationComplianceForEvent", () => { }); }); -describe("readMedicationCompliance", () => { - it("returns a trailing-window zero-filled bucket array", async () => { - rollupFindMany.mockResolvedValue([ - { day: "2026-05-18", scheduled: 2, taken: 1, skipped: 1 }, - ]); - - const now = new Date("2026-05-18T12:00:00.000Z"); - const buckets = await readMedicationCompliance( - "user-1", - 3, - "Europe/Berlin", - now, - ); - - expect(buckets).toHaveLength(3); - expect(buckets[buckets.length - 1]).toEqual({ - date: "2026-05-18", - scheduled: 2, - taken: 1, - }); - expect(buckets[0].scheduled).toBe(0); - expect(buckets[0].taken).toBe(0); - expect(buckets.map((b) => b.date)).toEqual([ - "2026-05-16", - "2026-05-17", - "2026-05-18", - ]); - }); - - it("folds per-medication rows into one per-day total", async () => { - rollupFindMany.mockResolvedValue([ - { day: "2026-05-18", scheduled: 1, taken: 1, skipped: 0 }, - { day: "2026-05-18", scheduled: 2, taken: 0, skipped: 1 }, - ]); - const now = new Date("2026-05-18T12:00:00.000Z"); - const buckets = await readMedicationCompliance( - "user-1", - 1, - "Europe/Berlin", - now, - ); - expect(buckets[0].scheduled).toBe(3); - expect(buckets[0].taken).toBe(1); - }); -}); - -describe("hasMedicationComplianceCoverage", () => { - it("returns true when rolled-day count meets the event-day count in window", async () => { - queryRaw.mockResolvedValue([ - { rolled_days: BigInt(7), event_days: BigInt(7) }, - ]); - await expect( - hasMedicationComplianceCoverage( - "user-1", - 7, - "Europe/Berlin", - new Date("2026-05-18T12:00:00.000Z"), - ), - ).resolves.toBe(true); - }); - - it("returns true when the user has zero events in window (trivially covered)", async () => { - // Zero events → the read path returns a trailing-window zero-fill - // from an empty rollup table; covered semantically. - queryRaw.mockResolvedValue([ - { rolled_days: BigInt(0), event_days: BigInt(0) }, - ]); - await expect( - hasMedicationComplianceCoverage("user-1", 7, "Europe/Berlin"), - ).resolves.toBe(true); - }); - - it("returns false on partial coverage (boot backfill mid-fold)", async () => { - // QA F-H-01 (v1.4.39): the legacy "any row exists" probe would - // flip true here and serve zero-filled buckets for days N..days-1 - // that the backfill hasn't reached yet. With the tightened probe - // partial coverage forces the route into the legacy aggregator - // until the fold completes. - queryRaw.mockResolvedValue([ - { rolled_days: BigInt(2), event_days: BigInt(7) }, - ]); - await expect( - hasMedicationComplianceCoverage( - "user-1", - 7, - "Europe/Berlin", - new Date("2026-05-18T12:00:00.000Z"), - ), - ).resolves.toBe(false); - }); - - it("returns false on zero rollups when events exist (legacy account cold start)", async () => { - queryRaw.mockResolvedValue([ - { rolled_days: BigInt(0), event_days: BigInt(7) }, - ]); - await expect( - hasMedicationComplianceCoverage("user-1", 7, "Europe/Berlin"), - ).resolves.toBe(false); - }); -}); - describe("recomputeUserMedicationCompliance", () => { it("upserts one rollup per (medication, day) pair returned by discovery", async () => { queryRaw.mockResolvedValue([ diff --git a/src/lib/rollups/medication-compliance-rollups.ts b/src/lib/rollups/medication-compliance-rollups.ts index 46f9e2e29..8e6163ac3 100644 --- a/src/lib/rollups/medication-compliance-rollups.ts +++ b/src/lib/rollups/medication-compliance-rollups.ts @@ -9,10 +9,11 @@ * Write hooks fire from every `MedicationIntakeEvent` mutation site * (create / update / delete / bulk import) plus the reminder-worker's * mint path that creates `takenAt:null` rows on RED-phase reminders. - * Reads collapse the trailing N-day window into a single bounded - * `findMany` against `medication_compliance_rollups`; the route-level - * coverage probe falls back to live + fires a boot backfill when a - * user has intake events but zero rollup rows. + * The dashboard tile it was built for reads the schedule-anchored + * compliance (`src/lib/analytics/schedule-anchored-compliance.ts`) since + * v1.26, and the per-day recompute below is now what keeps a day's + * supplement nutrients in step with its doses; the table itself has no + * reader left. * * v1.15.18 DESIGN DECISION — this tier stays a RAW per-day COVERAGE count, * NOT the band-attributed compliance %. It aggregates, across ALL of a @@ -77,14 +78,6 @@ export interface MedicationComplianceBackfillPayload { enqueuedAt: string; } -/** Reader-result row shape — matches `buildComplianceBuckets` output. */ -export interface ComplianceBucket { - /** YYYY-MM-DD in the user's tz. */ - date: string; - scheduled: number; - taken: number; -} - /** * Resolve a safe tz string — fall back to the server default when the * supplied value is missing or not an IANA zone the host recognises. @@ -254,138 +247,6 @@ export async function recomputeMedicationComplianceForEvent(input: { } } -/** - * Read trailing `days` of compliance buckets for the user, oldest day - * first. Returns the same `{date, scheduled, taken}` shape the legacy - * `buildComplianceBuckets` helper produced so the route swap is - * byte-identical. - * - * Days without a rollup row land as zero-filled buckets. The caller's - * cache wrapper carries the user-tz in its key so a tz change forces - * a fresh read. - */ -export async function readMedicationCompliance( - userId: string, - days: number, - tz: string | null | undefined, - now: Date = new Date(), -): Promise { - const safeTz = safeTimezone(tz); - - // Build the expected day-key window oldest → newest so the response - // is sorted and zero-filled regardless of which days hold rollups. - const todayKey = userDayKey(now, safeTz); - const expectedKeys: string[] = []; - for (let i = days - 1; i >= 0; i--) { - const probe = new Date(now.getTime() - i * 86_400_000); - expectedKeys.push(userDayKey(probe, safeTz)); - } - // Guard against a DST day-of-the-year landing the loop on the same - // dayKey twice — dedup by Map insertion order. - const dedup = new Set(); - const orderedKeys: string[] = []; - for (const key of expectedKeys) { - if (!dedup.has(key)) { - dedup.add(key); - orderedKeys.push(key); - } - } - // Always include today even if the trailing window dropped it (e.g. - // days===1 on a DST boundary). Cheap defensive insert. - if (!dedup.has(todayKey)) { - orderedKeys.push(todayKey); - } - - const oldestKey = orderedKeys[0]; - const newestKey = orderedKeys[orderedKeys.length - 1]; - - const rows = await prisma.medicationComplianceRollup.findMany({ - where: { - userId, - day: { gte: oldestKey, lte: newestKey }, - }, - select: { day: true, scheduled: true, taken: true, skipped: true }, - }); - - // Fold per-medication rows into one per-day total. The legacy route - // shape was aggregated across all the user's medications; preserving - // that surface keeps the front-end byte-stable. - const totals = new Map(); - for (const key of orderedKeys) { - totals.set(key, { scheduled: 0, taken: 0 }); - } - for (const row of rows) { - const bucket = totals.get(row.day); - if (!bucket) continue; - bucket.scheduled += row.scheduled; - bucket.taken += row.taken; - } - - return orderedKeys.map((date) => { - const bucket = totals.get(date) ?? { scheduled: 0, taken: 0 }; - return { date, scheduled: bucket.scheduled, taken: bucket.taken }; - }); -} - -/** - * Returns true when the rollup tier covers every day inside the - * trailing `days` window that has at least one intake event. Coverage - * is full when the count of DISTINCT rolled days equals the count of - * DISTINCT days that hold a `medication_intake_events` row in window — - * partial coverage (boot backfill mid-fold on a multi-medication - * account) returns false so the route falls through to the legacy - * aggregator instead of serving zero-filled tiles for un-rolled days. - * - * v1.4.39 hotfix (QA F-H-01): the previous "any row exists" probe - * could short-circuit to the rollup path while only the first few days - * had been backfilled, exposing zero-filled tiles for the un-rolled - * days until the boot fold completed. - */ -export async function hasMedicationComplianceCoverage( - userId: string, - days: number, - tz: string | null | undefined, - now: Date = new Date(), -): Promise { - const safeTz = safeTimezone(tz); - const oldestKey = userDayKey( - new Date(now.getTime() - (days - 1) * 86_400_000), - safeTz, - ); - const oldestStart = startOfLocalDayKey(oldestKey, safeTz); - - // Single SQL aggregate: count DISTINCT rolled days vs DISTINCT - // event-days in window. The event side is anchored on the same - // user-tz day-key (`to_char(... AT TIME ZONE $tz, 'YYYY-MM-DD')`) so - // the comparison stays apples-to-apples even on DST boundaries. - const result = await prisma.$queryRaw< - Array<{ rolled_days: bigint; event_days: bigint }> - >` - SELECT - ( - SELECT COUNT(DISTINCT "day")::bigint - FROM "medication_compliance_rollups" - WHERE "user_id" = ${userId} - AND "day" >= ${oldestKey} - ) AS rolled_days, - ( - SELECT COUNT(DISTINCT to_char("scheduled_for" AT TIME ZONE ${safeTz}, 'YYYY-MM-DD'))::bigint - FROM "medication_intake_events" - WHERE "user_id" = ${userId} - AND "deleted_at" IS NULL - AND "scheduled_for" >= ${oldestStart} - ) AS event_days - `; - if (result.length === 0) return true; - const { rolled_days, event_days } = result[0]; - const rolled = Number(rolled_days); - const events = Number(event_days); - // Zero events in window → trivially covered (the read path produces - // a zero-filled trailing window from the empty rollup table). - if (events === 0) return true; - return rolled >= events; -} - /** * Full-fold helper. Walks every `(medicationId, day)` pair the user * has logged across the trailing `days` window and upserts each @@ -395,7 +256,7 @@ export async function hasMedicationComplianceCoverage( * * Bounded SQL: one `DISTINCT (medicationId, dayKey)` $queryRaw scan * over the trailing window, then one upsert per row. The dayKey is - * computed in Postgres via `to_char(... AT TIME ZONE $tz)` so we + * computed in Postgres in the user's zone so we * don't have to ship millions of rows back to Node. */ export async function recomputeUserMedicationCompliance( @@ -414,14 +275,20 @@ export async function recomputeUserMedicationCompliance( const oldestStart = startOfLocalDayKey(oldestKey, safeTz); // Distinct (medication_id, dayKey) pairs the user has any intake - // event for inside the trailing window. `to_char(... AT TIME ZONE - // $tz)` keeps the day-bucket on the user's wall-clock. + // event for inside the trailing window. `scheduled_for` is a naive UTC + // timestamp: it is read as UTC first and only then shifted into the + // user's zone. A single `AT TIME ZONE $tz` reads it as local time and + // shifts it the wrong way, which keyed every slot within the zone's + // offset of midnight onto the neighbouring day. const pairs = await prisma.$queryRaw< Array<{ medication_id: string; day: string }> >` SELECT DISTINCT "medication_id", - to_char("scheduled_for" AT TIME ZONE ${safeTz}, 'YYYY-MM-DD') AS "day" + to_char( + ("scheduled_for" AT TIME ZONE 'UTC') AT TIME ZONE ${safeTz}, + 'YYYY-MM-DD' + ) AS "day" FROM "medication_intake_events" WHERE "user_id" = ${userId} AND "deleted_at" IS NULL diff --git a/src/lib/rollups/tiered-context.ts b/src/lib/rollups/tiered-context.ts index b73f6073d..931911e5b 100644 --- a/src/lib/rollups/tiered-context.ts +++ b/src/lib/rollups/tiered-context.ts @@ -43,6 +43,7 @@ import { readDayAggregates } from "@/lib/measurements/day-aggregates"; import { dayWeightedRowsSql, windowMeanSql } from "@/lib/measurements/day-mean"; import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { Prisma } from "@/generated/prisma/client"; +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -203,9 +204,8 @@ function windowStats(buckets: RollupBucket[]): { const means = usable.map((b) => b.mean); const mean = means.reduce((s, m) => s + m, 0) / means.length; if (means.length < 2) return { mean: round(mean), sd: null, n }; - const variance = - means.reduce((s, m) => s + (m - mean) ** 2, 0) / means.length; - return { mean: round(mean), sd: round(Math.sqrt(variance)), n }; + const sd = standardDeviation(means, READING_SPREAD) ?? 0; + return { mean: round(mean), sd: round(sd), n }; } /** diff --git a/src/lib/sources/__tests__/pick-canonical-workout.test.ts b/src/lib/sources/__tests__/pick-canonical-workout.test.ts deleted file mode 100644 index ecb9afbda..000000000 --- a/src/lib/sources/__tests__/pick-canonical-workout.test.ts +++ /dev/null @@ -1,478 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - DEFAULT_WORKOUT_PROXIMITY_MINUTES, - DEFAULT_WORKOUT_SOURCE_PRIORITY, - pickCanonicalWorkout, - type WorkoutPickerRow, -} from "../pick-canonical-workout"; - -/** - * v1.4.25 W16a — `pickCanonicalWorkout` unit tests. - * - * Three primary fixtures match the W16a brief: - * (a) two sources record the SAME workout (within proximity, same - * sport) — picker keeps the canonical one. - * (b) two sources record DIFFERENT workouts close in time (within - * proximity but different sport types) — picker keeps both. - * (c) single source — pass-through. - * - * Edge fixtures verify the proximity-window boundary, the determinism - * contract, and the empty-input fast path. - */ - -const D = (iso: string): Date => new Date(iso); - -const row = ( - id: string, - source: WorkoutPickerRow["source"], - startedAt: string, - sportType: WorkoutPickerRow["sportType"] = "running", - extra: Partial = {}, -): WorkoutPickerRow => ({ - id, - source, - startedAt: D(startedAt), - sportType, - ...extra, -}); - -describe("DEFAULT_WORKOUT_SOURCE_PRIORITY", () => { - it("places APPLE_HEALTH ahead of WITHINGS for cumulative workouts", () => { - const apple = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("APPLE_HEALTH"); - const withings = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("WITHINGS"); - expect(apple).toBeGreaterThanOrEqual(0); - expect(withings).toBeGreaterThan(apple); - }); - - it("places MANUAL ahead of IMPORT so user-typed entries beat bulk loads", () => { - const manual = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("MANUAL"); - const importIdx = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("IMPORT"); - expect(importIdx).toBeGreaterThan(manual); - }); -}); - -describe("pickCanonicalWorkout — fixture (a) two-source same workout", () => { - it("keeps the APPLE_HEALTH row when both sources record the same run", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - // Withings ingested the same workout 60 seconds later (typical - // API-pull lag). Same sport, well within the 5-minute window. - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - ]); - expect(result.canonical).toHaveLength(1); - expect(result.canonical[0].id).toBe("apple-1"); - expect(result.canonical[0].source).toBe("APPLE_HEALTH"); - expect(result.clusters).toHaveLength(1); - expect(result.clusters[0].members).toHaveLength(2); - expect(result.clusters[0].pickedSource).toBe("APPLE_HEALTH"); - }); - - it("keeps every workout from the winning source in one cluster", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - row("apple-2", "APPLE_HEALTH", "2026-05-14T07:32:00.000Z"), - ]); - - expect(result.canonical.map((item) => item.id)).toEqual([ - "apple-1", - "apple-2", - ]); - }); - - it("still picks APPLE_HEALTH when the Withings row arrives first", () => { - const result = pickCanonicalWorkout([ - row("withings-1", "WITHINGS", "2026-05-14T07:30:00.000Z"), - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:31:30.000Z"), - ]); - expect(result.canonical.map((r) => r.id)).toEqual(["apple-1"]); - }); - - it("honours a per-user override that flips the ladder", () => { - const result = pickCanonicalWorkout( - [ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - ], - { sourcePriority: ["WITHINGS", "APPLE_HEALTH", "MANUAL", "IMPORT"] }, - ); - expect(result.canonical.map((r) => r.id)).toEqual(["withings-1"]); - }); -}); - -describe("pickCanonicalWorkout — fixture (b) two-source different workouts close in time", () => { - it("keeps both rows when sport types differ within the proximity window", () => { - const result = pickCanonicalWorkout([ - // 07:30 Walking + 07:32 Running — same wall-clock burst (user - // tapped one workout, the watch logged a separate one) but - // different sport types, so they must stay distinct rows. - row("walk-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "walking"), - row("run-1", "WITHINGS", "2026-05-14T07:32:00.000Z", "running"), - ]); - expect(result.canonical).toHaveLength(2); - expect(result.canonical.map((r) => r.id).sort()).toEqual([ - "run-1", - "walk-1", - ]); - expect(result.clusters).toHaveLength(2); - }); - - it("keeps both rows when the same sport occurs outside the proximity window", () => { - const result = pickCanonicalWorkout([ - row("run-am", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - // 2 hours later — well past the 5-minute cluster window. A - // morning run and an afternoon run on the same day must NOT - // collapse into one canonical row. - row("run-pm", "WITHINGS", "2026-05-14T09:30:00.000Z"), - ]); - expect(result.canonical).toHaveLength(2); - expect(result.clusters).toHaveLength(2); - }); -}); - -describe("pickCanonicalWorkout — fixture (c) single source pass-through", () => { - it("returns the only row unchanged when no other source contributed", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - ]); - expect(result.canonical).toHaveLength(1); - expect(result.canonical[0].id).toBe("apple-1"); - expect(result.clusters[0].pickedSource).toBe("APPLE_HEALTH"); - }); - - it("passes through a batch of disjoint single-source workouts", () => { - const result = pickCanonicalWorkout([ - row("run-1", "APPLE_HEALTH", "2026-05-13T07:30:00.000Z"), - row("run-2", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("run-3", "APPLE_HEALTH", "2026-05-15T07:30:00.000Z"), - ]); - expect(result.canonical.map((r) => r.id)).toEqual([ - "run-1", - "run-2", - "run-3", - ]); - }); - - it("returns empty for empty input", () => { - const result = pickCanonicalWorkout([]); - expect(result.canonical).toHaveLength(0); - expect(result.clusters).toHaveLength(0); - }); -}); - -describe("pickCanonicalWorkout — proximity-window boundary", () => { - it("exposes the default proximity window as 5 minutes", () => { - expect(DEFAULT_WORKOUT_PROXIMITY_MINUTES).toBe(5); - }); - - it("clusters two rows exactly at the proximity boundary", () => { - // 5-minute gap — picker accepts the boundary as in-window so a - // server's coarse cron tick (Withings hourly pull at HH:00 paired - // with HKWorkout export at HH:05) still dedups cleanly. - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:35:00.000Z"), - ]); - expect(result.canonical).toHaveLength(1); - }); - - it("anchors a cluster to its first row instead of chaining neighbours", () => { - const result = pickCanonicalWorkout([ - row("anchor", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("neighbour", "WITHINGS", "2026-05-14T07:34:00.000Z"), - row("chained", "WITHINGS", "2026-05-14T07:38:00.000Z"), - ]); - - expect(result.clusters).toHaveLength(2); - expect(result.canonical.map((item) => item.id)).toEqual([ - "anchor", - "chained", - ]); - }); - - it("keeps two rows just outside the proximity window", () => { - // 5 minutes and 1 second — picker rejects, two clusters. - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:35:01.000Z"), - ]); - expect(result.canonical).toHaveLength(2); - }); - - it("respects a custom proximity window", () => { - // Same fixture as the same-workout case but with a 30-second - // window — the 60-second-later Withings row falls outside and - // both rows survive. - const result = pickCanonicalWorkout( - [ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - ], - { proximityMinutes: 0.5 }, - ); - expect(result.canonical).toHaveLength(2); - }); -}); - -describe("pickCanonicalWorkout — determinism", () => { - it("returns the same canonical list regardless of input order", () => { - const a = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - row("manual-1", "MANUAL", "2026-05-14T07:30:30.000Z"), - ]); - const b = pickCanonicalWorkout([ - row("manual-1", "MANUAL", "2026-05-14T07:30:30.000Z"), - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z"), - ]); - expect(a.canonical.map((r) => r.id)).toEqual(b.canonical.map((r) => r.id)); - }); - - it("preserves same-source workouts at the same timestamp", () => { - const result = pickCanonicalWorkout([ - row("a-2", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - row("a-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z"), - ]); - - expect(result.canonical.map((item) => item.id)).toEqual(["a-1", "a-2"]); - }); -}); - -describe("pickCanonicalWorkout — field-merge (v1.29.x, WHOOP HR-loss fix)", () => { - it("backfills avgHeartRate/maxHeartRate from a lower-priority twin when the winner's is null", () => { - // Regression fixture from the brief: a WHOOP cycling row carries live - // HR; a higher-priority Apple Health twin for the SAME ride has no HR - // (e.g. a manually-logged / GPS-only Apple entry). Before the merge, - // the picker kept the Apple row whole and threw the WHOOP HR away. - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling", { - avgHeartRate: null, - maxHeartRate: null, - }), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling", { - avgHeartRate: 142, - maxHeartRate: 168, - }), - ]); - expect(result.canonical).toHaveLength(1); - const winner = result.canonical[0]; - expect(winner.id).toBe("apple-1"); // ladder winner stays the base row - expect(winner.avgHeartRate).toBe(142); - expect(winner.maxHeartRate).toBe(168); - expect(result.clusters[0].picked.avgHeartRate).toBe(142); - }); - - it("backfills a lower-priority donor only onto its nearest winning-source workout", () => { - const fixtures = [ - row( - "apple-earlier", - "APPLE_HEALTH", - "2026-05-14T07:30:00.000Z", - "running", - { - avgHeartRate: null, - }, - ), - row( - "apple-later", - "APPLE_HEALTH", - "2026-05-14T07:34:00.000Z", - "running", - { - avgHeartRate: null, - }, - ), - row("whoop-later", "WHOOP", "2026-05-14T07:34:00.000Z", "running", { - avgHeartRate: 151, - }), - ]; - - for (const input of [fixtures, [...fixtures].reverse()]) { - const result = pickCanonicalWorkout(input); - - expect( - result.canonical.map(({ id, source, avgHeartRate }) => ({ - id, - source, - avgHeartRate, - })), - ).toEqual([ - { - id: "apple-earlier", - source: "APPLE_HEALTH", - avgHeartRate: null, - }, - { - id: "apple-later", - source: "APPLE_HEALTH", - avgHeartRate: 151, - }, - ]); - } - }); - - it("never overwrites a field the base row already has", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling", { - avgHeartRate: 120, - }), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling", { - avgHeartRate: 142, - }), - ]); - expect(result.canonical[0].avgHeartRate).toBe(120); - }); - - it("backfills totalEnergyKcal, totalDistanceM, and elevationM independently", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling", { - totalEnergyKcal: null, - totalDistanceM: 15000, - elevationM: null, - }), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling", { - totalEnergyKcal: 480, - totalDistanceM: 14800, - elevationM: 220, - }), - ]); - const winner = result.canonical[0]; - expect(winner.totalEnergyKcal).toBe(480); // backfilled - expect(winner.totalDistanceM).toBe(15000); // base row's own value kept - expect(winner.elevationM).toBe(220); // backfilled - }); - - it("prefers the ladder-highest member that HAS a field over a lower one, when the base row lacks it", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling", { - avgHeartRate: null, - }), - row("whoop-1", "WHOOP", "2026-05-14T07:30:30.000Z", "cycling", { - avgHeartRate: 142, - }), - row("withings-1", "WITHINGS", "2026-05-14T07:31:00.000Z", "cycling", { - avgHeartRate: 99, - }), - ]); - // WHOOP outranks WITHINGS on the default ladder — its HR wins the - // backfill even though WITHINGS also had one. - expect(result.canonical[0].avgHeartRate).toBe(142); - }); - - it("adopts a specific sport type from a member when the base row's own sportType is generic", () => { - // The confirmed bug's downstream symptom: before the WHOOP sport-map - // fix, a WHOOP cycling workout could carry sportType "other" (or a - // non-canonical raw label) while a lower-priority twin correctly - // tagged "cycling". The merge adopts the specific sport so the - // cluster still surfaces as a bike ride. - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "other"), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling"), - ]); - expect(result.canonical[0].sportType).toBe("cycling"); - }); - - it("keeps the base row's own specific sportType even when a member disagrees", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling"), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling"), - ]); - expect(result.canonical[0].sportType).toBe("cycling"); - }); - - it("does not run the merge on a single-element cluster", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "cycling", { - avgHeartRate: null, - }), - ]); - expect(result.canonical[0].avgHeartRate).toBeNull(); - }); - - it("end-to-end: a WHOOP-cycling+HR row clustered with a generic no-HR twin yields a cycling canonical row WITH the HR", () => { - const result = pickCanonicalWorkout([ - row("apple-1", "APPLE_HEALTH", "2026-05-14T07:30:00.000Z", "other", { - avgHeartRate: null, - totalEnergyKcal: null, - }), - row("whoop-1", "WHOOP", "2026-05-14T07:31:00.000Z", "cycling", { - avgHeartRate: 145, - totalEnergyKcal: 512, - }), - ]); - expect(result.canonical).toHaveLength(1); - const winner = result.canonical[0]; - expect(winner.source).toBe("APPLE_HEALTH"); // ladder still picks Apple as the base - expect(winner.sportType).toBe("cycling"); // adopted from WHOOP - expect(winner.avgHeartRate).toBe(145); // backfilled from WHOOP - expect(winner.totalEnergyKcal).toBe(512); // backfilled from WHOOP - }); -}); - -describe("pickCanonicalWorkout — Strava brick-workout regression", () => { - // Legacy Strava labels ("Ride", "Run") are both generic to this picker, - // so they share a cluster. The source is identical, however, which means - // both legs are distinct records and must survive. - it("preserves two same-source raw Strava workouts in one cluster", () => { - const result = pickCanonicalWorkout([ - row("strava-ride", "STRAVA", "2026-06-01T15:00:00.000Z", "Ride"), - row("strava-run", "STRAVA", "2026-06-01T15:03:00.000Z", "Run"), - ]); - - expect(result.clusters).toHaveLength(1); - expect(result.canonical.map((item) => item.id)).toEqual([ - "strava-ride", - "strava-run", - ]); - }); - - it("post-fix: a canonical Strava Ride + Run pair survives as two distinct rows", () => { - // Same timing as the pre-fix fixture above, but with the sportType - // values `mapActivity()` now writes (via `mapStravaSportType()`): - // "Ride" → "cycling", "Run" → "running". Both are SPECIFIC canonical - // buckets and differ from each other, so `isSpecificSportType()` no - // longer treats them as sport-compatible and the brick session - // survives as two workouts. - const result = pickCanonicalWorkout([ - row("strava-ride", "STRAVA", "2026-06-01T15:00:00.000Z", "cycling"), - row("strava-run", "STRAVA", "2026-06-01T15:03:00.000Z", "running"), - ]); - expect(result.clusters).toHaveLength(2); - expect(result.canonical).toHaveLength(2); - const sports = result.canonical.map((w) => w.sportType).sort(); - expect(sports).toEqual(["cycling", "running"]); - }); -}); - -/** - * #1054 — workouts a `workouts:write` bridge pushes in carry `EXTERNAL`, which - * is on no default ladder. Pinned because the settings copy promises what - * follows from that: a bridged session the phone also recorded shows once, as - * the phone's copy, and a bridged session nothing else recorded still shows. - */ -describe("pickCanonicalWorkout — the unranked EXTERNAL source", () => { - it("is on no default ladder", () => { - expect(DEFAULT_WORKOUT_SOURCE_PRIORITY).not.toContain("EXTERNAL"); - }); - - it("yields to the phone's copy of the same session", () => { - const result = pickCanonicalWorkout([ - row("bridge-1", "EXTERNAL", "2026-09-20T06:30:00.000Z"), - row("apple-1", "APPLE_HEALTH", "2026-09-20T06:31:00.000Z"), - ]); - expect(result.canonical.map((w) => w.id)).toEqual(["apple-1"]); - }); - - it("survives when nothing else recorded the session", () => { - const result = pickCanonicalWorkout([ - row("bridge-1", "EXTERNAL", "2026-09-20T06:30:00.000Z"), - row("apple-1", "APPLE_HEALTH", "2026-09-20T09:00:00.000Z"), - ]); - expect(result.canonical.map((w) => w.id).sort()).toEqual([ - "apple-1", - "bridge-1", - ]); - }); -}); diff --git a/src/lib/sources/pick-canonical-workout.ts b/src/lib/sources/pick-canonical-workout.ts deleted file mode 100644 index 9f1da7274..000000000 --- a/src/lib/sources/pick-canonical-workout.ts +++ /dev/null @@ -1,437 +0,0 @@ -/** - * v1.4.25 W16a — cross-source workout canonical picker. - * - * Companion to `pickCanonicalSourceRows()` (the Measurement-row picker - * in `src/lib/analytics/source-priority.ts`). Workouts have a similar - * double-ingest problem coming in v1.5: the iOS app will write an - * HKWorkout row at the same wall-clock time the Withings Activity API - * already supplied — same run, two rows. - * - * This helper is scaffolding only — it returns the canonical workout - * list, doesn't mutate the DB, and is not yet wired into any query - * path. The wiring lands alongside the workout ingest endpoint - * (W16b in the v1.4.25 plan) or with the v1.5 iOS rollout, whichever - * arrives first. - * - * Algorithm - * ───────── - * - * 1. Cluster workouts whose start timestamps land within - * `proximityMinutes` of each other (default 5 min — Apple's - * ScanWatch passthrough lag plus a small clock-skew buffer). - * Sport type must match across the cluster; a "Walking" workout - * and a "Running" workout starting at the same instant stay - * distinct rows so the cardio-vs-strength split stays clean. Once - * an integration writes a canonical `sportType` (every source does, - * as of the WHOOP fix below), a cross-source pair of the SAME - * workout always shares a sport bucket and clusters correctly — - * before that fix a WHOOP "whoop_sport_1" row could never cluster - * with an Apple Health "cycling" row. - * 2. Per cluster, walk the per-user source-priority ladder for - * workouts (default `APPLE_HEALTH > WITHINGS > MANUAL > IMPORT`) - * and keep the first source that has a row in the cluster as the - * BASE of the canonical row. All other rows from the same cluster - * are dropped from the canonical list. - * 3. Field-merge (multi-element clusters only): each workout from the - * winning source is backfilled only from losing-source members matched - * to that workout by sport identity and start-time proximity. A donor is - * assigned to one deterministic best winner (exact sport before generic - * compatibility, then nearest timestamp). Missing `avgHeartRate`, - * `maxHeartRate`, `totalEnergyKcal`, `totalDistanceM`, and `elevationM` - * fields come from the ladder-highest matched donor that has the field. - * The same step adopts a specific sport type when the base row is generic - * (`"other"` / unrecognised). Every OTHER field on the base row (id, - * source, startedAt, route, metadata, ...) is untouched — the merge only - * ever fills a gap, never overwrites a value the base row already has. - * 4. Single-element clusters (no near-duplicate) pass through - * unchanged regardless of source — no merge step runs. - * - * Rationale for the default ladder - * ──────────────────────────────── - * - * - `APPLE_HEALTH` first: the iOS-app passthrough carries the - * watch's per-second sample stream (energy, HR, optional GPS), - * so when both sources have the same workout it's the richer - * row. - * - `WITHINGS` second: ScanWatch syncs into Withings cloud too, - * but the API caps each workout at coarser aggregates (mean HR, - * totals, no per-sample HR series). Loses cleanly when paired - * with the HKWorkout twin. - * - `MANUAL` third: the user typed it; trust beats anything from - * `IMPORT` (CSV / XML) but loses to a sensor stream. - * - `IMPORT` last: bulk historical imports (Apple Health XML, - * Strava export, Garmin TCX) — useful when no live sensor source - * covers the period but the lowest fidelity when one does. - * - * The picker is deterministic on `(rows, ladder, proximityMinutes)` - * and stable on input order. The caller is responsible for sorting - * rows by `startedAt ASC, id ASC` before invocation so the picker's - * cluster-building walks a fixed sequence. - */ - -import type { MeasurementSource } from "@/generated/prisma/client"; - -import { - workoutSportTypeEnum, - type WorkoutSportType, -} from "@/lib/validations/workout"; - -/** - * Minimum row shape the helper consults. Exposed as an interface so the - * Prisma `Workout` model and the not-yet-existing ingest DTO both - * compose with the picker without a transform step. Anything else on - * the row (route blob, metadata, ...) flows through untouched — ONLY the - * fields declared below are eligible for the step-3 field-merge, and even - * those are backfilled (fills a gap on the base row), never overwritten. - */ -export interface WorkoutPickerRow { - /** Stable identifier — `id` for DB rows, externalId+source for - * not-yet-persisted ingest payloads. Used for deterministic - * tie-breaking when two sources both win the priority round. */ - id: string; - startedAt: Date; - source: MeasurementSource; - /** Canonical sport string. Workouts of different sport types never - * collapse into one cluster — see step 1 of the algorithm. May be - * adopted from a losing cluster member by the step-3 merge when the - * base row's own value is generic — see `mergeClusterFields()`. */ - sportType: WorkoutSportType | string; - /** Backfillable richer fields — present when the source carries live - * sensor data (HR strap, GPS), absent/null on a coarser summary from - * another source recording the same workout. */ - avgHeartRate?: number | null; - maxHeartRate?: number | null; - totalEnergyKcal?: number | null; - totalDistanceM?: number | null; - elevationM?: number | null; -} - -/** - * The numeric fields step 3 of the algorithm may backfill onto the base - * row. Listed explicitly (not derived via `keyof`) so each field's copy - * stays independently type-checked in `mergeClusterFields()`. - */ -const MERGEABLE_NUMERIC_FIELDS = [ - "avgHeartRate", - "maxHeartRate", - "totalEnergyKcal", - "totalDistanceM", - "elevationM", -] as const; - -/** - * `true` for a specific canonical sport bucket; `false` for the generic - * `"other"` bucket or a string outside `workoutSportTypeEnum` entirely (a - * not-yet-normalised integration's raw label). Drives the sport-type - * adoption half of the merge. - */ -function isSpecificSportType(value: string): boolean { - return ( - value !== "other" && - (workoutSportTypeEnum.options as readonly string[]).includes(value) - ); -} - -function areSportTypesCompatible(left: string, right: string): boolean { - return ( - left === right || !isSpecificSportType(left) || !isSpecificSportType(right) - ); -} - -/** - * Match one losing-source donor to one winning-source workout. Exact sport - * identity is stronger than generic compatibility; within the same identity - * strength, the nearest start wins. `winners` is already startedAt/id-sorted, - * so an exact tie resolves deterministically to the first winner. - */ -function findMatchingWinner( - donor: T, - winners: readonly T[], - proximityMs: number, -): T | undefined { - let best: T | undefined; - let bestHasExactSport = false; - let bestDeltaMs = Infinity; - - for (const winner of winners) { - if (!areSportTypesCompatible(winner.sportType, donor.sportType)) continue; - - const deltaMs = Math.abs( - winner.startedAt.getTime() - donor.startedAt.getTime(), - ); - if (deltaMs > proximityMs) continue; - - const hasExactSport = winner.sportType === donor.sportType; - if ( - !best || - (hasExactSport && !bestHasExactSport) || - (hasExactSport === bestHasExactSport && deltaMs < bestDeltaMs) - ) { - best = winner; - bestHasExactSport = hasExactSport; - bestDeltaMs = deltaMs; - } - } - - return best; -} - -/** Ladder position of `source`, or `ladder.length` (last) when the source - * isn't on the ladder at all — an unranked source only wins a merge field - * when no ranked member has it. */ -function ladderRank( - source: MeasurementSource, - ladder: readonly MeasurementSource[], -): number { - const idx = ladder.indexOf(source); - return idx === -1 ? ladder.length : idx; -} - -/** - * Step 3 of the algorithm: backfill the ladder-winning base row's missing - * richer fields, and its generic sport type, from other cluster members. - * - * Deterministic: for each candidate field, the ladder-highest member that - * HAS a non-null value wins that field. `members` is already - * startedAt/id-sorted by the caller, so a same-rank tie resolves to - * whichever member sorts first. - * - * A single-element cluster returns `winning` unchanged — nothing to merge. - */ -function mergeClusterFields( - winning: T, - members: readonly T[], - ladder: readonly MeasurementSource[], -): T { - if (members.length <= 1) return winning; - - const merged: T = { ...winning }; - - for (const field of MERGEABLE_NUMERIC_FIELDS) { - if (merged[field] != null) continue; - let best: T | undefined; - let bestRank = Infinity; - for (const member of members) { - const value = member[field]; - if (value == null) continue; - const rank = ladderRank(member.source, ladder); - if (rank < bestRank) { - bestRank = rank; - best = member; - } - } - if (best) merged[field] = best[field]; - } - - if (!isSpecificSportType(merged.sportType)) { - let best: T | undefined; - let bestRank = Infinity; - for (const member of members) { - if (!isSpecificSportType(member.sportType)) continue; - const rank = ladderRank(member.source, ladder); - if (rank < bestRank) { - bestRank = rank; - best = member; - } - } - if (best) merged.sportType = best.sportType; - } - - return merged; -} - -/** - * Per-user, per-source priority list for workouts. The shape is - * parallel to `SOURCE_PRIORITY_METRIC_KEYS` entries but stored - * separately because workouts are their own entity (not a Measurement - * row), and the metric-priority Json blob on `User.sourcePriorityJson` - * never carries a `workouts` key today. - * - * Persistence of the per-user value lands when the dedup wiring lands; - * the helper accepts an explicit ladder so the future caller can pass - * the merged value without dragging schema work into this file. - */ -export const DEFAULT_WORKOUT_SOURCE_PRIORITY: readonly MeasurementSource[] = [ - "APPLE_HEALTH", - // v1.11.0 — WHOOP ranks second. Apple Watch GPS + HR is the richest run - // record; a WHOOP strap is the next-best when no watch logged the same - // session. The read-time picker clusters a WHOOP run and an Apple-Health - // run by (activityType, startedAt ± window) and keeps the ladder winner. - "WHOOP", - // v1.12.0 — Fitbit/Pixel exercise logs rank just below WHOOP: a wrist - // wearable in the same class, the next-best when no watch logged the same - // session. - "FITBIT", - "WITHINGS", - // v1.32.15 — Polar ranks below Withings and above Strava: a Polar watch is a - // device-native HR capture (unlike a Strava re-upload with degraded/absent - // HR), so a Polar exercise outranks a Strava twin of the same session but - // sits below the primary on-wrist wearables. Matches Polar's slot in the - // read-time `steps` ladder (`DEFAULT_SOURCE_PRIORITY.steps`). - "POLAR", - // v1.28.x — Strava ranks below the device-native captures (Apple/WHOOP/ - // Fitbit/Withings on-wrist HR) but above MANUAL/IMPORT: a Strava activity is - // often itself a re-upload of an Apple/Garmin recording, so its HR/calories - // are lower-fidelity or absent on the summary. A user who records primarily - // on Strava (phone GPS) can promote it in Settings → Sources. - "STRAVA", - "MANUAL", - "IMPORT", -] as const; - -/** - * Default proximity window. Apple's HealthKit passthrough is observed - * to arrive 0–60 seconds after the watch closes the workout; Withings - * Activity API rounds to the minute. 5 minutes covers both lags plus a - * small clock-skew buffer between the iOS app's clock and the server. - */ -export const DEFAULT_WORKOUT_PROXIMITY_MINUTES = 5; - -export interface PickCanonicalWorkoutOptions { - /** - * Per-user source ladder. Defaults to - * `DEFAULT_WORKOUT_SOURCE_PRIORITY`. The picker walks left-to-right - * and keeps the first source present in each cluster. - */ - sourcePriority?: readonly MeasurementSource[]; - /** - * Cluster window. Defaults to `DEFAULT_WORKOUT_PROXIMITY_MINUTES`. - * Workouts whose start timestamps are within this many minutes of - * the cluster anchor join that cluster; later neighbours cannot extend - * the window. - */ - proximityMinutes?: number; -} - -/** - * Pick the canonical-source workout list. See file-level docs for the - * algorithm; returns the surviving input rows plus one representative - * record per overlap cluster for debug overlays and audit logging. - */ -export function pickCanonicalWorkout( - rows: readonly T[], - options: PickCanonicalWorkoutOptions = {}, -): { - canonical: T[]; - clusters: Array<{ - /** Every row that clustered together, unmerged. */ - members: T[]; - pickedSource: MeasurementSource; - /** Representative winning row after cross-source field merge. */ - picked: T; - }>; -} { - if (rows.length === 0) { - return { canonical: [], clusters: [] }; - } - - const ladder = options.sourcePriority ?? DEFAULT_WORKOUT_SOURCE_PRIORITY; - const proximityMinutes = - options.proximityMinutes ?? DEFAULT_WORKOUT_PROXIMITY_MINUTES; - const proximityMs = proximityMinutes * 60_000; - - // Sort by startedAt so cluster building is a single linear pass. - // Stable secondary sort on `id` keeps determinism on ties — two rows - // recorded at the exact same instant with the same id ordering give - // the same picker output on every call. - const sorted = [...rows].sort((a, b) => { - const dt = a.startedAt.getTime() - b.startedAt.getTime(); - if (dt !== 0) return dt; - return a.id.localeCompare(b.id); - }); - - // Build clusters keyed by sport type (a Walking + Running pair - // starting at the same instant must stay distinct — different - // workouts entirely) — WITH one relaxation: a GENERIC sport - // ("other" / an unrecognised raw string) is treated as compatible - // with any SPECIFIC sport, so a thinly-classified row from one - // source (an unmapped import, a not-yet-normalised integration) can - // still cluster with a specific-sport twin from another source - // recording the SAME real-world workout — the merge step below then - // adopts the specific sport onto the canonical row. Two DIFFERENT - // specific sports (Walking vs Running) never become compatible this - // way — the core invariant is unchanged. - const clusters: T[][] = []; - for (const row of sorted) { - // Compare against the first row in the cluster. A later neighbour - // inside the window must not extend the cluster beyond its anchor. - let placed = false; - for (let i = clusters.length - 1; i >= 0; i--) { - const cluster = clusters[i]; - const head = cluster[0]; - const sameSport = head.sportType === row.sportType; - if (!areSportTypesCompatible(head.sportType, row.sportType)) continue; - if (row.startedAt.getTime() - head.startedAt.getTime() <= proximityMs) { - cluster.push(row); - placed = true; - break; - } - if (sameSport) break; - } - if (!placed) { - clusters.push([row]); - } - } - - // Preserve single-source clusters. For a real cross-source overlap, - // select the winning source and retain every workout it contributed. - const canonical: T[] = []; - const clusterRecords: Array<{ - members: T[]; - pickedSource: MeasurementSource; - picked: T; - }> = []; - for (const cluster of clusters) { - const sources = new Set(cluster.map((row) => row.source)); - if (sources.size === 1) { - canonical.push(...cluster); - clusterRecords.push({ - members: cluster, - pickedSource: cluster[0].source, - picked: cluster[0], - }); - continue; - } - - let pickedSource: MeasurementSource | undefined; - for (const candidate of ladder) { - if (cluster.some((row) => row.source === candidate)) { - pickedSource = candidate; - break; - } - } - if (!pickedSource) { - canonical.push(...cluster); - clusterRecords.push({ - members: cluster, - pickedSource: cluster[0].source, - picked: cluster[0], - }); - continue; - } - const winningRows = cluster.filter((row) => row.source === pickedSource); - const membersByWinner = new Map( - winningRows.map((winner) => [winner, [winner]]), - ); - for (const donor of cluster) { - if (donor.source === pickedSource) continue; - const matchedWinner = findMatchingWinner(donor, winningRows, proximityMs); - if (matchedWinner) membersByWinner.get(matchedWinner)?.push(donor); - } - const winners = winningRows.map((winner) => - mergeClusterFields( - winner, - membersByWinner.get(winner) ?? [winner], - ladder, - ), - ); - canonical.push(...winners); - clusterRecords.push({ - members: cluster, - pickedSource, - picked: winners[0], - }); - } - - return { canonical, clusters: clusterRecords }; -} diff --git a/src/lib/stats/__tests__/spread.test.ts b/src/lib/stats/__tests__/spread.test.ts new file mode 100644 index 000000000..5192b17f7 --- /dev/null +++ b/src/lib/stats/__tests__/spread.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; + +import { READING_SPREAD, standardDeviation, variance } from "../spread"; + +describe("spread", () => { + const values = [120, 130, 140]; + + it("divides by n for a population and by n - 1 for a sample", () => { + expect(variance(values, "population")).toBeCloseTo(200 / 3, 10); + expect(variance(values, "sample")).toBe(100); + expect(standardDeviation(values, "sample")).toBe(10); + }); + + it("is undefined below what the kind needs", () => { + expect(variance([], "population")).toBeNull(); + expect(variance([5], "population")).toBe(0); + expect(variance([5], "sample")).toBeNull(); + }); + + it("reports readings as a population", () => { + expect(READING_SPREAD).toBe("population"); + }); +}); diff --git a/src/lib/stats/spread.ts b/src/lib/stats/spread.ts new file mode 100644 index 000000000..c06c944c5 --- /dev/null +++ b/src/lib/stats/spread.ts @@ -0,0 +1,49 @@ +/** + * Variance and standard deviation, with the kind named at every call site. + * + * - `population` divides by n: the spread of exactly these readings. It is + * what a surface reports when it describes a series the person can see. + * - `sample` divides by n − 1: an estimate of the spread that readings like + * these come from. It belongs where the spread feeds a decision (a + * significance verdict, a changepoint threshold) or where a clinical + * definition asks for it. + * + * The two differ by sqrt(n / (n − 1)), about 22 % at n = 3, so one series + * shown with both rules looks like two different series. Every hand-rolled + * spread used to pick its rule silently; the API reported the population SD + * of a blood-pressure series while the MCP and AI payload reported the + * sample SD of the same readings. + */ + +export type VarianceKind = "population" | "sample"; + +/** + * The kind every surface uses to report the spread of a person's readings, + * so the API, the MCP tools, the Coach and the AI payload say the same + * number for the same series. + */ +export const READING_SPREAD: VarianceKind = "population"; + +/** `null` when the kind is undefined for this many values (n < 1, or n < 2 for a sample). */ +export function variance( + values: readonly number[], + kind: VarianceKind, +): number | null { + const n = values.length; + const divisor = kind === "sample" ? n - 1 : n; + if (divisor < 1) return null; + let sum = 0; + for (const v of values) sum += v; + const mean = sum / n; + let squares = 0; + for (const v of values) squares += (v - mean) ** 2; + return squares / divisor; +} + +export function standardDeviation( + values: readonly number[], + kind: VarianceKind, +): number | null { + const v = variance(values, kind); + return v === null ? null : Math.sqrt(v); +} diff --git a/src/lib/strava/__tests__/dedup.test.ts b/src/lib/strava/__tests__/dedup.test.ts index 889c2e2a2..e74f9075f 100644 --- a/src/lib/strava/__tests__/dedup.test.ts +++ b/src/lib/strava/__tests__/dedup.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; -import { pickCanonicalWorkoutRows } from "@/lib/measurements/pick-canonical-workout-rows"; -import { DEFAULT_WORKOUT_SOURCE_PRIORITY } from "@/lib/sources/pick-canonical-workout"; +import { + pickCanonicalWorkoutRows, + workoutSourceLadder, +} from "@/lib/measurements/pick-canonical-workout-rows"; import { DEFAULT_SOURCE_PRIORITY } from "@/lib/validations/source-priority"; /** @@ -67,10 +69,10 @@ describe("STRAVA workout dedup — read-time canonical picker", () => { }); describe("STRAVA sits in the workout source ladders", () => { - it("is present in the workout default ladder, below the wearables, above MANUAL", () => { - const strava = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("STRAVA"); - const withings = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("WITHINGS"); - const manual = DEFAULT_WORKOUT_SOURCE_PRIORITY.indexOf("MANUAL"); + it("is present in the workout ladder, below the wearables, above MANUAL", () => { + const strava = workoutSourceLadder(null).indexOf("STRAVA"); + const withings = workoutSourceLadder(null).indexOf("WITHINGS"); + const manual = workoutSourceLadder(null).indexOf("MANUAL"); expect(strava).toBeGreaterThan(withings); expect(strava).toBeLessThan(manual); }); diff --git a/src/lib/strava/sport-map.ts b/src/lib/strava/sport-map.ts index c3702e9f8..b60c7a675 100644 --- a/src/lib/strava/sport-map.ts +++ b/src/lib/strava/sport-map.ts @@ -7,11 +7,10 @@ * to write `summary.sport_type ?? summary.type ?? detail?.sport_type ?? * "workout"` straight into `Workout.sportType`, so a Strava "Run" or * "VirtualRide" never matched a canonical bucket — `/insights/workouts` - * rendered every Strava row generic, and the read-time picker in - * `src/lib/sources/pick-canonical-workout.ts` treats any two GENERIC rows - * as sport-compatible, so a same-source brick session (a Strava "Ride" - * ending 15:00 immediately followed by a Strava "Run" starting 15:03) - * clustered into one and silently dropped a leg. Fitbit + * rendered every Strava row generic, and a picker that treats two generic + * rows as the same sport clusters a same-source brick session (a Strava + * "Ride" ending 15:00 immediately followed by a Strava "Run" starting + * 15:03) into one and silently drops a leg. Fitbit * (`mapFitbitSportType()` in `src/lib/fitbit/client.ts`), Google Health * (`mapGoogleHealthSportType()` in `src/lib/google-health/mappers.ts`), and * WHOOP (`mapWhoopSportType()` in `src/lib/whoop/sport-map.ts`) all diff --git a/src/lib/targets/build-response.ts b/src/lib/targets/build-response.ts index 7b0ddc569..ac8ad62ce 100644 --- a/src/lib/targets/build-response.ts +++ b/src/lib/targets/build-response.ts @@ -32,6 +32,7 @@ import { } from "@/lib/rollups/mood-rollups"; import { readRestingPulseProxy } from "@/lib/analytics/resting-pulse-read"; import { readSevenDayMeans } from "@/lib/analytics/summaries-slice"; +import { readLatestEver } from "@/lib/dashboard/summary-reads"; import { summarizeSleepNights, type SleepStageRow, @@ -145,31 +146,7 @@ export async function buildTargetsResponse(user: AuthedUser) { select: { type: true, value: true, measuredAt: true }, }), ), - limit(() => { - for (const type of types) { - if (!/^[A-Z0-9_]+$/.test(type)) { - throw new Error(`invalid measurement type: ${type}`); - } - } - const typeList = types - .map((type) => `'${type}'::"measurement_type"`) - .join(","); - return prisma.$queryRawUnsafe< - Array<{ type: MeasurementType; value: number }> - >( - `SELECT DISTINCT ON (m."type") - m."type"::text AS type, - m."value" AS value - FROM measurements m - WHERE m."user_id" = $1 - AND m."type" IN (${typeList}) - AND m."measured_at" >= $2 - AND m."deleted_at" IS NULL - ORDER BY m."type" ASC, m."measured_at" DESC`, - userId, - oneYearAgo, - ); - }), + limit(() => readLatestEver(prisma, userId, types, oneYearAgo)), limit(() => prisma.measurement.findMany({ where: { diff --git a/src/lib/targets/mood-builder.ts b/src/lib/targets/mood-builder.ts index 2e48e76c7..fdb917ae7 100644 --- a/src/lib/targets/mood-builder.ts +++ b/src/lib/targets/mood-builder.ts @@ -5,6 +5,7 @@ import type { TargetMoodRollup, TargetTrend, } from "./types"; +import { READING_SPREAD, standardDeviation } from "@/lib/stats/spread"; interface MoodTargetInput { moodRollups: TargetMoodRollup[]; @@ -24,9 +25,6 @@ export function buildMoodTargets({ const thirtyDaysAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); let dailyEvents: Array<{ measuredAt: Date; value: number }>; let entryCount: number; - let sumScores: number; - let varianceCount: number; - let sumSquares: number; if (moodRollups.length > 0) { entryCount = moodRollups.reduce((sum, rollup) => sum + rollup.count, 0); @@ -36,12 +34,6 @@ export function buildMoodTargets({ measuredAt: rollup.bucketStart, value: rollup.mean, })); - sumScores = dailyEvents.reduce((sum, event) => sum + event.value, 0); - varianceCount = dailyEvents.length; - sumSquares = dailyEvents.reduce( - (sum, event) => sum + event.value * event.value, - 0, - ); } else if (latestMoodEntry !== null) { const rawMood = recentRawMood ?? []; entryCount = rawMood.length; @@ -49,26 +41,20 @@ export function buildMoodTargets({ measuredAt: entry.moodLoggedAt, value: entry.score, })); - sumScores = rawMood.reduce((sum, entry) => sum + entry.score, 0); - varianceCount = rawMood.length; - sumSquares = rawMood.reduce( - (sum, entry) => sum + entry.score * entry.score, - 0, - ); } else { entryCount = 0; dailyEvents = []; - sumScores = 0; - varianceCount = 0; - sumSquares = 0; } + const values = dailyEvents.map((event) => event.value); if (entryCount < 3) return []; const current = latestMoodEntry?.score ?? null; const average30 = - dailyEvents.length > 0 - ? Math.round((sumScores / dailyEvents.length) * 10) / 10 + values.length > 0 + ? Math.round( + (values.reduce((sum, v) => sum + v, 0) / values.length) * 10, + ) / 10 : null; let trend: TargetTrend = null; if (dailyEvents.length >= 4) { @@ -118,23 +104,21 @@ export function buildMoodTargets({ }, ]; - if (varianceCount >= 5) { - const mean = sumScores / varianceCount; - const variance = sumSquares / varianceCount - mean * mean; - const standardDeviation = - Math.round(Math.sqrt(Math.max(0, variance)) * 100) / 100; + if (values.length >= 5) { + const spread = + Math.round((standardDeviation(values, READING_SPREAD) ?? 0) * 100) / 100; targets.push({ type: "MOOD_STABILITY", label: "Mood stability", - current: standardDeviation, - average30: standardDeviation, + current: spread, + average30: spread, trend: null, unit: "σ", range: { min: 0, max: 0.5 }, classification: - standardDeviation <= 0.5 + spread <= 0.5 ? { category: "Very stable", color: "var(--success)" } - : standardDeviation <= 1 + : spread <= 1 ? { category: "Stable", color: "var(--dracula-yellow)" } : { category: "Fluctuating", color: "var(--destructive)" }, source: "Mood entries", diff --git a/src/lib/workouts/__tests__/canonical-rows.test.ts b/src/lib/workouts/__tests__/canonical-rows.test.ts deleted file mode 100644 index 4250b906a..000000000 --- a/src/lib/workouts/__tests__/canonical-rows.test.ts +++ /dev/null @@ -1,396 +0,0 @@ -/** - * v1.4.42 W5 — write-time canonical-row picker tests. - * - * Pairs with `src/lib/workouts/canonical-rows.ts`. The helper is the - * pre-`createMany` dedup pass invoked by `POST /api/workouts/batch` - * when the v1.5 iOS sprint posts a batch carrying both an Apple Watch - * row and a Withings ScanWatch row for the same Sunday-morning run. - * The tests pin the algorithm contract so a future tightening of the - * window or a tie-breaker tweak surfaces a failing assertion rather - * than a silent regression in the duplicate-drop count. - */ -import { describe, expect, it } from "vitest"; - -import { - dedupeWorkoutBatch, - WORKOUT_DEDUP_WINDOW_MS, - type WorkoutRow, -} from "../canonical-rows"; - -import type { MeasurementSource } from "@/generated/prisma/client"; - -interface TestRow extends WorkoutRow { - /** Marker so the test can assert which input row survived. */ - id: string; -} - -function row(partial: Partial & { id: string }): TestRow { - return { - id: partial.id, - userId: partial.userId ?? "user-1", - activityType: partial.activityType ?? "running", - startedAt: partial.startedAt ?? new Date("2026-05-21T08:00:00Z"), - source: partial.source ?? "APPLE_HEALTH", - caloriesKcal: partial.caloriesKcal ?? null, - createdAt: partial.createdAt ?? null, - index: partial.index, - }; -} - -describe("dedupeWorkoutBatch", () => { - it("returns an empty array on empty input", () => { - expect(dedupeWorkoutBatch([])).toEqual([]); - }); - - it("preserves overlapping workouts from the same source", () => { - const a = row({ - id: "a", - source: "APPLE_HEALTH", - caloriesKcal: 420, - createdAt: new Date("2026-05-21T08:30:00Z"), - }); - const b = row({ - id: "b", - source: "APPLE_HEALTH", - caloriesKcal: 420, - createdAt: new Date("2026-05-21T08:31:00Z"), - }); - - expect(dedupeWorkoutBatch([a, b]).map((item) => item.id)).toEqual([ - "a", - "b", - ]); - }); - - it("prefers the higher-priority source on a cross-source overlap inside the window", () => { - // Apple Watch + Withings ScanWatch both record the same run. The - // window is satisfied (rows are 60 s apart, well under the 90 s - // ceiling) and the activityType matches → group dedups to one - // survivor. APPLE_HEALTH outranks WITHINGS per the ladder. - const apple = row({ - id: "apple", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - caloriesKcal: 400, - }); - const withings = row({ - id: "withings", - source: "WITHINGS", - startedAt: new Date("2026-05-21T08:01:00Z"), - caloriesKcal: 500, - }); - - const out = dedupeWorkoutBatch([apple, withings]); - expect(out).toHaveLength(1); - // Source-axis wins outright — APPLE_HEALTH beats WITHINGS even - // when WITHINGS has higher calories. The calories tie-break only - // fires WITHIN a single source tier. - expect(out[0]!.id).toBe("apple"); - }); - - it("retains every row from the winning source in a cross-source overlap", () => { - const appleFirst = row({ - id: "apple-first", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const withings = row({ - id: "withings", - source: "WITHINGS", - startedAt: new Date("2026-05-21T08:00:30Z"), - }); - const appleSecond = row({ - id: "apple-second", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:01:00Z"), - }); - - expect( - dedupeWorkoutBatch([appleFirst, withings, appleSecond]).map( - (item) => item.id, - ), - ).toEqual(["apple-first", "apple-second"]); - }); - - it("returns every row unchanged when no overlap exists", () => { - // Two runs 10 min apart from the SAME source — well outside the - // 90 s window. Both must survive; the helper is not a generic - // "one run per day" filter. - const first = row({ - id: "first", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const second = row({ - id: "second", - startedAt: new Date("2026-05-21T08:10:00Z"), - }); - - const out = dedupeWorkoutBatch([first, second]); - expect(out.map((r) => r.id)).toEqual(["first", "second"]); - }); - - it("returns rows from different users untouched even on identical timestamps", () => { - // A multi-user batch (theoretically possible from an admin path) - // must never collapse rows across `userId`. The window check - // includes the userId in the grouping key. - const u1 = row({ - id: "u1", - userId: "user-1", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const u2 = row({ - id: "u2", - userId: "user-2", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - - const out = dedupeWorkoutBatch([u1, u2]); - expect(out.map((r) => r.id).sort()).toEqual(["u1", "u2"]); - }); - - it("returns rows of different activity types untouched at the same instant", () => { - // A user opens a "Walk" workout on the Watch and a "Strength" - // workout on the iPhone at the same instant. The window matches - // but the activityType doesn't — two distinct rows, both survive. - const walk = row({ - id: "walk", - activityType: "walking", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const strength = row({ - id: "strength", - activityType: "strength", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - - const out = dedupeWorkoutBatch([walk, strength]); - expect(out.map((r) => r.id).sort()).toEqual(["strength", "walk"]); - }); - - it("preserves same-source workouts when calories differ", () => { - const lo = row({ - id: "lo", - source: "APPLE_HEALTH", - caloriesKcal: 200, - }); - const hi = row({ - id: "hi", - source: "APPLE_HEALTH", - caloriesKcal: 500, - createdAt: new Date("2026-05-21T09:00:00Z"), - }); - - expect(dedupeWorkoutBatch([lo, hi]).map((item) => item.id)).toEqual([ - "lo", - "hi", - ]); - }); - - it("preserves same-source input order when creation times differ", () => { - const early = row({ - id: "early", - source: "WITHINGS", - caloriesKcal: 350, - createdAt: new Date("2026-05-21T08:30:00Z"), - }); - const late = row({ - id: "late", - source: "WITHINGS", - caloriesKcal: 350, - createdAt: new Date("2026-05-21T08:45:00Z"), - }); - - expect(dedupeWorkoutBatch([late, early]).map((item) => item.id)).toEqual([ - "late", - "early", - ]); - }); - - it("preserves same-source input order when every field ties", () => { - const first = row({ id: "first", source: "MANUAL" }); - const second = row({ id: "second", source: "MANUAL" }); - - expect(dedupeWorkoutBatch([first, second]).map((item) => item.id)).toEqual([ - "first", - "second", - ]); - expect(dedupeWorkoutBatch([second, first]).map((item) => item.id)).toEqual([ - "second", - "first", - ]); - }); - - it("treats a row exactly at the ±90 s boundary as in-window", () => { - // The window is inclusive (`<=`). A row landing on the boundary - // joins the group; one millisecond past stays separate. - const anchor = row({ - id: "anchor", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const onBoundary = row({ - id: "boundary", - source: "WITHINGS", - startedAt: new Date(anchor.startedAt.getTime() + WORKOUT_DEDUP_WINDOW_MS), - }); - const justOutside = row({ - id: "outside", - source: "WITHINGS", - startedAt: new Date( - anchor.startedAt.getTime() + WORKOUT_DEDUP_WINDOW_MS + 1, - ), - }); - - // anchor + onBoundary → one group; APPLE_HEALTH wins. - expect(dedupeWorkoutBatch([anchor, onBoundary]).map((r) => r.id)).toEqual([ - "anchor", - ]); - - // anchor + justOutside → two distinct groups, both survive. - expect(dedupeWorkoutBatch([anchor, justOutside]).map((r) => r.id)).toEqual([ - "anchor", - "outside", - ]); - }); - - it("anchors overlap groups to the first row instead of chaining neighbours", () => { - const anchor = row({ - id: "anchor", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const neighbour = row({ - id: "neighbour", - source: "WITHINGS", - startedAt: new Date( - anchor.startedAt.getTime() + WORKOUT_DEDUP_WINDOW_MS - 1_000, - ), - }); - const chained = row({ - id: "chained", - source: "WITHINGS", - startedAt: new Date( - anchor.startedAt.getTime() + 2 * (WORKOUT_DEDUP_WINDOW_MS - 1_000), - ), - }); - - expect( - dedupeWorkoutBatch([anchor, neighbour, chained]).map((item) => item.id), - ).toEqual(["anchor", "chained"]); - }); - - it("keeps anchor windows stable when the batch arrives out of order", () => { - const anchor = row({ - id: "anchor", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const neighbour = row({ - id: "neighbour", - source: "WITHINGS", - startedAt: new Date( - anchor.startedAt.getTime() + WORKOUT_DEDUP_WINDOW_MS - 1_000, - ), - }); - const chained = row({ - id: "chained", - source: "WITHINGS", - startedAt: new Date( - anchor.startedAt.getTime() + 2 * (WORKOUT_DEDUP_WINDOW_MS - 1_000), - ), - }); - - expect( - dedupeWorkoutBatch([neighbour, chained, anchor]).map((item) => item.id), - ).toEqual(["chained", "anchor"]); - }); - - it("walks the full source ladder (APPLE_HEALTH > WITHINGS > MANUAL > IMPORT)", () => { - const sources: MeasurementSource[] = [ - "IMPORT", - "MANUAL", - "WITHINGS", - "APPLE_HEALTH", - ]; - const rows = sources.map((source, idx) => - row({ - id: source, - source, - // Stagger by 10 s so each row sits inside the 90 s window - // relative to the anchor (the first row inserted). - startedAt: new Date( - new Date("2026-05-21T08:00:00Z").getTime() + idx * 10_000, - ), - }), - ); - - expect(dedupeWorkoutBatch(rows).map((r) => r.id)).toEqual(["APPLE_HEALTH"]); - }); - - it("honours a custom user ladder that promotes MANUAL above APPLE_HEALTH", () => { - // v1.4.43 W9 — when the user has customised Settings → Sources to - // promote MANUAL above APPLE_HEALTH, a Manual row inside the 90 s - // window MUST win over the Apple Watch twin. Without the - // user-priority wiring this helper would silently drop the - // user's preferred row at write-time before the read-time picker - // ever saw it. - const apple = row({ - id: "apple", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - caloriesKcal: 500, - }); - const manual = row({ - id: "manual", - source: "MANUAL", - startedAt: new Date("2026-05-21T08:01:00Z"), - caloriesKcal: 300, - }); - - const userPriorityJson = { - steps: ["MANUAL", "APPLE_HEALTH", "WITHINGS"] as MeasurementSource[], - }; - - const out = dedupeWorkoutBatch([apple, manual], userPriorityJson); - expect(out).toHaveLength(1); - expect(out[0]!.id).toBe("manual"); - }); - - it("falls back to the canonical default ladder when userPriorityJson is null", () => { - // Null = no user preference persisted. The helper must walk the - // canonical `DEFAULT_WORKOUT_SOURCE_PRIORITY` ladder so existing - // callers that don't supply the blob keep their v1.4.42 behaviour. - const apple = row({ - id: "apple", - source: "APPLE_HEALTH", - startedAt: new Date("2026-05-21T08:00:00Z"), - }); - const manual = row({ - id: "manual", - source: "MANUAL", - startedAt: new Date("2026-05-21T08:01:00Z"), - }); - - expect(dedupeWorkoutBatch([apple, manual], null).map((r) => r.id)).toEqual([ - "apple", - ]); - expect( - dedupeWorkoutBatch([apple, manual], undefined).map((r) => r.id), - ).toEqual(["apple"]); - // Argument omitted entirely — same behaviour as null/undefined. - expect(dedupeWorkoutBatch([apple, manual]).map((r) => r.id)).toEqual([ - "apple", - ]); - }); - - it("does not mutate caller rows", () => { - // The picker is pure — the caller's array reference and row - // identity survive the call so a future audit-overlay can - // re-inspect the dropped rows. - const a = row({ id: "a", source: "APPLE_HEALTH", caloriesKcal: 100 }); - const b = row({ id: "b", source: "WITHINGS", caloriesKcal: 100 }); - const before = JSON.stringify([a, b]); - dedupeWorkoutBatch([a, b]); - expect(JSON.stringify([a, b])).toBe(before); - }); -}); diff --git a/src/lib/workouts/canonical-rows.ts b/src/lib/workouts/canonical-rows.ts deleted file mode 100644 index b6fbf1688..000000000 --- a/src/lib/workouts/canonical-rows.ts +++ /dev/null @@ -1,308 +0,0 @@ -/** - * v1.4.42 W5 — write-time cross-source canonical-row picker for - * incoming `Workout` ingest payloads. - * - * Why this exists alongside the read-time helper - * ────────────────────────────────────────────── - * - * Two related-but-distinct dedup paths exist in the codebase: - * - * - `pickCanonicalWorkoutRows()` in `src/lib/measurements/` (v1.4.30) - * — READ-time picker invoked by `GET /api/workouts` / - * `GET /api/workouts/[id]`. It walks DB rows, looks up the user's - * persisted `sourcePriorityJson`, buckets by 5-minute slot, and - * filters the read result so the dashboard tile shows one row per - * logical workout. - * - * - This helper (`dedupeWorkoutBatch`, v1.4.42) — WRITE-time picker - * invoked by `POST /api/workouts/batch` BEFORE - * `prisma.workout.createMany`. The name intentionally diverges - * from the read-time picker to make the auto-completion path - * unambiguous (write-time payload-internal dedup vs. read-time - * cross-batch dedup with user-priority resolution). - * The v1.5 iOS sprint will ingest `HKWorkoutType` data in batches - * against `/api/workouts/batch`; Apple Watch + Withings ScanWatch - * paired to the same iPhone frequently capture the same workout - * simultaneously (e.g. a Sunday-morning run) and the iOS app may - * post both rows in one batch. Without write-time dedup, both - * rows would persist (their `externalId`s differ, so the - * `(userId, source, externalId)` unique index doesn't fire) and - * the read-time picker would carry the duplicate forward at every - * downstream read. - * - * The two pickers are layered, not redundant: - * - Write-time picker drops PAYLOAD-internal duplicates inside a - * single batch so we don't pay the storage cost. - * - Read-time picker still runs to collapse cross-batch duplicates - * (e.g. Apple Watch row from Monday's batch + ScanWatch row from - * Tuesday's backfill batch). - * - * Algorithm - * ───────── - * - * 1. Group by `(userId, activityType, startedAt ± 90 s window)`. - * The 90-second window is tighter than the read-time picker's - * 5-minute slot because a single batch from one iOS client has - * no cross-device clock skew to absorb — both rows are stamped - * with HealthKit's `startDate`, which Apple aggregates to the - * same instant across paired sensors. A 90-second cushion - * covers HK's documented ±60s smoothing plus a small buffer. - * 2. Groups containing one source are not duplicates and remain - * unchanged. For a cross-source group, select the winning source - * in order `APPLE_HEALTH ≻ WITHINGS ≻ MANUAL ≻ IMPORT`, then keep - * every row from that source. The literal source ladder comes from - * `DEFAULT_WORKOUT_SOURCE_PRIORITY` so both pickers share one source - * of truth. - * 3. When two sources share a priority tier, prefer the source whose - * best candidate has higher `caloriesKcal`, then earliest `createdAt`, - * then input order. These tie-breakers choose a source; they never - * collapse multiple workouts recorded by that winning source. - * - * Purity contract - * ─────────────── - * - * Zero Prisma imports, zero IO, zero user lookups. The helper takes - * an array, returns an array. Testable in isolation; safely invoked - * from a hot ingest path with no DB round-trip cost. - * - * Source-priority parity with the read-time picker - * ──────────────────────────────────────────────── - * - * CLOSED in v1.4.43 (W9-WORKOUTS-PRIORITY). The read-time picker - * resolves the per-user `sourcePriorityJson` (from - * `User.sourcePriorityJson`) for every read so a user who promoted - * MANUAL above APPLE_HEALTH in Settings → Sources sees their Manual - * rows surface above the Apple Watch ones. This helper now mirrors - * that contract — the optional `userPriorityJson` argument feeds - * through `parseSourcePriority()` + `getSourceLadder(..., "steps")`, - * the same chain the read-time picker walks, so a customised user - * keeps their preferred source at BOTH write- and read-time. - * - * When the argument is omitted (or null/undefined) the helper falls - * back to the canonical `DEFAULT_WORKOUT_SOURCE_PRIORITY` ladder so - * callers that don't yet have the user blob handy stay on the - * v1.4.42 behaviour. The batch route at - * `src/app/api/workouts/batch/route.ts` does one indexed - * `User.sourcePriorityJson` lookup at the start of the handler and - * passes the blob through here; the cost is a single `findUnique` - * per batch, well under the cost of letting a customised user's - * preferred row drop at write-time. - */ - -import { DEFAULT_WORKOUT_SOURCE_PRIORITY } from "@/lib/sources/pick-canonical-workout"; -import { - getSourceLadder, - parseSourcePriority, -} from "@/lib/validations/source-priority"; -import type { MeasurementSource } from "@/generated/prisma/client"; - -/** - * Minimum row shape consulted by the picker. Field names mirror the - * batch-route's prepared `Prisma.WorkoutCreateManyInput` plus a - * `createdAt` slot for the deterministic tie-breaker. Anything else - * the caller carried alongside the row (route geometry, HR series, - * metadata) survives untouched — the picker filters but never - * mutates and is generic in `T` so call sites keep their own - * carrier type. - * - * Naming aligned with the v1.5 iOS contract: - * - `activityType` — the sport type (`"running"`, `"cycling"`, - * ...). Matches `Workout.sportType` at the DB layer and - * `WorkoutSportType` at the validator layer; cross-source - * overlap requires identical sport so a "Walking" + "Running" - * pair at the same instant stays as two distinct rows. - * - `caloriesKcal` — the active-energy aggregate. The v1.5 iOS - * contract uses `caloriesKcal` for parity with HealthKit's - * `HKQuantityTypeIdentifierActiveEnergyBurned`; the server's - * write path maps it to `Workout.totalEnergyKcal`. - * - `createdAt` — when the row was minted. The batch path doesn't - * have this yet (it's set by Prisma's `@default(now())` at - * insert), so the helper falls back to insertion order via the - * `index` field when `createdAt` is undefined on every member. - */ -export interface WorkoutRow { - userId: string; - activityType: string; - startedAt: Date; - source: MeasurementSource; - caloriesKcal?: number | null; - /** - * Optional row-mint timestamp. The write-time picker is called - * BEFORE `createMany`, so prepared rows usually have no - * `createdAt` yet — undefined falls through to the next - * tie-breaker (insertion order via the `index` field). - */ - createdAt?: Date | null; - /** - * Stable per-batch ordinal. Set by the caller (e.g. the position - * of the entry in the inbound payload) so two rows with identical - * priority + calories + createdAt still resolve deterministically. - * Optional; defaults to 0 so single-row groups never need it. - */ - index?: number; -} - -/** - * Cross-source dedup window. Within a single ingest batch from one - * iOS client, two rows whose `startedAt` falls within ±90 s AND - * share the same `activityType` AND belong to the same user are - * treated as the same logical workout. The window is symmetric: - * the comparison uses `|a.startedAt - b.startedAt| ≤ 90 000 ms`. - */ -export const WORKOUT_DEDUP_WINDOW_MS = 90 * 1000; - -/** - * Source-priority rank lookup. Lower rank = higher priority. A - * source absent from the ladder gets a rank past the end of the - * array — the picker still keeps that row when no ladder-listed - * row competes for the slot (single-source group), but loses to - * any ladder-listed competitor. - * - * The ladder is resolved per-call (from the user's - * `sourcePriorityJson` or the canonical default), so the rank - * function is parameterised on the ladder rather than reading a - * module-level constant. - */ -function sourceRank( - source: MeasurementSource, - ladder: readonly MeasurementSource[], -): number { - const idx = ladder.indexOf(source); - return idx === -1 ? ladder.length : idx; -} - -/** - * Comparator: returns negative when `a` should win over `b`. - * Priority order: source rank → higher calories → earliest - * createdAt → earliest input index. - */ -function compareCandidates( - a: T, - b: T, - ladder: readonly MeasurementSource[], -): number { - const sourceDelta = - sourceRank(a.source, ladder) - sourceRank(b.source, ladder); - if (sourceDelta !== 0) return sourceDelta; - - // Tie-breaker 1 — higher caloriesKcal wins (more sensor data). - // `null` / `undefined` calories sort below any numeric value. - const aCal = a.caloriesKcal ?? -1; - const bCal = b.caloriesKcal ?? -1; - if (aCal !== bCal) return bCal - aCal; - - // Tie-breaker 2 — earliest `createdAt` wins (stable across - // re-runs). Undefined sorts below any concrete date so a - // pre-insert row (no createdAt yet) doesn't beat an already- - // persisted row carrying a real timestamp. - const aCreated = a.createdAt?.getTime() ?? Number.POSITIVE_INFINITY; - const bCreated = b.createdAt?.getTime() ?? Number.POSITIVE_INFINITY; - if (aCreated !== bCreated) return aCreated - bCreated; - - // Tie-breaker 3 — input order. Guarantees determinism even when - // every other field ties (rare; same source, same calories, - // same / no createdAt). Default of 0 keeps single-row groups - // untouched. - return (a.index ?? 0) - (b.index ?? 0); -} - -/** - * Pick canonical rows across sources for an inbound ingest batch. - * - * Group by `(userId, activityType, startedAt ± 90 s)`; a group with - * multiple sources keeps every row from its highest-priority source. - * Single-source overlaps pass through unchanged. The returned - * array preserves the input order of the surviving rows so a - * caller that pre-sorted by `startedAt ASC` keeps that ordering - * downstream. - * - * Implementation walks the input once with a union-find-style - * pass: each row joins an existing group whose anchor falls - * within the dedup window AND matches `(userId, activityType)`, - * or seeds a new group. The anchor is the FIRST row that landed - * in the group; subsequent rows compare against the anchor so the - * window can't drift across an arbitrarily long chain of - * within-window-of-neighbour rows. This is O(n²) worst case but - * a typical batch is ≤ 100 entries so the constant is fine. - */ -export function dedupeWorkoutBatch( - rows: readonly T[], - userPriorityJson?: unknown, -): T[] { - if (rows.length === 0) return []; - - // Resolve the source ladder from the user's persisted preference - // when one was supplied. The read-time picker - // (`pickCanonicalWorkoutRows`) keys workouts off the `steps` metric - // ladder — the canonical-activity bucket — so both pickers consult - // the same per-user configuration surface. When no preference is - // supplied (legacy callers, tests that don't care, or a route that - // hasn't been wired yet), fall back to the canonical default - // ladder so behaviour matches the v1.4.42 baseline. - const ladder: readonly MeasurementSource[] = - userPriorityJson == null - ? DEFAULT_WORKOUT_SOURCE_PRIORITY - : getSourceLadder(parseSourcePriority(userPriorityJson), "steps"); - - // Stamp each input with its original index so the comparator - // has a stable tie-breaker without mutating the caller's row. - type Tagged = { row: T; origIndex: number }; - const tagged: Tagged[] = rows.map((row, origIndex) => ({ row, origIndex })); - - // Group anchors: each group is a list of (row, origIndex). The - // anchor row sets the window comparison reference. - const groups: Tagged[][] = []; - - for (const entry of [...tagged].sort((a, b) => { - const delta = a.row.startedAt.getTime() - b.row.startedAt.getTime(); - return delta || a.origIndex - b.origIndex; - })) { - let placed = false; - for (const group of groups) { - const anchor = group[0]!.row; - if ( - anchor.userId === entry.row.userId && - anchor.activityType === entry.row.activityType && - Math.abs(anchor.startedAt.getTime() - entry.row.startedAt.getTime()) <= - WORKOUT_DEDUP_WINDOW_MS - ) { - group.push(entry); - placed = true; - break; - } - } - if (!placed) { - groups.push([entry]); - } - } - - // Preserve single-source groups. For genuine cross-source overlaps, - // select the winning source and retain every row it contributed. - const survivors: Tagged[] = []; - for (const group of groups) { - if (group.length === 1) { - survivors.push(group[0]!); - continue; - } - const sources = new Set(group.map((entry) => entry.row.source)); - if (sources.size === 1) { - survivors.push(...group); - continue; - } - const annotated = group.map((entry) => ({ - ...entry, - row: { - ...entry.row, - index: entry.row.index ?? entry.origIndex, - } as T, - })); - annotated.sort((a, b) => compareCandidates(a.row, b.row, ladder)); - const winningSource = annotated[0]!.row.source; - survivors.push( - ...group.filter((entry) => entry.row.source === winningSource), - ); - } - - survivors.sort((a, b) => a.origIndex - b.origIndex); - return survivors.map((entry) => entry.row); -} diff --git a/tests/integration/insights-comprehensive-cache.test.ts b/tests/integration/insights-comprehensive-cache.test.ts index acd0cced9..fec430c3d 100644 --- a/tests/integration/insights-comprehensive-cache.test.ts +++ b/tests/integration/insights-comprehensive-cache.test.ts @@ -159,7 +159,7 @@ describe("GET /api/insights/comprehensive — server cache", () => { // A tight fixture: 5 paired BP readings + 5 weight rows + 5 mood // entries spread across 5 distinct days. Enough to populate every - // correlation block (n >= 5 is the pearsonCorrelation gate). + // correlation key (each is null below the significance bar). const now = Date.now(); const DAY_MS = 24 * 60 * 60 * 1000; const dayDates = Array.from( diff --git a/tests/integration/latest-canonical-read.test.ts b/tests/integration/latest-canonical-read.test.ts new file mode 100644 index 000000000..88ffbe801 --- /dev/null +++ b/tests/integration/latest-canonical-read.test.ts @@ -0,0 +1,114 @@ +/** + * The latest canonical reading per type, against a real Postgres. + * + * `readLatestCanonical` replaced five `DISTINCT ON (type)` reads that sorted + * every row of the account. The contract it keeps: the latest UTC day decides, + * inside it the source ladder decides (Withings outranks Apple Health for + * weight), inside the winning source the latest reading; a type with no + * reading drops out, and `since` drops a type whose latest reading is older. + * The day window is closed: an older day of the ladder winner never beats a + * newer day of a lower source. + */ +import { beforeAll, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { readLatestCanonical } from "@/lib/measurements/latest-canonical"; + +const prisma = getPrismaClient(); +const USER = "latest-canonical"; +const now = new Date(); +/** UTC midnight two days back. */ +const D = Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() - 2, +); +const at = (day: number, hour: number) => + new Date(D + day * 86_400_000 + hour * 3_600_000); + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, email: `${USER}@example.test` }, + }); + const rows: Array<[string, string, number, Date, string]> = [ + // WEIGHT: the latest day holds Apple at 20:00 and Withings at 07:00; + // Withings ranks first, so 82 wins over the later 81. + ["WEIGHT", "WITHINGS", 80, at(-1, 8), "kg"], + ["WEIGHT", "WITHINGS", 82, at(0, 7), "kg"], + ["WEIGHT", "APPLE_HEALTH", 81, at(0, 20), "kg"], + // BODY_FAT: Withings is the ladder winner but only on an older day; the + // newer Apple day wins. + ["BODY_FAT", "WITHINGS", 20, at(-1, 8), "%"], + ["BODY_FAT", "APPLE_HEALTH", 21, at(0, 9), "%"], + // PULSE: one source, the latest reading of the latest day. + ["PULSE", "APPLE_HEALTH", 60, at(0, 6), "bpm"], + ["PULSE", "APPLE_HEALTH", 64, at(0, 22), "bpm"], + // BLOOD_GLUCOSE: only a reading far in the past. + ["BLOOD_GLUCOSE", "MANUAL", 95, new Date(D - 200 * 86_400_000), "mg/dL"], + ]; + await prisma.measurement.createMany({ + data: rows.map(([type, source, value, measuredAt, unit], i) => ({ + id: `${USER}-${i}`, + userId: USER, + type: type as "WEIGHT", + source: source as "WITHINGS", + value, + unit, + measuredAt, + })), + }); + // A deleted later reading never counts. + await prisma.measurement.create({ + data: { + id: `${USER}-deleted`, + userId: USER, + type: "PULSE", + source: "APPLE_HEALTH", + value: 99, + unit: "bpm", + measuredAt: at(0, 23), + deletedAt: new Date(), + }, + }); +}); + +describe("readLatestCanonical", () => { + it("picks the latest day, then the ladder, then the latest reading", async () => { + const rows = await readLatestCanonical(USER, null); + const byType = new Map(rows.map((r) => [r.type, r])); + expect([...byType.keys()].sort()).toEqual([ + "BLOOD_GLUCOSE", + "BODY_FAT", + "PULSE", + "WEIGHT", + ]); + expect(byType.get("WEIGHT")?.value).toBe(82); + expect(byType.get("WEIGHT")?.measured_at.toISOString()).toBe( + at(0, 7).toISOString(), + ); + expect(byType.get("BODY_FAT")?.value).toBe(21); + expect(byType.get("PULSE")?.value).toBe(64); + expect(byType.get("BLOOD_GLUCOSE")?.value).toBe(95); + }); + + it("drops a type whose latest reading is older than `since`", async () => { + const rows = await readLatestCanonical( + USER, + null, + new Date(Date.now() - 90 * 86_400_000), + ); + expect(rows.map((r) => r.type).sort()).toEqual([ + "BODY_FAT", + "PULSE", + "WEIGHT", + ]); + }); + + it("honours a stored source order", async () => { + const rows = await readLatestCanonical(USER, { + weight: ["APPLE_HEALTH", "WITHINGS"], + }); + expect(rows.find((r) => r.type === "WEIGHT")?.value).toBe(81); + }); +}); diff --git a/tests/integration/local-day-keys.test.ts b/tests/integration/local-day-keys.test.ts new file mode 100644 index 000000000..04b891ea6 --- /dev/null +++ b/tests/integration/local-day-keys.test.ts @@ -0,0 +1,124 @@ +/** + * Local day keys from naive UTC timestamps, against a real Postgres. + * + * `measured_at` and `scheduled_for` hold UTC wall-clock time in `timestamp` + * columns. The dashboard streak and the medication-compliance backfill keyed + * them with a single `AT TIME ZONE tz`, which reads the value as local time + * and shifts it the wrong way. Both now agree with `userDayKey`: a reading at + * 23:30 UTC is the next day in Berlin and still the same day in Los Angeles, + * a reading at 03:00 UTC is the previous day in Los Angeles. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { readActivityDayKeys } from "@/lib/dashboard/summary-reads"; +import { recomputeUserMedicationCompliance } from "@/lib/rollups/medication-compliance-rollups"; +import { userDayKey } from "@/lib/tz/format"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const USER = "local-day-keys"; +const now = new Date(); +/** UTC midnight ten days back. */ +const D = Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() - 10, +); +const LATE = new Date(D + 23.5 * 3_600_000); // 23:30 UTC on D +const EARLY = new Date(D + 3 * 3_600_000); // 03:00 UTC on D +const key = (ms: number) => new Date(ms).toISOString().slice(0, 10); + +beforeEach(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { + id: USER, + username: USER, + email: `${USER}@example.test`, + timezone: "Europe/Berlin", + }, + }); +}); + +describe("readActivityDayKeys", () => { + beforeEach(async () => { + await prisma.measurement.createMany({ + data: [ + { at: LATE, deleted: false }, + { at: EARLY, deleted: false }, + { at: new Date(D - 3 * 86_400_000), deleted: true }, + ].map(({ at, deleted }, i) => ({ + id: `${USER}-m${i}`, + userId: USER, + type: "WEIGHT" as const, + source: "MANUAL" as const, + value: 80, + unit: "kg", + measuredAt: at, + deletedAt: deleted ? new Date() : null, + })), + }); + }); + + const since = new Date(D - 5 * 86_400_000); + + it("keys a late reading onto the next day east of UTC", async () => { + const keys = await readActivityDayKeys( + prisma, + USER, + "Europe/Berlin", + since, + now, + ); + expect(keys).toEqual([key(D), key(D + 86_400_000)]); + expect(keys).toEqual( + [EARLY, LATE].map((at) => userDayKey(at, "Europe/Berlin")), + ); + }); + + it("keys an early reading onto the previous day west of UTC", async () => { + const keys = await readActivityDayKeys( + prisma, + USER, + "America/Los_Angeles", + since, + now, + ); + expect(keys).toEqual([key(D - 86_400_000), key(D)]); + expect(keys).toEqual( + [EARLY, LATE].map((at) => userDayKey(at, "America/Los_Angeles")), + ); + }); +}); + +describe("recomputeUserMedicationCompliance", () => { + it("keys a late dose onto the person's own day", async () => { + const med = await prisma.medication.create({ + data: { userId: USER, name: "Ramipril", dose: "5mg", active: true }, + }); + await prisma.medicationIntakeEvent.create({ + data: { + userId: USER, + medicationId: med.id, + scheduledFor: LATE, + takenAt: LATE, + skipped: false, + source: "API", + }, + }); + + await recomputeUserMedicationCompliance(USER, 30, "Europe/Berlin"); + + const rows = await prisma.medicationComplianceRollup.findMany({ + where: { userId: USER }, + select: { day: true, scheduled: true, taken: true }, + }); + expect(rows).toEqual([ + { day: key(D + 86_400_000), scheduled: 1, taken: 1 }, + ]); + }); +}); diff --git a/tests/integration/workout-batch-create.test.ts b/tests/integration/workout-batch-create.test.ts index b7f7f3274..190fa34ef 100644 --- a/tests/integration/workout-batch-create.test.ts +++ b/tests/integration/workout-batch-create.test.ts @@ -356,8 +356,16 @@ describe("POST /api/workouts/batch (real Postgres)", () => { it("preserves same-source workouts and surfaces exact re-posts as duplicates", async () => { const { POST } = await import("@/app/api/workouts/batch/route"); + // Two sessions of one source in the same five-minute slot: different + // start and length, so two workouts. const body = { - workouts: [baseWorkout("hk-uuid-dup-1"), baseWorkout("hk-uuid-dup-2")], + workouts: [ + baseWorkout("hk-uuid-dup-1"), + baseWorkout("hk-uuid-dup-2", { + startedAt: "2026-05-14T06:32:00.000Z", + endedAt: "2026-05-14T06:50:00.000Z", + }), + ], }; const first = await POST(makeRequest(body)); @@ -393,6 +401,31 @@ describe("POST /api/workouts/batch (real Postgres)", () => { expect(stored).toBe(2); }); + it("stores a session re-sent under a new id once, the way the list shows it", async () => { + // HealthKit can hand the same session over twice with two external ids. + // The list collapses it to one workout; the write keeps one row and + // reports the other as a duplicate instead of storing what no read shows. + const { POST } = await import("@/app/api/workouts/batch/route"); + const res = await POST( + makeRequest({ + workouts: [ + baseWorkout("hk-uuid-resend-1"), + baseWorkout("hk-uuid-resend-2"), + ], + }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { + data: { inserted: number; duplicates: number }; + }; + expect(json.data.inserted).toBe(1); + expect(json.data.duplicates).toBe(1); + const stored = await getPrismaClient().workout.count({ + where: { userId: TEST_USER_ID }, + }); + expect(stored).toBe(1); + }); + it("replays a cached response when the same Idempotency-Key is reused", async () => { const { POST } = await import("@/app/api/workouts/batch/route"); diff --git a/tests/integration/workout-batch-race.test.ts b/tests/integration/workout-batch-race.test.ts index bc5a9361c..d55aed62b 100644 --- a/tests/integration/workout-batch-race.test.ts +++ b/tests/integration/workout-batch-race.test.ts @@ -182,9 +182,9 @@ describe("POST /api/workouts/batch — concurrent-write race", () => { const { POST } = await import("@/app/api/workouts/batch/route"); // Batch A — a cross-source pair one minute apart plus one shared key. - // The v1.4.42 `dedupeWorkoutBatch` picker collapses the Apple Health + - // Manual pair (same activity, within the 90 s window) to one - // surviving row, leaving 2 effective rows per batch. + // The workout picker collapses the Apple Health + Manual pair (same + // activity, same five-minute slot) to one surviving row, leaving 2 + // effective rows per batch. const batchA: WorkoutFixture[] = [ workout("hk-uuid-only-a-0", 1), workout("hk-uuid-only-a-1", 2, "MANUAL"),