From 8ffa9cd87ada0e70672751bef40fb4640937279c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 23 Aug 2026 17:37:11 +0200 Subject: [PATCH] fix(insights): scan one discovery matrix from every correlation surface Four surfaces assembled the correlation-discovery matrix and three carried a different channel set. The Coach `get_correlations` tool had neither the environmental-exposure channels nor the custom-metric ones, so asking it whether air pressure relates to your sleep met a matrix the pair had never been in. Its own file header said the opposite - "the SAME full-matrix scan the route runs" - and had said it since the weather channels arrived in v1.25. The period narrative was missing those two families and the compliance / symptom pair on top. `src/lib/insights/discovery-matrix.ts` is now the one place a matrix is built. v1.30.3 hoisted the fetch discipline into a shared module for exactly this reason and left the assembly duplicated behind; this finishes it. The route, the per-metric card, the Coach tool and the period narrative all call `assembleDiscoveryMatrix` and get the same families. What still differs is declared as an option with its reason at the declaration: the measurement read mode (a read-cost choice, left per surface), the extra metric types the narrative's delta beat needs, and the RATED mood factors. Mood factors stay on the narrative alone. That capture kind is frozen, the two surfaces that persist a discovered pair write its channel key into a plaintext column, and a custom factor's key is an opaque id - so admitting them elsewhere is a decision about what to ask, not a parity fix. Restore the per-metric filter on the assessment card's relations. It was written in place of the pattern-dismissal filter rather than chained with it, so a weight card's prompt had been receiving every surviving pair in the record while the function name, the docstring and the early-return guard all still said one metric. `discovery-matrix-guard.test.ts` freezes who may fetch a channel, who may construct one, and who may scan. Each matcher asserts a non-zero match count, so a matcher that has stopped matching fails rather than agreeing with the allowlist. The statistics are untouched - same Pearson, same exact p-value, same Benjamini-Hochberg, same floors. The Coach's family is larger now, which is the point: it is corrected over the same family the insights page is. --- src/__tests__/discovery-matrix-guard.test.ts | 195 +++++++++++ src/app/api/insights/correlations/route.ts | 141 ++------ .../coach/__tests__/correlations-read.test.ts | 103 ++++++ src/lib/ai/coach/tools/correlations-read.ts | 156 ++++----- .../metric-correlation-context.test.ts | 25 +- .../insights/correlation-channel-series.ts | 156 ++++++++- .../__tests__/coach-read-locale.test.ts | 10 + src/lib/insights/discovery-matrix.ts | 326 ++++++++++++++++++ .../insights/metric-correlation-context.ts | 134 +++---- .../__tests__/period-narrative-db.test.ts | 74 ++++ .../insights/narrative/period-narrative.ts | 259 ++++---------- 11 files changed, 1060 insertions(+), 519 deletions(-) create mode 100644 src/__tests__/discovery-matrix-guard.test.ts create mode 100644 src/lib/insights/discovery-matrix.ts diff --git a/src/__tests__/discovery-matrix-guard.test.ts b/src/__tests__/discovery-matrix-guard.test.ts new file mode 100644 index 000000000..ca4d3c9f2 --- /dev/null +++ b/src/__tests__/discovery-matrix-guard.test.ts @@ -0,0 +1,195 @@ +/** + * Structural guards on the correlation-discovery matrix. + * + * The matrix is a channel SET, and a channel set copied into four files is a + * channel set that will differ in three of them. It did: the route and the + * per-metric card folded in the environmental-exposure and custom-metric + * families, the Coach `get_correlations` tool folded in neither, and the + * period narrative folded in neither of those plus neither of the + * compliance / symptom pair. Nobody wrote anything wrong — a family was added + * where it was needed and the other call sites were not on anyone's screen. + * `src/lib/insights/discovery-matrix.ts` removes the copies; these guards are + * what stop a fifth one appearing. + * + * ## What each guard rests on + * + * T1 freezes who may FETCH a channel. A new surface that wants a matrix has to + * either call the assembler or import a fetcher, and the second is the one this + * catches. + * + * T2 freezes who may CONSTRUCT a channel — the `role: "behaviour" | "outcome"` + * literal that makes a `NamedSeries`. A hand-rolled channel that never touches + * `correlation-channel-series.ts` would slip T1 and lands here. + * + * T3 freezes who may SCAN. This is the weakest of the three and the most + * likely to read as the strong one, so: passing T3 proves only that the list of + * files calling `discoverCorrelations` has not changed silently. It says + * nothing about what series those files pass. T1 and T2 are what make the + * series the same series. + * + * ## What they cannot do + * + * They are tripwires, not proofs. A reviewer who waves through an addition to + * an allowlist defeats all three. They match import statements and source + * text, so a fetcher reached through a re-export, a dynamic `import()`, or a + * channel built by a helper that takes `role` as a parameter would pass + * unseen. And they cannot tell a deliberate divergence from an accidental one: + * `assembleDiscoveryMatrix`'s options are where a real difference gets + * declared, and only a reader can judge whether the reason written there is + * a reason. + * + * Every match set below asserts a non-zero count. A matcher that has silently + * stopped matching agrees with any allowlist, and that failure mode is exactly + * how `bearer-scope-enforcement-guard` stayed green over a file it never saw. + */ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +/** The one assembler. Every allowlist below is "this, plus the unavoidable". */ +const ASSEMBLER = "lib/insights/discovery-matrix.ts"; + +/** The module the channel fetchers live in — it defines them, so it matches. */ +const CHANNEL_SERIES = "lib/insights/correlation-channel-series.ts"; + +/** + * Every non-test `.ts` / `.tsx` under `src/`, excluding the generated Prisma + * client (9 MB; never read it) and test files themselves. + */ +function sourceFiles(): string[] { + return walkSourceFiles(SRC, { floor: 3000 }) + .filter((p) => !p.startsWith("generated/")) + .filter((p) => !p.includes("__tests__")) + .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx")) + .sort(); +} + +function read(rel: string): string { + return readFileSync(join(SRC, rel), "utf8"); +} + +function filesMatching(re: RegExp): string[] { + return sourceFiles().filter((rel) => re.test(read(rel))); +} + +describe("T1 — only the assembler fetches discovery channels", () => { + /** + * The per-channel DB reads. A surface holding these is assembling a matrix + * by hand, whatever it calls the variable. + * + * `fetchLabDraws` is deliberately absent: the labs pass runs over + * `LabDrawPoint[]` in a separate point-vs-window scan, not over the + * behaviour × outcome day grid, so the route and the Coach tool fetch it + * directly and the assembler does not own it. + */ + const CHANNEL_FETCHERS = [ + "fetchComplianceSeries", + "fetchSymptomSeries", + "fetchEnvironmentSeries", + "fetchCustomMetricBehaviourSeries", + "fetchMoodWindowSeries", + "fetchMoodFactorWindowSeries", + "fetchMeasurementDailySeriesTiered", + "fetchMeasurementWindowSeries", + ] as const; + + /** + * `correlation-channel-series.ts` defines them all. The assembler is the one + * consumer. Nothing else. + */ + const FETCH_ALLOWLIST = [ASSEMBLER, CHANNEL_SERIES].sort(); + + for (const fetcher of CHANNEL_FETCHERS) { + it(`${fetcher} is reached only from the assembler`, () => { + const matches = filesMatching(new RegExp(`\\b${fetcher}\\b`)); + // A matcher that found nothing would agree with any allowlist. + expect(matches.length).toBeGreaterThan(0); + expect(matches).toEqual(FETCH_ALLOWLIST); + }); + } +}); + +describe("T2 — only the declared builders construct a discovery channel", () => { + /** + * A `NamedSeries` is identified by its `role` literal. Whitespace-tolerant: + * the same call written across two lines is the same call, and a matcher + * that demands one line is a matcher that can be defeated by prettier. + */ + const ROLE_LITERAL = /role:\s*"(?:behaviour|outcome)"/; + + /** + * Files allowed to mint a channel. + * + * - the assembler, which folds the matrix; + * - `correlation-channel-series.ts` + `correlation-series-builders.ts`, + * which shape one channel each from its own source; + * - `correlation-discovery.ts`, which declares the `NamedSeries` type and + * re-runs the engine over a filtered copy in the emerging pass; + * - `cycle/phase-crosstab.ts`, which builds a DELIBERATELY separate, + * cycle-gated matrix (CYCLE_PHASE × outcomes). That one must never reach + * the non-gated surfaces, which is the opposite requirement to this + * file's, and is why it is listed rather than merged. + */ + const CONSTRUCTION_ALLOWLIST = [ + ASSEMBLER, + CHANNEL_SERIES, + "lib/insights/correlation-series-builders.ts", + "lib/insights/correlation-discovery.ts", + "lib/cycle/phase-crosstab.ts", + ].sort(); + + it("no file outside the allowlist constructs a NamedSeries", () => { + const matches = filesMatching(ROLE_LITERAL); + expect(matches.length).toBeGreaterThan(0); + expect(matches).toEqual(CONSTRUCTION_ALLOWLIST); + }); +}); + +describe("T3 — the set of files that scan a matrix is frozen", () => { + /** + * Whitespace-tolerant on purpose: `discoverCorrelations(\n series,` is the + * same call, and a literal `discoverCorrelations(` matcher would miss it. + */ + const SCAN_CALL = /\bdiscoverCorrelations\s*\(/; + + /** + * The four discovery surfaces, plus the two engine-internal uses. + * + * Being on this list is NOT permission to build a matrix — T1 and T2 decide + * that. It is permission to scan one, and every entry here gets its series + * from `assembleDiscoveryMatrix` except the two that are the engine. + */ + const SCAN_ALLOWLIST = [ + // The four surfaces. Each calls the assembler for its series. + "app/api/insights/correlations/route.ts", + "lib/insights/metric-correlation-context.ts", + "lib/ai/coach/tools/correlations-read.ts", + "lib/insights/narrative/period-narrative.ts", + // The engine itself: the declaration and the emerging-window re-run. + "lib/insights/correlation-discovery.ts", + // The cycle-gated phase matrix — see T2's note. + "lib/cycle/phase-crosstab.ts", + ].sort(); + + it("no file outside the allowlist runs the discovery scan", () => { + const matches = filesMatching(SCAN_CALL); + expect(matches.length).toBeGreaterThan(0); + expect(matches).toEqual(SCAN_ALLOWLIST); + }); + + it("every scanning surface takes its series from the assembler", () => { + const surfaces = SCAN_ALLOWLIST.filter( + (rel) => + rel !== "lib/insights/correlation-discovery.ts" && + rel !== "lib/cycle/phase-crosstab.ts", + ); + expect(surfaces).toHaveLength(4); + for (const rel of surfaces) { + expect(read(rel)).toMatch(/\bassembleDiscoveryMatrix\s*\(/); + } + }); +}); diff --git a/src/app/api/insights/correlations/route.ts b/src/app/api/insights/correlations/route.ts index bde103c8f..7ae6557da 100644 --- a/src/app/api/insights/correlations/route.ts +++ b/src/app/api/insights/correlations/route.ts @@ -11,9 +11,12 @@ * causal. * * Reads daily series bounded to a trailing window, day-keyed in the user's - * display timezone (late-night readings mis-bucket under UTC). Pure compute - * lives in `src/lib/insights/correlation-discovery.ts`; this route only - * fetches + day-keys + responds. No LLM, no narrative, no cache table. + * display timezone (late-night readings mis-bucket under UTC). The channel set + * comes from `src/lib/insights/discovery-matrix.ts`, the one assembler every + * discovery surface shares; the pure compute lives in + * `src/lib/insights/correlation-discovery.ts`. This route only asks for the + * matrix, scans it, persists the pattern decisions and responds. No LLM, no + * narrative, no cache table. */ import { apiHandler, requireRecordAuth } from "@/lib/api-handler"; import { apiError, apiSuccess } from "@/lib/api-response"; @@ -25,34 +28,19 @@ import { requireModuleEnabled } from "@/lib/modules/gate"; import { resolveServerLocale } from "@/lib/i18n/server-locale"; import { prisma } from "@/lib/db"; import { wallClockInTz } from "@/lib/tz/wall-clock"; -import type { MeasurementType } from "@/generated/prisma/client"; import { discoverCorrelations, discoverEmergingCorrelations, discoverLabOutcomeCorrelations, - discoveryMeasurementTypes, - DISCOVERY_BEHAVIOURS, - DISCOVERY_OUTCOMES, EARLY_WINDOW_DAYS, - MEDICATION_COMPLIANCE_CHANNEL_KEY, - SYMPTOM_SEVERITY_CHANNEL_KEY, - type DailySeriesPoint, - type NamedSeries, } from "@/lib/insights/correlation-discovery"; import { decisionForEvidence, PATTERN_FAMILIES, syncAcceptedPatterns, } from "@/lib/insights/correlation-patterns"; -import { - fetchComplianceSeries, - fetchCustomMetricBehaviourSeries, - fetchEnvironmentSeries, - fetchLabDraws, - fetchMeasurementDailySeriesTiered, - fetchMoodWindowSeries, - fetchSymptomSeries, -} from "@/lib/insights/correlation-channel-series"; +import { assembleDiscoveryMatrix } from "@/lib/insights/discovery-matrix"; +import { fetchLabDraws } from "@/lib/insights/correlation-channel-series"; export const dynamic = "force-dynamic"; @@ -125,85 +113,22 @@ async function buildCorrelationsResponse( const now = new Date(); const since = new Date(now.getTime() - WINDOW_DAYS * MS_PER_DAY); - // Non-MeasurementType channels are backed by other models, not measurements — - // MOOD (MoodEntry), MEDICATION_COMPLIANCE (the dose-history ledger), and - // SYMPTOM_SEVERITY (the illness day-log). `discoveryMeasurementTypes` drops - // them so the `type IN (...)` query carries only real enum values; each is - // built from its own source below and folded into the series. - const behaviourTypes = discoveryMeasurementTypes( - DISCOVERY_BEHAVIOURS, - ) as MeasurementType[]; - const outcomeTypes = discoveryMeasurementTypes( - DISCOVERY_OUTCOMES, - ) as MeasurementType[]; - - // Rollup read-swap for the measurement channels: eligible spot channels - // read their per-day means from the DAY rollup tier, with per-channel - // fallback to the raw `fetchMeasurementWindowSeries` path on any miss - // (coverage gap, far-from-UTC profile tz, sleep / cumulative grain) — - // see `fetchMeasurementDailySeriesTiered` for the full parity contract. - // The SWR cache cell above stays in front of this read: the swap only - // lowers the cost of a cache MISS, it does not replace the cache. - const [measurementDaily, moodWindow] = await Promise.all([ - fetchMeasurementDailySeriesTiered(userId, tz, since, [ - ...behaviourTypes, - ...outcomeTypes, - ]), - fetchMoodWindowSeries(userId, tz, since), - ]); - const { measurementsCapped } = measurementDaily; - const { moodDaily, moodCapped } = moodWindow; - - // v1.21.0 (FDREXTEND) — build the two non-measurement, non-mood channels from - // their own sources. Each degrades to an empty series when the user has no - // data, so the discovery loop drops the channel (it cannot clear n ≥ 20). - // v1.22 — lab draws (for the labs ↔ outcome pass) fetch alongside. - const [ - complianceSeries, - symptomSeries, - labDraws, - environmentSeries, - customMetricSeries, - ] = await Promise.all([ - fetchComplianceSeries(userId, tz, since), - fetchSymptomSeries(userId, tz, since), + // The channel set comes from the one assembler every discovery surface + // shares (`discovery-matrix.ts`) — this route, the per-metric card, the Coach + // tool and the period narrative all scan the same matrix by construction. + // `"tiered"` puts the eligible measurement channels on the DAY rollup + // read-swap, with per-channel fallback to the raw path on any miss (coverage + // gap, far-from-UTC profile tz, sleep / cumulative grain). The SWR cache cell + // above stays in front of this read: the swap only lowers the cost of a cache + // MISS, it does not replace the cache. + // + // v1.22 — lab draws (for the labs ↔ outcome pass) fetch alongside; they feed + // a different pass over a different grain, so they are not matrix channels. + const [matrix, labDraws] = await Promise.all([ + assembleDiscoveryMatrix(userId, { tz, since, fetchMode: "tiered" }), fetchLabDraws(userId, tz, since), - // v1.25 (W-ENV) — environmental-exposure behaviour channels (weather / - // daylight). Empty when the module is off / no home set, so the channels - // degrade to absent. The module gate is implicit: no rows ⇒ no channels. - fetchEnvironmentSeries(userId, since), - fetchCustomMetricBehaviourSeries(userId, tz, since), ]); - - const points = (key: string): DailySeriesPoint[] => - key === "MOOD" ? moodDaily : (measurementDaily.byType.get(key) ?? []); - - const series: NamedSeries[] = []; - for (const key of DISCOVERY_BEHAVIOURS) { - if (key === MEDICATION_COMPLIANCE_CHANNEL_KEY) { - series.push(complianceSeries); - } else if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "behaviour" }); - } else { - series.push({ key, role: "behaviour", points: points(key) }); - } - } - for (const key of DISCOVERY_OUTCOMES) { - if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "outcome" }); - } else { - series.push({ key, role: "outcome", points: points(key) }); - } - } - // v1.25 (W-ENV) — fold the environmental-exposure behaviour channels in. They - // pair (lag D → D+1) against every outcome above; the n ≥ 20 / FDR / effect- - // size gates apply unchanged, so a thin weather series degrades to absent. - for (const envSeries of environmentSeries) { - series.push(envSeries); - } - for (const customSeries of customMetricSeries) { - series.push(customSeries); - } + const { series, diagnostics } = matrix; const result = discoverCorrelations(series, { locale }); @@ -278,15 +203,15 @@ async function buildCorrelationsResponse( // half of the window may be thin. // The cap can only apply to raw-path channels — rollup-served // channels read one row per day per source and cannot reach it. - measurements_capped: measurementsCapped, + measurements_capped: diagnostics.measurementsCapped, // Rollup read-swap reach: how many measurement channels rode the DAY // rollup tier on this build (the rest took the raw fallback path). - measurement_rollup_channels: measurementDaily.rollupTypes.length, - mood_entries_capped: moodCapped, + measurement_rollup_channels: diagnostics.rollupTypes.length, + mood_entries_capped: diagnostics.moodCapped, // FDREXTEND — per-channel day-counts so a dashboard can see whether the // two sparse new channels reached the n ≥ 20 floor or degraded to absent. - compliance_days: complianceSeries.points.length, - symptom_days: symptomSeries.points.length, + compliance_days: diagnostics.complianceDays, + symptom_days: diagnostics.symptomDays, // v1.22 — early-detection + labs reach. emerging: emerging.emerging.length, emerging_window_days: emerging.windowDays, @@ -295,15 +220,9 @@ async function buildCorrelationsResponse( // v1.25 (W-ENV) — env channel reach (sum of stored daily points across // the exposure channels) so a dashboard can see whether weather was // available for the scan. - environment_days: environmentSeries.reduce( - (sum, s) => sum + s.points.length, - 0, - ), - custom_metric_channels: customMetricSeries.length, - custom_metric_days: customMetricSeries.reduce( - (sum, channel) => sum + channel.points.length, - 0, - ), + environment_days: diagnostics.environmentDays, + custom_metric_channels: diagnostics.customMetricChannels, + custom_metric_days: diagnostics.customMetricDays, }, }); diff --git a/src/lib/ai/coach/__tests__/correlations-read.test.ts b/src/lib/ai/coach/__tests__/correlations-read.test.ts index 98f93b41f..b5c324aaa 100644 --- a/src/lib/ai/coach/__tests__/correlations-read.test.ts +++ b/src/lib/ai/coach/__tests__/correlations-read.test.ts @@ -13,6 +13,8 @@ const intakeEventFindMany = vi.fn(); const illnessEpisodeFindMany = vi.fn(); const illnessDayLogFindMany = vi.fn(); const labResultFindMany = vi.fn(); +const environmentContextFindMany = vi.fn(); +const customMetricFindMany = vi.fn(); vi.mock("@/lib/modules/gate", () => ({ isModuleEnabled: vi.fn(async () => true), })); @@ -26,6 +28,14 @@ vi.mock("@/lib/db", () => ({ illnessEpisode: { findMany: (a: unknown) => illnessEpisodeFindMany(a) }, illnessDayLog: { findMany: (a: unknown) => illnessDayLogFindMany(a) }, labResult: { findMany: (a: unknown) => labResultFindMany(a) }, + // The reader assembles the SHARED discovery matrix, so the + // environmental-exposure and custom-metric channels are read here too. + // Before that these two models were never touched on this path, which is + // the whole defect: the Coach could not correlate weather with anything. + environmentContext: { + findMany: (a: unknown) => environmentContextFindMany(a), + }, + customMetric: { findMany: (a: unknown) => customMetricFindMany(a) }, }, })); @@ -65,6 +75,8 @@ describe("readCoachCorrelations", () => { illnessEpisodeFindMany.mockReset().mockResolvedValue([]); illnessDayLogFindMany.mockReset().mockResolvedValue([]); labResultFindMany.mockReset().mockResolvedValue([]); + environmentContextFindMany.mockReset().mockResolvedValue([]); + customMetricFindMany.mockReset().mockResolvedValue([]); loadBaselineProfile .mockReset() .mockResolvedValue({ ageYears: 40, sex: "MALE", heightCm: 180 }); @@ -260,4 +272,95 @@ describe("readCoachCorrelations", () => { expect(symptomDriver?.outcome).toBe("sleep duration"); expect(symptomDriver?.n).toBeGreaterThanOrEqual(20); }); + + // ── the matrix the Coach scans is the shared one ──────────────────────── + // + // The Coach tool's header promised, from v1.21.0, "the SAME full-matrix scan + // the /api/insights/correlations route + the per-metric card run". It was not + // true from v1.25 onward: the environmental-exposure channels landed at the + // route and at the card, never here, because each surface folded its own + // channel list. A user asking the Coach whether air pressure related to their + // sleep met a matrix the pair had never been in. + // + // These two tests are the pair that matters. The first pins the read (before + // the shared assembler, `EnvironmentContext` was never queried on this path + // at all — the mock below would record zero calls). The second drives the + // REAL engine end-to-end and asserts the pair comes out the other side as a + // narratable driver. + + it("reads the environmental-exposure channels (the route's matrix, not a subset)", async () => { + discoverCorrelations.mockReturnValue({ + discovered: [], + pairsTested: 0, + fdrQ: 0.1, + minPairs: 20, + }); + await readCoachCorrelations("u1", "en"); + expect(environmentContextFindMany).toHaveBeenCalledTimes(1); + expect(customMetricFindMany).toHaveBeenCalledTimes(1); + + // Not just "a query happened": the channels reach the scan. + const series = discoverCorrelations.mock.calls[0]?.[0] as Array<{ + key: string; + }>; + const keys = series.map((s) => s.key); + expect(keys).toContain("ENV_PRESSURE_MEAN"); + expect(keys).toContain("ENV_DAYLIGHT"); + }); + + it("surfaces a barometric-pressure driver through the real engine", async () => { + const real = await vi.importActual< + typeof import("@/lib/insights/correlation-discovery") + >("@/lib/insights/correlation-discovery"); + discoverCorrelations.mockImplementation((series) => + real.discoverCorrelations( + series as Parameters[0], + { locale: "en" }, + ), + ); + + // 45 consecutive days. Today's barometric pressure (behaviour, day D) + // tracks the NEXT day's sleep (outcome, day D+1) with a deterministic + // linear relationship, so the pair clears n ≥ 20 / p < 0.05 / FDR / the + // effect-size floor rather than riding on noise. + const DAYS = 45; + const base = Date.UTC(2026, 4, 1, 12, 0, 0); // midday → tz-stable day key + const dayKey = (offset: number): string => + new Date(base + offset * 86_400_000).toISOString().slice(0, 10); + + const envRows: Array<{ date: string; pressureMean: number }> = []; + const measurements: Array<{ + type: string; + value: number; + measuredAt: Date; + }> = []; + for (let i = 0; i < DAYS; i++) { + const swing = i % 5; // 0..4, real variance across the window + envRows.push({ date: dayKey(i), pressureMean: 1000 + swing }); + // Higher pressure today → shorter sleep tomorrow (a clean negative r). + measurements.push({ + type: "SLEEP_DURATION", + value: 8 - swing * 0.5, + measuredAt: new Date(base + (i + 1) * 86_400_000), + }); + } + + measurementFindMany.mockResolvedValue(measurements); + // Every other env column is null, so only the pressure channel carries + // points and the assertion below cannot be satisfied by a sibling field. + environmentContextFindMany.mockResolvedValue(envRows); + + const result = await readCoachCorrelations("u1", "en"); + + expect(result.present).toBe(true); + const pressureDriver = result.drivers?.find( + (d) => d.behaviour === "barometric pressure", + ); + expect(pressureDriver).toBeDefined(); + expect(pressureDriver?.outcome).toBe("sleep duration"); + expect(pressureDriver?.n).toBeGreaterThanOrEqual(20); + expect(pressureDriver?.direction).toBe("lower"); + // The channel key must never leak into prose the Coach reads out. + expect(pressureDriver?.behaviour).not.toContain("ENV_"); + }); }); diff --git a/src/lib/ai/coach/tools/correlations-read.ts b/src/lib/ai/coach/tools/correlations-read.ts index 8bf7f8fd4..2ad9c1716 100644 --- a/src/lib/ai/coach/tools/correlations-read.ts +++ b/src/lib/ai/coach/tools/correlations-read.ts @@ -6,11 +6,23 @@ * computed elsewhere and reused verbatim here (no new statistics): * * - The FDR-controlled, day-D→D+1 lagged all-pairs discovery - * (`@/lib/insights/correlation-discovery`). We run the SAME full-matrix scan - * the `/api/insights/correlations` route + the per-metric card run, so the - * Coach never surfaces a pair the insight pages would not — same Pearson, - * same exact p-value, same Benjamini-Hochberg control. We return every - * surviving pair (not filtered to one metric) as a descriptive driver row. + * (`@/lib/insights/correlation-discovery`), over the channel set the one + * shared assembler builds (`@/lib/insights/discovery-matrix`). That is the + * same matrix the `/api/insights/correlations` route and the per-metric card + * scan, so the Coach never surfaces a pair the insight pages would not, and + * never misses one they would — same channels, same Pearson, same exact + * p-value, same Benjamini-Hochberg control. We return every surviving pair + * (not filtered to one metric) as a descriptive driver row. + * + * This file claimed that parity from v1.21.0 and did not have it. When the + * environmental-exposure channels arrived in v1.25 and the custom-metric + * channels after them, both landed at the route and at the card and not + * here, because each surface folded its own channel list. The sentence above + * stayed, which is worse than no sentence: a reader checking whether the + * Coach could discuss barometric pressure and sleep found a promise that it + * could, and the pair had never been in its matrix. The promise is now + * structural — the assembler is the only place a matrix is built, frozen by + * `src/__tests__/discovery-matrix-guard.test.ts`. * - The coincident-deviation flag (`computeCoincidentDeviation`): "two or more * of your vitals are outside their usual band today", with the illness- * explained reframe carried through. @@ -22,8 +34,8 @@ * interpretation string — so the Coach states the observed linkage without * inventing a relationship. */ -import type { MeasurementType } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; +import { ENVIRONMENT_FIELDS } from "@/lib/environment/fields"; import type { Locale } from "@/lib/i18n/config"; import { annotate } from "@/lib/logging/context"; import { isModuleEnabled } from "@/lib/modules/gate"; @@ -32,23 +44,12 @@ import { discoverCorrelations, discoverEmergingCorrelations, discoverLabOutcomeCorrelations, - discoveryMeasurementTypes, - DISCOVERY_BEHAVIOURS, - DISCOVERY_OUTCOMES, EARLY_WINDOW_DAYS, MEDICATION_COMPLIANCE_CHANNEL_KEY, SYMPTOM_SEVERITY_CHANNEL_KEY, - type NamedSeries, } from "@/lib/insights/correlation-discovery"; -import { - buildMeasurementDailySeries, - fetchComplianceSeries, - fetchLabDraws, - fetchMeasurementWindowSeries, - fetchMoodWindowSeries, - fetchSymptomSeries, -} from "@/lib/insights/correlation-channel-series"; -import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { assembleDiscoveryMatrix } from "@/lib/insights/discovery-matrix"; +import { fetchLabDraws } from "@/lib/insights/correlation-channel-series"; import { computeCoincidentDeviation, loadBaselineProfile, @@ -133,14 +134,30 @@ function tzDayKey(at: Date, tz: string): string { return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`; } -/** Natural labels for the non-measurement channel keys (read cleanly in prose). */ +/** + * Natural labels for the non-measurement channel keys (read cleanly in prose). + * + * The environmental-exposure channels take their phrase from the one env field + * vocabulary rather than a second hand-written list, so a new exposure field + * cannot reach the Coach as "env pressure delta". + */ const CHANNEL_LABELS: Record = { [MEDICATION_COMPLIANCE_CHANNEL_KEY]: "medication adherence", [SYMPTOM_SEVERITY_CHANNEL_KEY]: "symptom severity", + ...Object.fromEntries( + ENVIRONMENT_FIELDS.map((field) => [field.key, field.narrationLabel]), + ), }; -/** Lower-case, space-separated label from a discovery channel key. */ -function humanise(key: string): string { +/** + * Lower-case, space-separated label for a discovery channel. + * + * `label` is the engine's per-pair label for a DYNAMIC channel — today that is + * a custom metric, whose key is `CUSTOM_METRIC:`. It has to win over the + * key: prettifying that key would put a database id into the Coach's prose. + */ +function humanise(key: string, label?: string): string { + if (label && label.trim().length > 0) return label.trim().toLowerCase(); return CHANNEL_LABELS[key] ?? key.replace(/_/g, " ").toLowerCase(); } @@ -194,80 +211,22 @@ export async function readCoachCorrelations( const tz = userRow?.timezone ?? "Europe/Berlin"; const since = new Date(Date.now() - WINDOW_DAYS * MS_PER_DAY); - // The non-MeasurementType channels (MOOD, MEDICATION_COMPLIANCE, - // SYMPTOM_SEVERITY) are backed by other models — `discoveryMeasurementTypes` - // drops them so the Postgres `type IN (...)` enum cast only ever sees real - // enum values; each is sourced separately below, exactly like the route. - const behaviourTypes = discoveryMeasurementTypes( - DISCOVERY_BEHAVIOURS, - ) as MeasurementType[]; - const outcomeTypes = discoveryMeasurementTypes( - DISCOVERY_OUTCOMES, - ) as MeasurementType[]; - - // v1.30.3 (QA F1) — the fetch + desc/cap/resort discipline (a dense - // account's cap must fall on the OLDEST rows, never the newest — the - // emerging-correlations pass below is entirely about the recent window) - // now lives in `fetchMeasurementWindowSeries` / `fetchMoodWindowSeries` - // (`correlation-channel-series.ts`), shared with the route, the - // per-metric card, and the period narrative. - const [ - { byType, measurementsCapped }, - { moodDaily, moodCapped }, - coincidentDerived, - priorityJson, - ] = await Promise.all([ - fetchMeasurementWindowSeries(userId, since, [ - ...behaviourTypes, - ...outcomeTypes, - ]), - fetchMoodWindowSeries(userId, tz, since), + // The channel set is the shared one, so the Coach scans exactly what the + // correlations page scans. The measurement read stays `"raw"` (the route's + // rollup read-swap is a read-cost choice, not a channel-set one). + // + // v1.22 — the lab draws feed a separate point-vs-window pass, not the + // matrix, so they are fetched alongside rather than assembled in. + const [matrix, coincidentDerived, labDraws] = await Promise.all([ + assembleDiscoveryMatrix(userId, { tz, since, fetchMode: "raw" }), // Coincident-deviation is its own derived metric — fail-soft to null so a // baseline hiccup never sinks the whole correlations read. D2-8: pass the // user's tz so the "today" grouping matches the user's calendar day, not // UTC's, before the fired flag is narrated as "out of band TODAY". computeCoincidentDeviation(userId, profile, { tz }).catch(() => null), - loadUserSourcePriority(userId), - ]); - const seriesPoints = (key: string) => - key === "MOOD" - ? moodDaily - : buildMeasurementDailySeries( - key as MeasurementType, - byType.get(key) ?? [], - tz, - priorityJson, - ); - - // The two non-measurement, non-mood channels come from their own sources - // (the dose-history ledger + the illness day-log), folded in below exactly - // like the route. Each degrades to an empty series when the user has no - // data, so the discovery loop drops the channel (it cannot clear the n ≥ 20 - // floor) — it can never surface a fabricated driver. - const [complianceSeries, symptomSeries, labDraws] = await Promise.all([ - fetchComplianceSeries(userId, tz, since), - fetchSymptomSeries(userId, tz, since), - // v1.22 — lab draws for the labs ↔ outcome pass (degrades to absent). fetchLabDraws(userId, tz, since), ]); - - const series: NamedSeries[] = []; - for (const key of DISCOVERY_BEHAVIOURS) { - if (key === MEDICATION_COMPLIANCE_CHANNEL_KEY) { - series.push(complianceSeries); - } else if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "behaviour" }); - } else { - series.push({ key, role: "behaviour", points: seriesPoints(key) }); - } - } - for (const key of DISCOVERY_OUTCOMES) { - if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "outcome" }); - } else { - series.push({ key, role: "outcome", points: seriesPoints(key) }); - } - } + const { series, diagnostics } = matrix; // QA F1 — surfaces when a dense account's window exceeded the read cap, // mirroring the route's identical annotation. The cap now falls on the @@ -276,15 +235,20 @@ export async function readCoachCorrelations( annotate({ action: { name: "coach.correlations.read" }, meta: { - measurements_capped: measurementsCapped, - mood_entries_capped: moodCapped, + measurements_capped: diagnostics.measurementsCapped, + mood_entries_capped: diagnostics.moodCapped, + // The environmental-exposure and custom-metric families + // reach this path for the first time; these say whether they carried + // anything on a given turn. + environment_days: diagnostics.environmentDays, + custom_metric_channels: diagnostics.customMetricChannels, }, }); const discovery = discoverCorrelations(series, { locale }); const drivers: CoachCorrelationDriver[] = discovery.discovered.map((d) => ({ - behaviour: humanise(d.behaviour), - outcome: humanise(d.outcome), + behaviour: humanise(d.behaviour, d.behaviourLabel), + outcome: humanise(d.outcome, d.outcomeLabel), direction: d.r >= 0 ? "higher" : "lower", lagDays: d.lagDays, n: d.n, @@ -306,8 +270,8 @@ export async function readCoachCorrelations( }); const emerging: CoachEmergingDriver[] = emergingResult.emerging.map( (d) => ({ - behaviour: humanise(d.behaviour), - outcome: humanise(d.outcome), + behaviour: humanise(d.behaviour, d.behaviourLabel), + outcome: humanise(d.outcome, d.outcomeLabel), direction: d.r >= 0 ? "higher" : "lower", lagDays: d.lagDays, n: d.n, @@ -323,6 +287,8 @@ export async function readCoachCorrelations( }); const labDrivers: CoachLabCorrelation[] = labResult.discovered.map((d) => ({ lab: d.lab.startsWith("LAB:") ? d.lab.slice("LAB:".length) : d.lab, + // A lab pair's outcome is always a declared channel key, never a dynamic + // one, so there is no per-pair label to prefer here. outcome: humanise(d.outcome), direction: d.r >= 0 ? "higher" : "lower", n: d.n, diff --git a/src/lib/insights/__tests__/metric-correlation-context.test.ts b/src/lib/insights/__tests__/metric-correlation-context.test.ts index 703af6931..5f4f411a5 100644 --- a/src/lib/insights/__tests__/metric-correlation-context.test.ts +++ b/src/lib/insights/__tests__/metric-correlation-context.test.ts @@ -48,22 +48,34 @@ describe("getRelevantCorrelationsForMetric", () => { }); it("surfaces only FDR-surviving pairs that involve the metric's channel", async () => { - // Build two strongly-correlated daily series over 60 days: - // TIME_IN_DAYLIGHT (behaviour) → next-day RESTING_HEART_RATE (outcome), - // anti-correlated, plus uncorrelated noise on other channels. + // Two strong, INDEPENDENT lag-1 relationships over 60 days: + // TIME_IN_DAYLIGHT → next-day RESTING_HEART_RATE (involves the metric) + // BLOOD_GLUCOSE → next-day WEIGHT (does not) + // + // The second pair is what makes this test say anything. With one + // correlated pair in the fixture, "every returned relation mentions the + // metric" is satisfied by a function that returns the whole matrix — which + // is what this one did for four releases while the assertion stayed green. const rows: Array<{ type: string; value: number; measuredAt: Date }> = []; const base = new Date("2026-01-01T12:00:00Z"); for (let d = 0; d < 60; d++) { const day = new Date(base.getTime() + d * 86_400_000); + const next = new Date(base.getTime() + (d + 1) * 86_400_000); const daylight = 30 + (d % 10) * 6; // varies 30..84 rows.push({ type: "TIME_IN_DAYLIGHT", value: daylight, measuredAt: day }); // next-day RHR moves opposite to today's daylight (lag-1 anti-corr). - const next = new Date(base.getTime() + (d + 1) * 86_400_000); rows.push({ type: "RESTING_HEART_RATE", value: 90 - daylight * 0.4, measuredAt: next, }); + const glucose = 80 + (d % 7) * 5; // varies 80..110 + rows.push({ type: "BLOOD_GLUCOSE", value: glucose, measuredAt: day }); + rows.push({ + type: "WEIGHT", + value: 60 + glucose * 0.1, + measuredAt: next, + }); } vi.mocked(prisma.measurement.findMany).mockResolvedValue(rows as never); @@ -79,6 +91,11 @@ describe("getRelevantCorrelationsForMetric", () => { expect(Number.isFinite(c.r)).toBe(true); expect(c.n).toBeGreaterThanOrEqual(20); } + // And the glucose → weight pair, which the same scan certainly found, is + // not in a resting-heart-rate card's grounded context. + for (const c of out) { + expect(c.interpretation.toLowerCase()).not.toContain("weight"); + } }); it("is best-effort: a DB failure resolves to [] rather than throwing", async () => { diff --git a/src/lib/insights/correlation-channel-series.ts b/src/lib/insights/correlation-channel-series.ts index 57ae69b63..7e1be8512 100644 --- a/src/lib/insights/correlation-channel-series.ts +++ b/src/lib/insights/correlation-channel-series.ts @@ -1,14 +1,19 @@ /** - * v1.21.0 (INTEGFIX) — shared data-fetching helpers for the two non-measurement - * FDR correlation channels (medication compliance + symptom severity). + * v1.21.0 (INTEGFIX) — the DB reads behind the correlation-discovery channels. * * The pure series shapers live in `correlation-series-builders.ts`; the FDR - * engine itself is pure over `NamedSeries[]`. These two helpers own the DB reads - * that feed those shapers — the dose-history ledger (compliance) and the illness - * day-log (symptom severity) — so EVERY consumer of the discovery matrix (the - * `/api/insights/correlations` route AND the Coach `get_correlations` tool) - * builds the channels identically rather than re-implementing the queries, the - * tz keying, and the episode-span clamping per call site. + * engine itself is pure over `NamedSeries[]`. These helpers own the reads that + * feed those shapers — the dose-history ledger (compliance), the illness day-log + * (symptom severity), the daily weather rows, the opt-in custom metrics, the + * measurement window (raw and rollup-tiered), the mood window — so the queries, + * the tz keying and the episode-span clamping exist once. + * + * This module owns FETCHING one channel at a time. It does NOT decide which + * channels a surface gets: that is `discovery-matrix.ts`, the one assembler + * every consumer of the matrix calls. Fetch discipline was hoisted here in + * v1.30.3 and the assembly was left behind at four call sites, which is how the + * Coach tool spent v1.25 onward scanning a matrix with no weather in it. Add a + * channel here, admit it there, and every surface has it. * * Each helper degrades to an EMPTY series when the user has no data, so the * channel drops out of discovery (it cannot clear the n ≥ 20 floor) rather than @@ -22,6 +27,7 @@ import { SCHEDULE_COMPLIANCE_SELECT, } from "@/lib/analytics/compliance"; import { + FACTOR_CHANNEL_PREFIX, MEDICATION_COMPLIANCE_CHANNEL_KEY, SYMPTOM_SEVERITY_CHANNEL_KEY, type DailySeriesPoint, @@ -389,11 +395,10 @@ export interface MoodWindowFetch { /** * v1.30.3 (QA F1/F2/F3) — shared mood-window fetch, same desc+cap+resort - * discipline as {@link fetchMeasurementWindowSeries}. Callers that also need - * per-entry factor tag-links (the period narrative's RATED-factor channels) - * read `MoodEntry` themselves for the extra `tagLinks` select, but MUST - * apply the same desc+resort discipline — this helper only covers the - * plain-score case the route / coach tool / per-metric card share. + * discipline as {@link fetchMeasurementWindowSeries}. This helper covers the + * plain-score case; the RATED-factor variant that also reads each entry's + * tag-links is {@link fetchMoodFactorWindowSeries}, which carries the same + * discipline over a wider select. */ export async function fetchMoodWindowSeries( userId: string, @@ -417,6 +422,131 @@ export async function fetchMoodWindowSeries( return { moodDaily, moodCapped }; } +/** One mood-window fetch that also carries the RATED-factor channels. */ +export interface MoodFactorWindowFetch extends MoodWindowFetch { + /** One inverse-flipped daily-mean series per `FACTOR:` channel. */ + factorSeries: Map; +} + +/** + * A RATED mood factor link, carrying the scale + `inverse` flag so the + * documented sign-flip is applied once, at the boundary. + */ +interface FactorLink { + key: string; + rating: number; + scaleMin: number; + scaleMax: number; + inverse: boolean; + at: Date; +} + +/** + * v1.14.0 — collapse RATED-factor links to one inverse-flipped daily-mean + * series per factor, tz-day-keyed exactly like {@link toDailyMeans} so a factor + * channel joins the discovery matrix on the same day grid as every vital. + * An inverse factor's rating `r` maps to `(scaleMin + scaleMax) - r` BEFORE + * averaging so "up" always reads as a better day — the same flip the mood + * aggregates apply, kept in lock-step. Returns one `FACTOR:` series per + * factor the user actually rated. Pure. + */ +function factorDailyMeans( + links: FactorLink[], + tz: string, +): Map { + const byFactor = new Map< + string, + Map + >(); + for (const l of links) { + if (!Number.isFinite(l.rating)) continue; + const value = l.inverse ? l.scaleMin + l.scaleMax - l.rating : l.rating; + const day = tzDayKey(l.at, tz); + const days = + byFactor.get(l.key) ?? new Map(); + const acc = days.get(day) ?? { sum: 0, count: 0 }; + acc.sum += value; + acc.count += 1; + days.set(day, acc); + byFactor.set(l.key, days); + } + const out = new Map(); + for (const [key, days] of byFactor) { + out.set( + `${FACTOR_CHANNEL_PREFIX}${key}`, + [...days.entries()] + .map(([day, acc]) => ({ day, value: acc.sum / acc.count })) + .sort((a, b) => (a.day < b.day ? -1 : 1)), + ); + } + return out; +} + +/** + * Mood-window fetch WITH the per-entry RATED-factor tag-links, for the one + * surface that admits `FACTOR:*` channels into the discovery matrix (see the + * `includeMoodFactors` option on `assembleDiscoveryMatrix` for why that is one + * surface and not four). Same desc+cap+resort discipline as + * {@link fetchMoodWindowSeries} over a wider select — the tag-links ride the + * existing mood read, so this costs no extra round-trip over the plain variant. + * + * BINARY links carry a null `rating` and are excluded at the query. + */ +export async function fetchMoodFactorWindowSeries( + userId: string, + tz: string, + since: Date, +): Promise { + const rowsDesc = await prisma.moodEntry.findMany({ + where: { userId, deletedAt: null, moodLoggedAt: { gte: since } }, + orderBy: { moodLoggedAt: "desc" }, + take: MOOD_READ_CAP, + select: { + score: true, + moodLoggedAt: true, + tagLinks: { + where: { moodTag: { kind: "RATED" }, rating: { not: null } }, + select: { + rating: true, + moodTag: { + select: { + key: true, + scaleMin: true, + scaleMax: true, + inverse: true, + }, + }, + }, + }, + }, + }); + const moodCapped = rowsDesc.length >= MOOD_READ_CAP; + const rows = [...rowsDesc].sort( + (a, b) => a.moodLoggedAt.getTime() - b.moodLoggedAt.getTime(), + ); + const moodDaily = toDailyMeans( + rows.map((e) => ({ value: e.score, at: e.moodLoggedAt })), + tz, + ); + + const links: FactorLink[] = []; + for (const entry of rows) { + for (const link of entry.tagLinks) { + if (link.rating == null) continue; + links.push({ + key: link.moodTag.key, + rating: link.rating, + scaleMin: link.moodTag.scaleMin, + scaleMax: link.moodTag.scaleMax, + inverse: link.moodTag.inverse, + at: entry.moodLoggedAt, + }); + } + } + + return { moodDaily, moodCapped, factorSeries: factorDailyMeans(links, tz) }; +} + /** * v1.29.6 — collapse rows to per-day MEANS keyed in the user's tz. This is * the correct grain for spot metrics (BP, glucose, HRV, resting HR, weight, diff --git a/src/lib/insights/derived/__tests__/coach-read-locale.test.ts b/src/lib/insights/derived/__tests__/coach-read-locale.test.ts index 504d628af..c861ffce9 100644 --- a/src/lib/insights/derived/__tests__/coach-read-locale.test.ts +++ b/src/lib/insights/derived/__tests__/coach-read-locale.test.ts @@ -24,6 +24,8 @@ const intakeEventFindMany = vi.fn(); const illnessEpisodeFindMany = vi.fn(); const illnessDayLogFindMany = vi.fn(); const labResultFindMany = vi.fn(); +const environmentContextFindMany = vi.fn(); +const customMetricFindMany = vi.fn(); vi.mock("@/lib/modules/gate", () => ({ isModuleEnabled: vi.fn(async () => true), @@ -42,6 +44,12 @@ vi.mock("@/lib/db", () => ({ illnessEpisode: { findMany: (a: unknown) => illnessEpisodeFindMany(a) }, illnessDayLog: { findMany: (a: unknown) => illnessDayLogFindMany(a) }, labResult: { findMany: (a: unknown) => labResultFindMany(a) }, + // The reader assembles the shared discovery matrix, which reads the + // environmental-exposure and custom-metric channels too. + environmentContext: { + findMany: (a: unknown) => environmentContextFindMany(a), + }, + customMetric: { findMany: (a: unknown) => customMetricFindMany(a) }, }, })); @@ -129,6 +137,8 @@ describe("buildCoachReadStrip — the pattern line speaks the reader's language" illnessEpisodeFindMany.mockReset().mockResolvedValue([]); illnessDayLogFindMany.mockReset().mockResolvedValue([]); labResultFindMany.mockReset().mockResolvedValue([]); + environmentContextFindMany.mockReset().mockResolvedValue([]); + customMetricFindMany.mockReset().mockResolvedValue([]); seedSymptomDrivesSleep(); }); diff --git a/src/lib/insights/discovery-matrix.ts b/src/lib/insights/discovery-matrix.ts new file mode 100644 index 000000000..d5dc07993 --- /dev/null +++ b/src/lib/insights/discovery-matrix.ts @@ -0,0 +1,326 @@ +/** + * The one assembler for the correlation-discovery matrix. + * + * `discoverCorrelations` is pure over `NamedSeries[]`. Deciding WHICH series go + * in is the part that used to be copied: the `/api/insights/correlations` route, + * the per-metric assessment card, the Coach `get_correlations` tool and the + * period narrative each folded their own channel list, and three of the four + * carried a different one. The Coach's was missing the environmental and + * custom-metric families entirely, so a question about air pressure and sleep + * met a matrix that had never contained the pair — while the tool's own file + * header promised "the SAME full-matrix scan the route runs". v1.30.3 hoisted + * the FETCH discipline into `correlation-channel-series.ts` for exactly this + * reason and left the assembly duplicated behind; this module finishes that. + * + * Every surface that scans the matrix calls {@link assembleDiscoveryMatrix} and + * gets the same channel set. The differences that remain are declared as + * options, each with its reason written at the option — a surface cannot drop a + * channel family by omission any more, only by naming what it is doing. + * + * What this does NOT own: + * + * - the scan. Each caller still runs `discoverCorrelations` (and, where it + * applies, `discoverEmergingCorrelations`) over the returned series, because + * what each does with the result differs — the route persists patterns, the + * card filters to one metric, the Coach shapes driver rows, the narrative + * cites them. Same input, same statistics, different presentation. + * - the labs ↔ outcome pass. `discoverLabOutcomeCorrelations` runs over + * `LabDrawPoint[]` (point-vs-window over sparse draws), not the + * behaviour × outcome day grid this assembles. Its input comes from + * `fetchLabDraws` at the two call sites that want it. + * - the cycle-phase matrix. `discoverPhaseCorrelations` builds a deliberately + * separate, gated matrix (CYCLE_PHASE × outcomes) that must never reach the + * non-gated surfaces; see `src/lib/cycle/phase-crosstab.ts`. + * + * The freeze on all of this is `src/__tests__/discovery-matrix-guard.test.ts`. + */ +import type { MeasurementType } from "@/generated/prisma/client"; +import { + DISCOVERY_BEHAVIOURS, + DISCOVERY_OUTCOMES, + discoveryMeasurementTypes, + MEDICATION_COMPLIANCE_CHANNEL_KEY, + SYMPTOM_SEVERITY_CHANNEL_KEY, + type DailySeriesPoint, + type NamedSeries, +} from "@/lib/insights/correlation-discovery"; +import { + buildMeasurementDailySeries, + fetchComplianceSeries, + fetchCustomMetricBehaviourSeries, + fetchEnvironmentSeries, + fetchMeasurementDailySeriesTiered, + fetchMeasurementWindowSeries, + fetchMoodFactorWindowSeries, + fetchMoodWindowSeries, + fetchSymptomSeries, + type MoodFactorWindowFetch, +} from "@/lib/insights/correlation-channel-series"; +import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; + +/** + * How the measurement channels are read. + * + * - `"tiered"` takes the DAY rollup read-swap where a channel is eligible and + * the user's profile timezone sits in the near-UTC band, falling back to the + * raw path per channel on any miss. Cheaper on a dense account; carries the + * documented ±3 h day-key tolerance and drops the partial oldest day. + * - `"raw"` always reads raw rows and reduces them in process. + * + * This is a read-cost decision, NOT a channel-set decision, so it stays a + * per-caller choice rather than something the assembler picks. The route takes + * the swap because it scans 180 days on every cache miss; the three library + * surfaces stayed raw when the swap landed and are left there, because moving + * them is a value change at the tolerance boundary and belongs in its own + * change with its own parity proof, not folded into a channel-set fix. + */ +export type DiscoveryMatrixFetchMode = "tiered" | "raw"; + +export interface DiscoveryMatrixOptions { + /** The user's display timezone — every day key is minted in it. */ + tz: string; + /** Inclusive start of the read window. */ + since: Date; + /** See {@link DiscoveryMatrixFetchMode}. */ + fetchMode: DiscoveryMatrixFetchMode; + /** + * Admit the user's RATED mood factors as `FACTOR:` channels, in BOTH + * roles. Off everywhere except the period narrative, deliberately: + * + * - `MoodTag.kind = "RATED"` is FROZEN as of v1.37. The capture sheet stopped + * offering factor ratings when the level-A columns landed on `MoodEntry`; + * nothing was deleted, so existing scores keep rendering and keep feeding + * the factor crosstab, but no new ones are written. Widening a frozen + * capture path's reach to three more surfaces is a decision about what to + * ask, not a parity fix — the schema comment on that column says so in as + * many words. + * - The route and the per-metric card PERSIST every surviving pair into + * `CorrelationPattern.factorKey` / `.outcomeKey`, plaintext columns. A + * custom RATED tag's key is `custom:` with its label held encrypted, + * so a factor channel on those two surfaces would write a channel key that + * only resolves through a decrypt — and would narrate as the raw key + * meanwhile. + * + * So this is a real distinction, not an accident of where someone stopped: + * the narrative reads a bounded period and cites factors in prose it does not + * store. Turning it on costs no extra round-trip — the tag-links ride the + * existing mood read (see `fetchMoodFactorWindowSeries`) — but it does widen + * the BH-FDR family, since each factor enters as both lag source and lag + * target. + */ + includeMoodFactors?: boolean; + /** + * Measurement types to read ALONGSIDE the discovery channels, surfaced in + * {@link DiscoveryMatrix.byMetric} but never folded into `series`. Lets a + * caller that needs more per-metric series than the matrix contains (the + * period narrative's delta metrics and banded vitals) get them from the one + * measurement read instead of a second one. + */ + extraMeasurementTypes?: readonly MeasurementType[]; +} + +/** Per-channel reach, for the callers' wide-event annotations. */ +export interface DiscoveryMatrixDiagnostics { + /** The raw measurement read hit its cap — the window may miss OLDER rows. */ + measurementsCapped: boolean; + /** The mood read hit its cap. */ + moodCapped: boolean; + /** Measurement channels served from the DAY rollup tier (tiered mode only). */ + rollupTypes: MeasurementType[]; + complianceDays: number; + symptomDays: number; + /** Stored daily points summed across the environmental-exposure channels. */ + environmentDays: number; + customMetricChannels: number; + customMetricDays: number; + /** `FACTOR:*` channels admitted (0 unless `includeMoodFactors`). */ + moodFactorChannels: number; +} + +export interface DiscoveryMatrix { + /** The behaviour × outcome channel set, ready for `discoverCorrelations`. */ + series: NamedSeries[]; + /** + * Day-keyed series per metric key — every measurement type read (discovery + * channels plus `extraMeasurementTypes`), `MOOD`, and each admitted + * `FACTOR:*` channel. A key with no data is absent; callers read it as + * `byMetric.get(key) ?? []`. + */ + byMetric: Map; + diagnostics: DiscoveryMatrixDiagnostics; +} + +/** + * The mood read, normalised to the factor-carrying shape. With factors off the + * factor map is empty, so the fold below has one code path rather than two. + */ +async function readMood( + userId: string, + tz: string, + since: Date, + withFactors: boolean, +): Promise { + if (withFactors) return fetchMoodFactorWindowSeries(userId, tz, since); + const plain = await fetchMoodWindowSeries(userId, tz, since); + return { ...plain, factorSeries: new Map() }; +} + +/** One measurement read, reduced to per-day series under the chosen mode. */ +async function readMeasurementDaily( + userId: string, + tz: string, + since: Date, + types: MeasurementType[], + mode: DiscoveryMatrixFetchMode, +): Promise<{ + byType: Map; + measurementsCapped: boolean; + rollupTypes: MeasurementType[]; +}> { + if (mode === "tiered") { + const tiered = await fetchMeasurementDailySeriesTiered( + userId, + tz, + since, + types, + ); + return { + byType: tiered.byType, + measurementsCapped: tiered.measurementsCapped, + rollupTypes: tiered.rollupTypes, + }; + } + + const [raw, priorityJson] = await Promise.all([ + fetchMeasurementWindowSeries(userId, since, types), + loadUserSourcePriority(userId), + ]); + const byType = new Map(); + for (const type of types) { + const rows = raw.byType.get(type); + if (!rows || rows.length === 0) continue; + byType.set(type, buildMeasurementDailySeries(type, rows, tz, priorityJson)); + } + return { + byType, + measurementsCapped: raw.measurementsCapped, + rollupTypes: [], + }; +} + +/** + * Build the full discovery matrix for a user over a window. + * + * Reads every channel family in parallel and folds them in one fixed order: + * the declared behaviours, the declared outcomes, the environmental-exposure + * channels, the opt-in custom-metric channels, then (only when asked) the + * RATED mood factors in both roles. A family the user has no data for yields + * an empty series and drops out of the scan at the n ≥ 20 floor — absence is + * never a fabricated constant. + * + * The order is fixed on purpose. `discoverCorrelations` ranks by shrunk effect + * then q, and `Array.prototype.sort` is stable, so two pairs equal on both + * come out in the order they were tested — which is the order the channels + * were folded. A stable fold keeps two runs over the same data ranked the same + * way, and keeps this route's ranking identical to what it was before the + * assembly moved here. + */ +export async function assembleDiscoveryMatrix( + userId: string, + opts: DiscoveryMatrixOptions, +): Promise { + const { tz, since, fetchMode } = opts; + + // The non-MeasurementType channels (MOOD, MEDICATION_COMPLIANCE, + // SYMPTOM_SEVERITY) are backed by other models, so `discoveryMeasurementTypes` + // drops them before the list reaches a Postgres `type IN (...)` enum cast — + // a non-enum string in that list errors the cast. Each is sourced below. + const measurementTypes = Array.from( + new Set([ + ...discoveryMeasurementTypes(DISCOVERY_BEHAVIOURS), + ...discoveryMeasurementTypes(DISCOVERY_OUTCOMES), + ...(opts.extraMeasurementTypes ?? []), + ]), + ) as MeasurementType[]; + + const [ + measurements, + mood, + complianceSeries, + symptomSeries, + environmentSeries, + customMetricSeries, + ] = await Promise.all([ + readMeasurementDaily(userId, tz, since, measurementTypes, fetchMode), + readMood(userId, tz, since, opts.includeMoodFactors === true), + fetchComplianceSeries(userId, tz, since), + fetchSymptomSeries(userId, tz, since), + fetchEnvironmentSeries(userId, since), + fetchCustomMetricBehaviourSeries(userId, tz, since), + ]); + + const { factorSeries } = mood; + + const points = (key: string): DailySeriesPoint[] => + key === "MOOD" ? mood.moodDaily : (measurements.byType.get(key) ?? []); + + const series: NamedSeries[] = []; + for (const key of DISCOVERY_BEHAVIOURS) { + if (key === MEDICATION_COMPLIANCE_CHANNEL_KEY) { + series.push(complianceSeries); + } else if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { + // The symptom builder returns the outcome role; the points are + // role-invariant, so the behaviour arm re-tags the same series. + series.push({ ...symptomSeries, role: "behaviour" }); + } else { + series.push({ key, role: "behaviour", points: points(key) }); + } + } + for (const key of DISCOVERY_OUTCOMES) { + if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { + series.push({ ...symptomSeries, role: "outcome" }); + } else { + series.push({ key, role: "outcome", points: points(key) }); + } + } + // Environmental exposure and custom metrics are lag SOURCES only: they pair + // (D → D+1) against every outcome above, under the unchanged n ≥ 20 / FDR / + // effect-size gates, so a thin weather or custom series degrades to absent. + series.push(...environmentSeries, ...customMetricSeries); + // A factor is plausibly both a lag source ("rated work today → next-day + // sleep") and a lag target ("steps today → next-day rated energy"), so it + // enters as both roles. The engine skips self-pairs — every factor shares + // MOOD's family — and BH-FDR controls the wider family this opens. + for (const [key, factorPoints] of factorSeries) { + series.push({ key, role: "behaviour", points: factorPoints }); + series.push({ key, role: "outcome", points: factorPoints }); + } + + const byMetric = new Map(measurements.byType); + if (mood.moodDaily.length > 0) byMetric.set("MOOD", mood.moodDaily); + for (const [key, factorPoints] of factorSeries) { + byMetric.set(key, factorPoints); + } + + return { + series, + byMetric, + diagnostics: { + measurementsCapped: measurements.measurementsCapped, + moodCapped: mood.moodCapped, + rollupTypes: measurements.rollupTypes, + complianceDays: complianceSeries.points.length, + symptomDays: symptomSeries.points.length, + environmentDays: environmentSeries.reduce( + (sum, s) => sum + s.points.length, + 0, + ), + customMetricChannels: customMetricSeries.length, + customMetricDays: customMetricSeries.reduce( + (sum, s) => sum + s.points.length, + 0, + ), + moodFactorChannels: factorSeries.size, + }, + }; +} diff --git a/src/lib/insights/metric-correlation-context.ts b/src/lib/insights/metric-correlation-context.ts index 10cea2f33..f626b5ef3 100644 --- a/src/lib/insights/metric-correlation-context.ts +++ b/src/lib/insights/metric-correlation-context.ts @@ -13,35 +13,27 @@ * This module runs the SAME full-matrix discovery the * `/api/insights/correlations` route runs (so a card never surfaces a pair * the correlations page wouldn't), then filters to the surviving pairs that - * INVOLVE the current metric's discovery channel. The result is the - * engine's own conservative, descriptive, never-causal `interpretation` - * strings — passed verbatim into the prompt as grounded context. + * INVOLVE the current metric's discovery channel. That parity is structural + * rather than asserted: both take their channel set from the one assembler in + * `src/lib/insights/discovery-matrix.ts`, and + * `src/__tests__/discovery-matrix-guard.test.ts` freezes the assembler as the + * only place a matrix is built. The result is the engine's own conservative, + * descriptive, never-causal `interpretation` strings — passed verbatim into + * the prompt as grounded context. * * Read-only consumption. No new statistics are computed here; the only - * change vs the route is the filter to one metric and a shorter window cap. + * changes vs the route are the filter to one metric and the raw (rather than + * rollup-tiered) measurement read. */ import type { MeasurementType } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import type { Locale } from "@/lib/i18n/config"; import { discoverCorrelations, - discoveryMeasurementTypes, DISCOVERY_BEHAVIOURS, DISCOVERY_OUTCOMES, - type NamedSeries, - MEDICATION_COMPLIANCE_CHANNEL_KEY, - SYMPTOM_SEVERITY_CHANNEL_KEY, } from "@/lib/insights/correlation-discovery"; -import { - buildMeasurementDailySeries, - fetchComplianceSeries, - fetchCustomMetricBehaviourSeries, - fetchEnvironmentSeries, - fetchMeasurementWindowSeries, - fetchMoodWindowSeries, - fetchSymptomSeries, -} from "@/lib/insights/correlation-channel-series"; -import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { assembleDiscoveryMatrix } from "@/lib/insights/discovery-matrix"; import { decisionForEvidence, PATTERN_FAMILIES, @@ -102,73 +94,15 @@ export async function getRelevantCorrelationsForMetric( const tz = profile?.timezone ?? "Europe/Berlin"; const since = new Date(Date.now() - WINDOW_DAYS * MS_PER_DAY); - // Non-measurement channels (MOOD, and v1.21.0 MEDICATION_COMPLIANCE / - // SYMPTOM_SEVERITY) are backed by other models — `discoveryMeasurementTypes` - // strips them from the measurement `type IN (...)` query (a non-enum string - // would error the cast). This per-metric-card surface does not populate the - // compliance / symptom channels (they degrade to absent here); the - // canonical `/api/insights/correlations` route builds them. - const behaviourTypes = discoveryMeasurementTypes( - DISCOVERY_BEHAVIOURS, - ) as MeasurementType[]; - const outcomeTypes = discoveryMeasurementTypes( - DISCOVERY_OUTCOMES, - ) as MeasurementType[]; - - // v1.30.3 (QA F1) — the fetch + desc/cap/resort discipline (a dense - // account's cap must fall on the OLDEST rows, never the newest) now - // lives in `fetchMeasurementWindowSeries` / `fetchMoodWindowSeries` - // (`correlation-channel-series.ts`), shared with the route, the Coach - // tool, and the period narrative. - const [ - { byType }, - { moodDaily }, - priorityJson, - complianceSeries, - symptomSeries, - environmentSeries, - customMetricSeries, - ] = await Promise.all([ - fetchMeasurementWindowSeries(userId, since, [ - ...behaviourTypes, - ...outcomeTypes, - ]), - fetchMoodWindowSeries(userId, tz, since), - loadUserSourcePriority(userId), - fetchComplianceSeries(userId, tz, since), - fetchSymptomSeries(userId, tz, since), - fetchEnvironmentSeries(userId, since), - fetchCustomMetricBehaviourSeries(userId, tz, since), - ]); - - const points = (key: string) => - key === "MOOD" - ? moodDaily - : buildMeasurementDailySeries( - key as MeasurementType, - byType.get(key) ?? [], - tz, - priorityJson, - ); - - const series: NamedSeries[] = []; - for (const key of DISCOVERY_BEHAVIOURS) { - if (key === MEDICATION_COMPLIANCE_CHANNEL_KEY) { - series.push(complianceSeries); - } else if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "behaviour" }); - } else { - series.push({ key, role: "behaviour", points: points(key) }); - } - } - for (const key of DISCOVERY_OUTCOMES) { - if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "outcome" }); - } else { - series.push({ key, role: "outcome", points: points(key) }); - } - } - series.push(...environmentSeries, ...customMetricSeries); + // The channel set is the shared one — every family the correlations page + // scans, so a card can never be missing a pair the page would show. The + // measurement read stays `"raw"` (the route's rollup read-swap is a + // read-cost choice, not a channel-set one; see the option's doc comment). + const { series } = await assembleDiscoveryMatrix(userId, { + tz, + since, + fetchMode: "raw", + }); const result = discoverCorrelations(series, { locale }); const decisions = await syncAcceptedPatterns({ @@ -184,16 +118,26 @@ export async function getRelevantCorrelationsForMetric( qValue: pattern.qValue, })), }); - return result.discovered - .filter((d) => { - const decision = decisionForEvidence(decisions, { - factorKey: d.behaviour, - outcomeKey: d.outcome, - lagDays: d.lagDays, - }); - return decision?.dismissed !== true; - }) - .map((d) => ({ interpretation: d.interpretation, n: d.n, r: d.r })); + return ( + result.discovered + // The pair has to INVOLVE this metric. This filter is the difference + // between the function's name and a full dump of the matrix: without + // it, a WEIGHT card's prompt is handed "your daylight goes with your + // sleep" as grounded context for a paragraph about weight. It was here + // from the start and was lost when the pattern-dismissal filter was + // written in its place rather than after it; the docstring, the name + // and the `channelKeyForType` guard above all kept saying otherwise. + .filter((d) => d.behaviour === channel || d.outcome === channel) + .filter((d) => { + const decision = decisionForEvidence(decisions, { + factorKey: d.behaviour, + outcomeKey: d.outcome, + lagDays: d.lagDays, + }); + return decision?.dismissed !== true; + }) + .map((d) => ({ interpretation: d.interpretation, n: d.n, r: d.r })) + ); } catch { return []; } diff --git a/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts b/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts index 2415fde27..2b24fcada 100644 --- a/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts +++ b/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts @@ -18,11 +18,31 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; const userFindUnique = vi.fn(); const measurementFindMany = vi.fn(); const moodFindMany = vi.fn(); +// The wrapper takes its channel set from the shared discovery +// assembler, so the four channel families that used to be missing here (the +// dose-history ledger, the illness day-log, the daily weather rows, the opt-in +// custom metrics) are read on this path too. Default every one empty: this +// file is about the measurement read's ORDER and GRAIN, and an empty channel +// degrades to absent without touching either. +const medicationFindMany = vi.fn(); +const intakeEventFindMany = vi.fn(); +const illnessEpisodeFindMany = vi.fn(); +const illnessDayLogFindMany = vi.fn(); +const environmentContextFindMany = vi.fn(); +const customMetricFindMany = vi.fn(); vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: (a: unknown) => userFindUnique(a) }, measurement: { findMany: (a: unknown) => measurementFindMany(a) }, moodEntry: { findMany: (a: unknown) => moodFindMany(a) }, + medication: { findMany: (a: unknown) => medicationFindMany(a) }, + medicationIntakeEvent: { findMany: (a: unknown) => intakeEventFindMany(a) }, + illnessEpisode: { findMany: (a: unknown) => illnessEpisodeFindMany(a) }, + illnessDayLog: { findMany: (a: unknown) => illnessDayLogFindMany(a) }, + environmentContext: { + findMany: (a: unknown) => environmentContextFindMany(a), + }, + customMetric: { findMany: (a: unknown) => customMetricFindMany(a) }, }, })); @@ -37,6 +57,12 @@ beforeEach(() => { }); measurementFindMany.mockReset(); moodFindMany.mockReset().mockResolvedValue([]); + medicationFindMany.mockReset().mockResolvedValue([]); + intakeEventFindMany.mockReset().mockResolvedValue([]); + illnessEpisodeFindMany.mockReset().mockResolvedValue([]); + illnessDayLogFindMany.mockReset().mockResolvedValue([]); + environmentContextFindMany.mockReset().mockResolvedValue([]); + customMetricFindMany.mockReset().mockResolvedValue([]); }); describe("buildPeriodNarrativeContext — grain (QA F3)", () => { @@ -135,5 +161,53 @@ describe("buildPeriodNarrativeContext — grain (QA F3)", () => { take: 20000, }), ); + // The mood read carries the same discipline, and it is the one that also + // pulls the RATED-factor tag-links this surface admits. + expect(moodFindMany).toHaveBeenCalledWith( + expect.objectContaining({ + orderBy: { moodLoggedAt: "desc" }, + take: 5000, + }), + ); + }); + + // The narrative used to fold in only the measurement channels, + // MOOD and the RATED mood factors: the compliance, symptom, environmental + // and custom-metric families had arrived at the correlations route and never + // reached this surface. Nothing explained the gap — the comment that looked + // like it did was the same sentence a sibling file carried while doing the + // opposite. It scans the shared matrix now. + it("reads every discovery channel family, not just the measurement ones", async () => { + const now = new Date("2026-06-30T12:00:00.000Z"); + measurementFindMany.mockResolvedValue([]); + + await buildPeriodNarrativeContext("u1", { + period: "month", + now, + locale: "en", + }); + + expect(medicationFindMany).toHaveBeenCalledTimes(1); + expect(illnessEpisodeFindMany).toHaveBeenCalledTimes(1); + expect(environmentContextFindMany).toHaveBeenCalledTimes(1); + expect(customMetricFindMany).toHaveBeenCalledTimes(1); + }); + + // The one declared divergence: this surface — and only this surface — admits + // the user's RATED mood factors. See `includeMoodFactors` for why. + it("pulls the RATED-factor tag-links the other three surfaces do not", async () => { + const now = new Date("2026-06-30T12:00:00.000Z"); + measurementFindMany.mockResolvedValue([]); + + await buildPeriodNarrativeContext("u1", { + period: "month", + now, + locale: "en", + }); + + const select = moodFindMany.mock.calls[0]?.[0]?.select as { + tagLinks?: { where?: { moodTag?: { kind?: string } } }; + }; + expect(select.tagLinks?.where?.moodTag?.kind).toBe("RATED"); }); }); diff --git a/src/lib/insights/narrative/period-narrative.ts b/src/lib/insights/narrative/period-narrative.ts index 2a160b63e..34a600fd1 100644 --- a/src/lib/insights/narrative/period-narrative.ts +++ b/src/lib/insights/narrative/period-narrative.ts @@ -12,7 +12,19 @@ * The honesty contract carries through verbatim from the layers it reuses: * - **Drivers** are ONLY the BH-FDR-surviving pairs from `discoverCorrelations` * (`benjaminiHochberg` already enforces descriptive-never-causal); each - * keeps its conservative `interpretation` string unchanged. + * keeps its conservative `interpretation` string unchanged. The channels + * scanned come from the one shared assembler in + * `src/lib/insights/discovery-matrix.ts`, so this surface sees the same + * families the correlations page and the Coach do. It differs in exactly + * one declared way — it admits the user's RATED mood factors as + * `FACTOR:` channels, which the three persisting surfaces do not; the + * reason is written at the `includeMoodFactors` option. + * + * It also silently lacked, until the assembler landed, the + * medication-compliance, symptom-severity, environmental-exposure and + * custom-metric families, which had arrived at the route and never + * here. Nothing explained that; the comment that appeared to was the + * same sentence a sibling file carried while doing the opposite. * - **Band transitions** are a personal-baseline (median ± k·MAD, Hampel/Leys) * comparison of the current period against the band established over the * PRIOR period — never an invented threshold. @@ -36,25 +48,14 @@ import { annotate } from "@/lib/logging/context"; import { wallClockInTz } from "@/lib/tz/wall-clock"; import { discoverCorrelations, - discoveryMeasurementTypes, - DISCOVERY_BEHAVIOURS, - DISCOVERY_OUTCOMES, - FACTOR_CHANNEL_PREFIX, type DailySeriesPoint, type NamedSeries, } from "@/lib/insights/correlation-discovery"; import { buildBaselineBand, median } from "@/lib/insights/derived/baseline"; import { VITALS_BASELINE_TYPES } from "@/lib/insights/derived/registry"; -import { - buildMeasurementDailySeries, - fetchMeasurementWindowSeries, - toDailyMeans, -} from "@/lib/insights/correlation-channel-series"; -import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { assembleDiscoveryMatrix } from "@/lib/insights/discovery-matrix"; const MS_PER_DAY = 24 * 60 * 60 * 1000; -/** Cap on the mood-entry window read, mirroring the route / Coach tool. */ -const MOOD_READ_CAP = 5000; /** The two supported narrative periods and their length in days. */ export const PERIOD_DAYS = { week: 7, month: 30 } as const; @@ -191,61 +192,6 @@ function tzDayKey(at: Date, tz: string): string { return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`; } -/** - * A RATED mood factor link the period read pulls alongside the score. - * Carries the scale + `inverse` so the series build can apply the - * documented sign-flip once, at the boundary. - */ -interface FactorLink { - key: string; - rating: number; - scaleMin: number; - scaleMax: number; - inverse: boolean; - at: Date; -} - -/** - * v1.14.0 — collapse RATED-factor links to one inverse-flipped daily-mean - * series per factor, tz-day-keyed exactly like `toDailyMeans` so a factor - * channel joins the discovery matrix on the same day grid as every vital. - * An inverse factor's rating `r` maps to `(scaleMin + scaleMax) - r` BEFORE - * averaging so "up" always reads as a better day — the same flip the mood - * aggregates apply, kept in lock-step. Returns one `FACTOR:` series per - * factor the user actually rated. Pure. - */ -function factorDailyMeans( - links: FactorLink[], - tz: string, -): Map { - const byFactor = new Map< - string, - Map - >(); - for (const l of links) { - if (!Number.isFinite(l.rating)) continue; - const value = l.inverse ? l.scaleMin + l.scaleMax - l.rating : l.rating; - const day = tzDayKey(l.at, tz); - const days = - byFactor.get(l.key) ?? new Map(); - const acc = days.get(day) ?? { sum: 0, count: 0 }; - acc.sum += value; - acc.count += 1; - days.set(day, acc); - byFactor.set(l.key, days); - } - const out = new Map(); - for (const [key, days] of byFactor) { - out.set( - `${FACTOR_CHANNEL_PREFIX}${key}`, - [...days.entries()] - .map(([day, acc]) => ({ day, value: acc.sum / acc.count })) - .sort((a, b) => (a.day < b.day ? -1 : 1)), - ); - } - return out; -} - /** Round to 2 decimals. */ function round2(n: number): number { return Math.round(n * 100) / 100; @@ -508,8 +454,15 @@ export interface BuildPeriodNarrativeContextOpts { * Fetch + day-key + assemble. Reads a single bounded window covering the * current AND prior period (2× the period length plus one extra day so the * day-1 lag join in discovery has its source), day-keys in the user's tz, - * and delegates to the pure core. No LLM, no migration, no new heavy query — - * one measurement read + one mood read, both bounded. + * and delegates to the pure core. No LLM, no migration — every read is the + * shared assembler's, bounded, and the delta / band beats ride the same + * measurement query as the discovery channels. + * + * A note on what the `week` period can produce: its window is 15 days, and + * `discoverCorrelations` needs 20 lag-joined pairs before it will test one, so + * a weekly narrative's `drivers` list is empty by arithmetic, not by data. + * That predates this file's current shape and is left as it is; the deltas, + * band transitions and coincident flags are what a weekly narrative rests on. */ export async function buildPeriodNarrativeContext( userId: string, @@ -537,77 +490,40 @@ export async function buildPeriodNarrativeContext( tz, ); - // The full set of types any beat needs: delta metrics ∪ banded vitals ∪ - // discovery channels (minus the NON-measurement channels: MOOD is mood-entry - // backed; v1.21.0 MEDICATION_COMPLIANCE / SYMPTOM_SEVERITY are - // ledger / illness-day-log backed — none are MeasurementType enum values, so - // `discoveryMeasurementTypes` drops them from the `type IN (...)` query). - // Those two channels are not populated on the period-narrative surface (they - // degrade to absent here); the canonical `/api/insights/correlations` route - // builds them. - const measurementTypes = Array.from( - new Set([ - ...DELTA_METRICS.map((m) => m.type), - ...VITALS_BASELINE_TYPES, - ...discoveryMeasurementTypes(DISCOVERY_BEHAVIOURS), - ...discoveryMeasurementTypes(DISCOVERY_OUTCOMES), - ]), - ) as MeasurementType[]; - + // One read for both jobs. The discovery channels come from the shared + // assembler; `extraMeasurementTypes` widens the SAME measurement query with + // the types only this surface's other beats need (the period deltas and the + // banded vitals), so the delta beat costs no second round-trip. + // // v1.30.3 (QA F2/F3) — the fetch + desc/cap/resort discipline AND the // per-type grain resolution (source-collapse sum for cumulative types, - // per-night reconstruction for sleep — not a blind per-row MEAN) now - // route through the shared `fetchMeasurementWindowSeries` + - // `buildMeasurementDailySeries` (`correlation-channel-series.ts`), the - // same helpers the route / Coach tool / per-metric card share. Before - // this fix the local `toDailyMeans` twin folded EVERY type through a - // blind per-row mean: a month's current-vs-prior period comparison for - // `SLEEP_DURATION` averaged per-stage segment durations (~45 min) - // instead of summing a night's total time asleep, and `ACTIVITY_STEPS` - // blended per-sample chunk means with drained daily totals — and the - // `asc, take 20000` cap dropped the CURRENT period first on a dense - // account, the worst possible direction for a current-vs-prior surface. - const [{ byType, measurementsCapped }, moodEntriesDesc, priorityJson] = - await Promise.all([ - fetchMeasurementWindowSeries(userId, since, measurementTypes), - prisma.moodEntry.findMany({ - where: { userId, deletedAt: null, moodLoggedAt: { gte: since } }, - orderBy: { moodLoggedAt: "desc" }, - take: MOOD_READ_CAP, - select: { - score: true, - moodLoggedAt: true, - // v1.14.0 — pull RATED-factor links so each factor the user scores - // (work / sleep-quality / stress …) joins the discovery matrix as a - // `FACTOR:` channel. One extra select on the existing mood read - // — no new round-trip. BINARY links carry a null `rating` and are - // dropped below. - tagLinks: { - where: { moodTag: { kind: "RATED" }, rating: { not: null } }, - select: { - rating: true, - moodTag: { - select: { - key: true, - scaleMin: true, - scaleMax: true, - inverse: true, - }, - }, - }, - }, - }, - }), - loadUserSourcePriority(userId), - ]); - // This surface reads mood alongside its RATED-factor tag-links (a select - // the shared `fetchMoodWindowSeries` helper does not carry), so the - // desc+cap+resort discipline is applied in place here rather than - // through that helper — same discipline, bespoke select. - const moodCapped = moodEntriesDesc.length >= MOOD_READ_CAP; - const moodEntries = [...moodEntriesDesc].sort( - (a, b) => a.moodLoggedAt.getTime() - b.moodLoggedAt.getTime(), - ); + // per-night reconstruction for sleep — not a blind per-row MEAN) live in + // `correlation-channel-series.ts`, which the assembler reads through. Before + // that fix the local `toDailyMeans` twin folded EVERY type through a blind + // per-row mean: a month's current-vs-prior comparison for `SLEEP_DURATION` + // averaged per-stage segment durations (~45 min) instead of summing a + // night's total time asleep, and `ACTIVITY_STEPS` blended per-sample chunk + // means with drained daily totals — and the `asc, take 20000` cap dropped + // the CURRENT period first on a dense account, the worst possible direction + // for a current-vs-prior surface. + const { + series: discoverySeries, + byMetric: seriesByMetric, + diagnostics, + } = await assembleDiscoveryMatrix(userId, { + tz, + since, + fetchMode: "raw", + // The one surface that admits RATED mood factors — see the option's doc + // comment for why it is one and not four. + includeMoodFactors: true, + extraMeasurementTypes: Array.from( + new Set([ + ...DELTA_METRICS.map((m) => m.type), + ...VITALS_BASELINE_TYPES, + ]), + ), + }); // QA F2 — surfaces when a dense account's window exceeded the read cap, // mirroring the route's identical annotation. The cap now falls on the @@ -617,71 +533,12 @@ export async function buildPeriodNarrativeContext( action: { name: "insights.period-narrative.read" }, meta: { period, - measurements_capped: measurementsCapped, - mood_entries_capped: moodCapped, + measurements_capped: diagnostics.measurementsCapped, + mood_entries_capped: diagnostics.moodCapped, + mood_factor_channels: diagnostics.moodFactorChannels, }, }); - const seriesByMetric = new Map(); - for (const type of measurementTypes) { - const rows = byType.get(type); - if (!rows || rows.length === 0) continue; - seriesByMetric.set( - type, - buildMeasurementDailySeries(type, rows, tz, priorityJson), - ); - } - const moodPoints = toDailyMeans( - moodEntries.map((e) => ({ value: e.score, at: e.moodLoggedAt })), - tz, - ); - if (moodPoints.length > 0) seriesByMetric.set("MOOD", moodPoints); - - // v1.14.0 — RATED-factor channels. Flatten every entry's RATED links into - // one inverse-flipped daily-mean series per factor, keyed `FACTOR:`. - const factorLinks: FactorLink[] = []; - for (const e of moodEntries) { - for (const link of e.tagLinks) { - if (link.rating == null) continue; - factorLinks.push({ - key: link.moodTag.key, - rating: link.rating, - scaleMin: link.moodTag.scaleMin, - scaleMax: link.moodTag.scaleMax, - inverse: link.moodTag.inverse, - at: e.moodLoggedAt, - }); - } - } - const factorSeries = factorDailyMeans(factorLinks, tz); - for (const [key, points] of factorSeries) seriesByMetric.set(key, points); - - // Discovery matrix over the same window. - const discoverySeries: NamedSeries[] = []; - for (const key of DISCOVERY_BEHAVIOURS) { - discoverySeries.push({ - key, - role: "behaviour", - points: seriesByMetric.get(key) ?? [], - }); - } - for (const key of DISCOVERY_OUTCOMES) { - discoverySeries.push({ - key, - role: "outcome", - points: seriesByMetric.get(key) ?? [], - }); - } - // A factor is plausibly both a lag source ("rated work today → next-day - // sleep") and a lag target ("steps today → next-day rated energy"), so it - // enters as both roles. The engine skips self-pairs and BH-FDR-controls the - // wider family this opens. Factors the user rated on < the paired-day floor - // simply never get tested — no new statistical machinery, same honesty. - for (const [key, points] of factorSeries) { - discoverySeries.push({ key, role: "behaviour", points }); - discoverySeries.push({ key, role: "outcome", points }); - } - return assemblePeriodNarrativeContext({ period, currentFrom,