diff --git a/src/__tests__/discovery-matrix-guard.test.ts b/src/__tests__/discovery-matrix-guard.test.ts new file mode 100644 index 000000000..ca4d3c9f2 --- /dev/null +++ b/src/__tests__/discovery-matrix-guard.test.ts @@ -0,0 +1,195 @@ +/** + * Structural guards on the correlation-discovery matrix. + * + * The matrix is a channel SET, and a channel set copied into four files is a + * channel set that will differ in three of them. It did: the route and the + * per-metric card folded in the environmental-exposure and custom-metric + * families, the Coach `get_correlations` tool folded in neither, and the + * period narrative folded in neither of those plus neither of the + * compliance / symptom pair. Nobody wrote anything wrong — a family was added + * where it was needed and the other call sites were not on anyone's screen. + * `src/lib/insights/discovery-matrix.ts` removes the copies; these guards are + * what stop a fifth one appearing. + * + * ## What each guard rests on + * + * T1 freezes who may FETCH a channel. A new surface that wants a matrix has to + * either call the assembler or import a fetcher, and the second is the one this + * catches. + * + * T2 freezes who may CONSTRUCT a channel — the `role: "behaviour" | "outcome"` + * literal that makes a `NamedSeries`. A hand-rolled channel that never touches + * `correlation-channel-series.ts` would slip T1 and lands here. + * + * T3 freezes who may SCAN. This is the weakest of the three and the most + * likely to read as the strong one, so: passing T3 proves only that the list of + * files calling `discoverCorrelations` has not changed silently. It says + * nothing about what series those files pass. T1 and T2 are what make the + * series the same series. + * + * ## What they cannot do + * + * They are tripwires, not proofs. A reviewer who waves through an addition to + * an allowlist defeats all three. They match import statements and source + * text, so a fetcher reached through a re-export, a dynamic `import()`, or a + * channel built by a helper that takes `role` as a parameter would pass + * unseen. And they cannot tell a deliberate divergence from an accidental one: + * `assembleDiscoveryMatrix`'s options are where a real difference gets + * declared, and only a reader can judge whether the reason written there is + * a reason. + * + * Every match set below asserts a non-zero count. A matcher that has silently + * stopped matching agrees with any allowlist, and that failure mode is exactly + * how `bearer-scope-enforcement-guard` stayed green over a file it never saw. + */ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { walkSourceFiles } from "./helpers/source-files"; + +const SRC = join(process.cwd(), "src"); + +/** The one assembler. Every allowlist below is "this, plus the unavoidable". */ +const ASSEMBLER = "lib/insights/discovery-matrix.ts"; + +/** The module the channel fetchers live in — it defines them, so it matches. */ +const CHANNEL_SERIES = "lib/insights/correlation-channel-series.ts"; + +/** + * Every non-test `.ts` / `.tsx` under `src/`, excluding the generated Prisma + * client (9 MB; never read it) and test files themselves. + */ +function sourceFiles(): string[] { + return walkSourceFiles(SRC, { floor: 3000 }) + .filter((p) => !p.startsWith("generated/")) + .filter((p) => !p.includes("__tests__")) + .filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx")) + .sort(); +} + +function read(rel: string): string { + return readFileSync(join(SRC, rel), "utf8"); +} + +function filesMatching(re: RegExp): string[] { + return sourceFiles().filter((rel) => re.test(read(rel))); +} + +describe("T1 — only the assembler fetches discovery channels", () => { + /** + * The per-channel DB reads. A surface holding these is assembling a matrix + * by hand, whatever it calls the variable. + * + * `fetchLabDraws` is deliberately absent: the labs pass runs over + * `LabDrawPoint[]` in a separate point-vs-window scan, not over the + * behaviour × outcome day grid, so the route and the Coach tool fetch it + * directly and the assembler does not own it. + */ + const CHANNEL_FETCHERS = [ + "fetchComplianceSeries", + "fetchSymptomSeries", + "fetchEnvironmentSeries", + "fetchCustomMetricBehaviourSeries", + "fetchMoodWindowSeries", + "fetchMoodFactorWindowSeries", + "fetchMeasurementDailySeriesTiered", + "fetchMeasurementWindowSeries", + ] as const; + + /** + * `correlation-channel-series.ts` defines them all. The assembler is the one + * consumer. Nothing else. + */ + const FETCH_ALLOWLIST = [ASSEMBLER, CHANNEL_SERIES].sort(); + + for (const fetcher of CHANNEL_FETCHERS) { + it(`${fetcher} is reached only from the assembler`, () => { + const matches = filesMatching(new RegExp(`\\b${fetcher}\\b`)); + // A matcher that found nothing would agree with any allowlist. + expect(matches.length).toBeGreaterThan(0); + expect(matches).toEqual(FETCH_ALLOWLIST); + }); + } +}); + +describe("T2 — only the declared builders construct a discovery channel", () => { + /** + * A `NamedSeries` is identified by its `role` literal. Whitespace-tolerant: + * the same call written across two lines is the same call, and a matcher + * that demands one line is a matcher that can be defeated by prettier. + */ + const ROLE_LITERAL = /role:\s*"(?:behaviour|outcome)"/; + + /** + * Files allowed to mint a channel. + * + * - the assembler, which folds the matrix; + * - `correlation-channel-series.ts` + `correlation-series-builders.ts`, + * which shape one channel each from its own source; + * - `correlation-discovery.ts`, which declares the `NamedSeries` type and + * re-runs the engine over a filtered copy in the emerging pass; + * - `cycle/phase-crosstab.ts`, which builds a DELIBERATELY separate, + * cycle-gated matrix (CYCLE_PHASE × outcomes). That one must never reach + * the non-gated surfaces, which is the opposite requirement to this + * file's, and is why it is listed rather than merged. + */ + const CONSTRUCTION_ALLOWLIST = [ + ASSEMBLER, + CHANNEL_SERIES, + "lib/insights/correlation-series-builders.ts", + "lib/insights/correlation-discovery.ts", + "lib/cycle/phase-crosstab.ts", + ].sort(); + + it("no file outside the allowlist constructs a NamedSeries", () => { + const matches = filesMatching(ROLE_LITERAL); + expect(matches.length).toBeGreaterThan(0); + expect(matches).toEqual(CONSTRUCTION_ALLOWLIST); + }); +}); + +describe("T3 — the set of files that scan a matrix is frozen", () => { + /** + * Whitespace-tolerant on purpose: `discoverCorrelations(\n series,` is the + * same call, and a literal `discoverCorrelations(` matcher would miss it. + */ + const SCAN_CALL = /\bdiscoverCorrelations\s*\(/; + + /** + * The four discovery surfaces, plus the two engine-internal uses. + * + * Being on this list is NOT permission to build a matrix — T1 and T2 decide + * that. It is permission to scan one, and every entry here gets its series + * from `assembleDiscoveryMatrix` except the two that are the engine. + */ + const SCAN_ALLOWLIST = [ + // The four surfaces. Each calls the assembler for its series. + "app/api/insights/correlations/route.ts", + "lib/insights/metric-correlation-context.ts", + "lib/ai/coach/tools/correlations-read.ts", + "lib/insights/narrative/period-narrative.ts", + // The engine itself: the declaration and the emerging-window re-run. + "lib/insights/correlation-discovery.ts", + // The cycle-gated phase matrix — see T2's note. + "lib/cycle/phase-crosstab.ts", + ].sort(); + + it("no file outside the allowlist runs the discovery scan", () => { + const matches = filesMatching(SCAN_CALL); + expect(matches.length).toBeGreaterThan(0); + expect(matches).toEqual(SCAN_ALLOWLIST); + }); + + it("every scanning surface takes its series from the assembler", () => { + const surfaces = SCAN_ALLOWLIST.filter( + (rel) => + rel !== "lib/insights/correlation-discovery.ts" && + rel !== "lib/cycle/phase-crosstab.ts", + ); + expect(surfaces).toHaveLength(4); + for (const rel of surfaces) { + expect(read(rel)).toMatch(/\bassembleDiscoveryMatrix\s*\(/); + } + }); +}); diff --git a/src/app/api/insights/correlations/route.ts b/src/app/api/insights/correlations/route.ts index bde103c8f..7ae6557da 100644 --- a/src/app/api/insights/correlations/route.ts +++ b/src/app/api/insights/correlations/route.ts @@ -11,9 +11,12 @@ * causal. * * Reads daily series bounded to a trailing window, day-keyed in the user's - * display timezone (late-night readings mis-bucket under UTC). Pure compute - * lives in `src/lib/insights/correlation-discovery.ts`; this route only - * fetches + day-keys + responds. No LLM, no narrative, no cache table. + * display timezone (late-night readings mis-bucket under UTC). The channel set + * comes from `src/lib/insights/discovery-matrix.ts`, the one assembler every + * discovery surface shares; the pure compute lives in + * `src/lib/insights/correlation-discovery.ts`. This route only asks for the + * matrix, scans it, persists the pattern decisions and responds. No LLM, no + * narrative, no cache table. */ import { apiHandler, requireRecordAuth } from "@/lib/api-handler"; import { apiError, apiSuccess } from "@/lib/api-response"; @@ -25,34 +28,19 @@ import { requireModuleEnabled } from "@/lib/modules/gate"; import { resolveServerLocale } from "@/lib/i18n/server-locale"; import { prisma } from "@/lib/db"; import { wallClockInTz } from "@/lib/tz/wall-clock"; -import type { MeasurementType } from "@/generated/prisma/client"; import { discoverCorrelations, discoverEmergingCorrelations, discoverLabOutcomeCorrelations, - discoveryMeasurementTypes, - DISCOVERY_BEHAVIOURS, - DISCOVERY_OUTCOMES, EARLY_WINDOW_DAYS, - MEDICATION_COMPLIANCE_CHANNEL_KEY, - SYMPTOM_SEVERITY_CHANNEL_KEY, - type DailySeriesPoint, - type NamedSeries, } from "@/lib/insights/correlation-discovery"; import { decisionForEvidence, PATTERN_FAMILIES, syncAcceptedPatterns, } from "@/lib/insights/correlation-patterns"; -import { - fetchComplianceSeries, - fetchCustomMetricBehaviourSeries, - fetchEnvironmentSeries, - fetchLabDraws, - fetchMeasurementDailySeriesTiered, - fetchMoodWindowSeries, - fetchSymptomSeries, -} from "@/lib/insights/correlation-channel-series"; +import { assembleDiscoveryMatrix } from "@/lib/insights/discovery-matrix"; +import { fetchLabDraws } from "@/lib/insights/correlation-channel-series"; export const dynamic = "force-dynamic"; @@ -125,85 +113,22 @@ async function buildCorrelationsResponse( const now = new Date(); const since = new Date(now.getTime() - WINDOW_DAYS * MS_PER_DAY); - // Non-MeasurementType channels are backed by other models, not measurements — - // MOOD (MoodEntry), MEDICATION_COMPLIANCE (the dose-history ledger), and - // SYMPTOM_SEVERITY (the illness day-log). `discoveryMeasurementTypes` drops - // them so the `type IN (...)` query carries only real enum values; each is - // built from its own source below and folded into the series. - const behaviourTypes = discoveryMeasurementTypes( - DISCOVERY_BEHAVIOURS, - ) as MeasurementType[]; - const outcomeTypes = discoveryMeasurementTypes( - DISCOVERY_OUTCOMES, - ) as MeasurementType[]; - - // Rollup read-swap for the measurement channels: eligible spot channels - // read their per-day means from the DAY rollup tier, with per-channel - // fallback to the raw `fetchMeasurementWindowSeries` path on any miss - // (coverage gap, far-from-UTC profile tz, sleep / cumulative grain) — - // see `fetchMeasurementDailySeriesTiered` for the full parity contract. - // The SWR cache cell above stays in front of this read: the swap only - // lowers the cost of a cache MISS, it does not replace the cache. - const [measurementDaily, moodWindow] = await Promise.all([ - fetchMeasurementDailySeriesTiered(userId, tz, since, [ - ...behaviourTypes, - ...outcomeTypes, - ]), - fetchMoodWindowSeries(userId, tz, since), - ]); - const { measurementsCapped } = measurementDaily; - const { moodDaily, moodCapped } = moodWindow; - - // v1.21.0 (FDREXTEND) — build the two non-measurement, non-mood channels from - // their own sources. Each degrades to an empty series when the user has no - // data, so the discovery loop drops the channel (it cannot clear n ≥ 20). - // v1.22 — lab draws (for the labs ↔ outcome pass) fetch alongside. - const [ - complianceSeries, - symptomSeries, - labDraws, - environmentSeries, - customMetricSeries, - ] = await Promise.all([ - fetchComplianceSeries(userId, tz, since), - fetchSymptomSeries(userId, tz, since), + // The channel set comes from the one assembler every discovery surface + // shares (`discovery-matrix.ts`) — this route, the per-metric card, the Coach + // tool and the period narrative all scan the same matrix by construction. + // `"tiered"` puts the eligible measurement channels on the DAY rollup + // read-swap, with per-channel fallback to the raw path on any miss (coverage + // gap, far-from-UTC profile tz, sleep / cumulative grain). The SWR cache cell + // above stays in front of this read: the swap only lowers the cost of a cache + // MISS, it does not replace the cache. + // + // v1.22 — lab draws (for the labs ↔ outcome pass) fetch alongside; they feed + // a different pass over a different grain, so they are not matrix channels. + const [matrix, labDraws] = await Promise.all([ + assembleDiscoveryMatrix(userId, { tz, since, fetchMode: "tiered" }), fetchLabDraws(userId, tz, since), - // v1.25 (W-ENV) — environmental-exposure behaviour channels (weather / - // daylight). Empty when the module is off / no home set, so the channels - // degrade to absent. The module gate is implicit: no rows ⇒ no channels. - fetchEnvironmentSeries(userId, since), - fetchCustomMetricBehaviourSeries(userId, tz, since), ]); - - const points = (key: string): DailySeriesPoint[] => - key === "MOOD" ? moodDaily : (measurementDaily.byType.get(key) ?? []); - - const series: NamedSeries[] = []; - for (const key of DISCOVERY_BEHAVIOURS) { - if (key === MEDICATION_COMPLIANCE_CHANNEL_KEY) { - series.push(complianceSeries); - } else if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "behaviour" }); - } else { - series.push({ key, role: "behaviour", points: points(key) }); - } - } - for (const key of DISCOVERY_OUTCOMES) { - if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "outcome" }); - } else { - series.push({ key, role: "outcome", points: points(key) }); - } - } - // v1.25 (W-ENV) — fold the environmental-exposure behaviour channels in. They - // pair (lag D → D+1) against every outcome above; the n ≥ 20 / FDR / effect- - // size gates apply unchanged, so a thin weather series degrades to absent. - for (const envSeries of environmentSeries) { - series.push(envSeries); - } - for (const customSeries of customMetricSeries) { - series.push(customSeries); - } + const { series, diagnostics } = matrix; const result = discoverCorrelations(series, { locale }); @@ -278,15 +203,15 @@ async function buildCorrelationsResponse( // half of the window may be thin. // The cap can only apply to raw-path channels — rollup-served // channels read one row per day per source and cannot reach it. - measurements_capped: measurementsCapped, + measurements_capped: diagnostics.measurementsCapped, // Rollup read-swap reach: how many measurement channels rode the DAY // rollup tier on this build (the rest took the raw fallback path). - measurement_rollup_channels: measurementDaily.rollupTypes.length, - mood_entries_capped: moodCapped, + measurement_rollup_channels: diagnostics.rollupTypes.length, + mood_entries_capped: diagnostics.moodCapped, // FDREXTEND — per-channel day-counts so a dashboard can see whether the // two sparse new channels reached the n ≥ 20 floor or degraded to absent. - compliance_days: complianceSeries.points.length, - symptom_days: symptomSeries.points.length, + compliance_days: diagnostics.complianceDays, + symptom_days: diagnostics.symptomDays, // v1.22 — early-detection + labs reach. emerging: emerging.emerging.length, emerging_window_days: emerging.windowDays, @@ -295,15 +220,9 @@ async function buildCorrelationsResponse( // v1.25 (W-ENV) — env channel reach (sum of stored daily points across // the exposure channels) so a dashboard can see whether weather was // available for the scan. - environment_days: environmentSeries.reduce( - (sum, s) => sum + s.points.length, - 0, - ), - custom_metric_channels: customMetricSeries.length, - custom_metric_days: customMetricSeries.reduce( - (sum, channel) => sum + channel.points.length, - 0, - ), + environment_days: diagnostics.environmentDays, + custom_metric_channels: diagnostics.customMetricChannels, + custom_metric_days: diagnostics.customMetricDays, }, }); diff --git a/src/lib/ai/coach/__tests__/correlations-read.test.ts b/src/lib/ai/coach/__tests__/correlations-read.test.ts index 98f93b41f..b5c324aaa 100644 --- a/src/lib/ai/coach/__tests__/correlations-read.test.ts +++ b/src/lib/ai/coach/__tests__/correlations-read.test.ts @@ -13,6 +13,8 @@ const intakeEventFindMany = vi.fn(); const illnessEpisodeFindMany = vi.fn(); const illnessDayLogFindMany = vi.fn(); const labResultFindMany = vi.fn(); +const environmentContextFindMany = vi.fn(); +const customMetricFindMany = vi.fn(); vi.mock("@/lib/modules/gate", () => ({ isModuleEnabled: vi.fn(async () => true), })); @@ -26,6 +28,14 @@ vi.mock("@/lib/db", () => ({ illnessEpisode: { findMany: (a: unknown) => illnessEpisodeFindMany(a) }, illnessDayLog: { findMany: (a: unknown) => illnessDayLogFindMany(a) }, labResult: { findMany: (a: unknown) => labResultFindMany(a) }, + // The reader assembles the SHARED discovery matrix, so the + // environmental-exposure and custom-metric channels are read here too. + // Before that these two models were never touched on this path, which is + // the whole defect: the Coach could not correlate weather with anything. + environmentContext: { + findMany: (a: unknown) => environmentContextFindMany(a), + }, + customMetric: { findMany: (a: unknown) => customMetricFindMany(a) }, }, })); @@ -65,6 +75,8 @@ describe("readCoachCorrelations", () => { illnessEpisodeFindMany.mockReset().mockResolvedValue([]); illnessDayLogFindMany.mockReset().mockResolvedValue([]); labResultFindMany.mockReset().mockResolvedValue([]); + environmentContextFindMany.mockReset().mockResolvedValue([]); + customMetricFindMany.mockReset().mockResolvedValue([]); loadBaselineProfile .mockReset() .mockResolvedValue({ ageYears: 40, sex: "MALE", heightCm: 180 }); @@ -260,4 +272,95 @@ describe("readCoachCorrelations", () => { expect(symptomDriver?.outcome).toBe("sleep duration"); expect(symptomDriver?.n).toBeGreaterThanOrEqual(20); }); + + // ── the matrix the Coach scans is the shared one ──────────────────────── + // + // The Coach tool's header promised, from v1.21.0, "the SAME full-matrix scan + // the /api/insights/correlations route + the per-metric card run". It was not + // true from v1.25 onward: the environmental-exposure channels landed at the + // route and at the card, never here, because each surface folded its own + // channel list. A user asking the Coach whether air pressure related to their + // sleep met a matrix the pair had never been in. + // + // These two tests are the pair that matters. The first pins the read (before + // the shared assembler, `EnvironmentContext` was never queried on this path + // at all — the mock below would record zero calls). The second drives the + // REAL engine end-to-end and asserts the pair comes out the other side as a + // narratable driver. + + it("reads the environmental-exposure channels (the route's matrix, not a subset)", async () => { + discoverCorrelations.mockReturnValue({ + discovered: [], + pairsTested: 0, + fdrQ: 0.1, + minPairs: 20, + }); + await readCoachCorrelations("u1", "en"); + expect(environmentContextFindMany).toHaveBeenCalledTimes(1); + expect(customMetricFindMany).toHaveBeenCalledTimes(1); + + // Not just "a query happened": the channels reach the scan. + const series = discoverCorrelations.mock.calls[0]?.[0] as Array<{ + key: string; + }>; + const keys = series.map((s) => s.key); + expect(keys).toContain("ENV_PRESSURE_MEAN"); + expect(keys).toContain("ENV_DAYLIGHT"); + }); + + it("surfaces a barometric-pressure driver through the real engine", async () => { + const real = await vi.importActual< + typeof import("@/lib/insights/correlation-discovery") + >("@/lib/insights/correlation-discovery"); + discoverCorrelations.mockImplementation((series) => + real.discoverCorrelations( + series as Parameters[0], + { locale: "en" }, + ), + ); + + // 45 consecutive days. Today's barometric pressure (behaviour, day D) + // tracks the NEXT day's sleep (outcome, day D+1) with a deterministic + // linear relationship, so the pair clears n ≥ 20 / p < 0.05 / FDR / the + // effect-size floor rather than riding on noise. + const DAYS = 45; + const base = Date.UTC(2026, 4, 1, 12, 0, 0); // midday → tz-stable day key + const dayKey = (offset: number): string => + new Date(base + offset * 86_400_000).toISOString().slice(0, 10); + + const envRows: Array<{ date: string; pressureMean: number }> = []; + const measurements: Array<{ + type: string; + value: number; + measuredAt: Date; + }> = []; + for (let i = 0; i < DAYS; i++) { + const swing = i % 5; // 0..4, real variance across the window + envRows.push({ date: dayKey(i), pressureMean: 1000 + swing }); + // Higher pressure today → shorter sleep tomorrow (a clean negative r). + measurements.push({ + type: "SLEEP_DURATION", + value: 8 - swing * 0.5, + measuredAt: new Date(base + (i + 1) * 86_400_000), + }); + } + + measurementFindMany.mockResolvedValue(measurements); + // Every other env column is null, so only the pressure channel carries + // points and the assertion below cannot be satisfied by a sibling field. + environmentContextFindMany.mockResolvedValue(envRows); + + const result = await readCoachCorrelations("u1", "en"); + + expect(result.present).toBe(true); + const pressureDriver = result.drivers?.find( + (d) => d.behaviour === "barometric pressure", + ); + expect(pressureDriver).toBeDefined(); + expect(pressureDriver?.outcome).toBe("sleep duration"); + expect(pressureDriver?.n).toBeGreaterThanOrEqual(20); + expect(pressureDriver?.direction).toBe("lower"); + // The channel key must never leak into prose the Coach reads out. + expect(pressureDriver?.behaviour).not.toContain("ENV_"); + }); }); diff --git a/src/lib/ai/coach/tools/correlations-read.ts b/src/lib/ai/coach/tools/correlations-read.ts index 8bf7f8fd4..2ad9c1716 100644 --- a/src/lib/ai/coach/tools/correlations-read.ts +++ b/src/lib/ai/coach/tools/correlations-read.ts @@ -6,11 +6,23 @@ * computed elsewhere and reused verbatim here (no new statistics): * * - The FDR-controlled, day-D→D+1 lagged all-pairs discovery - * (`@/lib/insights/correlation-discovery`). We run the SAME full-matrix scan - * the `/api/insights/correlations` route + the per-metric card run, so the - * Coach never surfaces a pair the insight pages would not — same Pearson, - * same exact p-value, same Benjamini-Hochberg control. We return every - * surviving pair (not filtered to one metric) as a descriptive driver row. + * (`@/lib/insights/correlation-discovery`), over the channel set the one + * shared assembler builds (`@/lib/insights/discovery-matrix`). That is the + * same matrix the `/api/insights/correlations` route and the per-metric card + * scan, so the Coach never surfaces a pair the insight pages would not, and + * never misses one they would — same channels, same Pearson, same exact + * p-value, same Benjamini-Hochberg control. We return every surviving pair + * (not filtered to one metric) as a descriptive driver row. + * + * This file claimed that parity from v1.21.0 and did not have it. When the + * environmental-exposure channels arrived in v1.25 and the custom-metric + * channels after them, both landed at the route and at the card and not + * here, because each surface folded its own channel list. The sentence above + * stayed, which is worse than no sentence: a reader checking whether the + * Coach could discuss barometric pressure and sleep found a promise that it + * could, and the pair had never been in its matrix. The promise is now + * structural — the assembler is the only place a matrix is built, frozen by + * `src/__tests__/discovery-matrix-guard.test.ts`. * - The coincident-deviation flag (`computeCoincidentDeviation`): "two or more * of your vitals are outside their usual band today", with the illness- * explained reframe carried through. @@ -22,8 +34,8 @@ * interpretation string — so the Coach states the observed linkage without * inventing a relationship. */ -import type { MeasurementType } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; +import { ENVIRONMENT_FIELDS } from "@/lib/environment/fields"; import type { Locale } from "@/lib/i18n/config"; import { annotate } from "@/lib/logging/context"; import { isModuleEnabled } from "@/lib/modules/gate"; @@ -32,23 +44,12 @@ import { discoverCorrelations, discoverEmergingCorrelations, discoverLabOutcomeCorrelations, - discoveryMeasurementTypes, - DISCOVERY_BEHAVIOURS, - DISCOVERY_OUTCOMES, EARLY_WINDOW_DAYS, MEDICATION_COMPLIANCE_CHANNEL_KEY, SYMPTOM_SEVERITY_CHANNEL_KEY, - type NamedSeries, } from "@/lib/insights/correlation-discovery"; -import { - buildMeasurementDailySeries, - fetchComplianceSeries, - fetchLabDraws, - fetchMeasurementWindowSeries, - fetchMoodWindowSeries, - fetchSymptomSeries, -} from "@/lib/insights/correlation-channel-series"; -import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { assembleDiscoveryMatrix } from "@/lib/insights/discovery-matrix"; +import { fetchLabDraws } from "@/lib/insights/correlation-channel-series"; import { computeCoincidentDeviation, loadBaselineProfile, @@ -133,14 +134,30 @@ function tzDayKey(at: Date, tz: string): string { return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`; } -/** Natural labels for the non-measurement channel keys (read cleanly in prose). */ +/** + * Natural labels for the non-measurement channel keys (read cleanly in prose). + * + * The environmental-exposure channels take their phrase from the one env field + * vocabulary rather than a second hand-written list, so a new exposure field + * cannot reach the Coach as "env pressure delta". + */ const CHANNEL_LABELS: Record = { [MEDICATION_COMPLIANCE_CHANNEL_KEY]: "medication adherence", [SYMPTOM_SEVERITY_CHANNEL_KEY]: "symptom severity", + ...Object.fromEntries( + ENVIRONMENT_FIELDS.map((field) => [field.key, field.narrationLabel]), + ), }; -/** Lower-case, space-separated label from a discovery channel key. */ -function humanise(key: string): string { +/** + * Lower-case, space-separated label for a discovery channel. + * + * `label` is the engine's per-pair label for a DYNAMIC channel — today that is + * a custom metric, whose key is `CUSTOM_METRIC:`. It has to win over the + * key: prettifying that key would put a database id into the Coach's prose. + */ +function humanise(key: string, label?: string): string { + if (label && label.trim().length > 0) return label.trim().toLowerCase(); return CHANNEL_LABELS[key] ?? key.replace(/_/g, " ").toLowerCase(); } @@ -194,80 +211,22 @@ export async function readCoachCorrelations( const tz = userRow?.timezone ?? "Europe/Berlin"; const since = new Date(Date.now() - WINDOW_DAYS * MS_PER_DAY); - // The non-MeasurementType channels (MOOD, MEDICATION_COMPLIANCE, - // SYMPTOM_SEVERITY) are backed by other models — `discoveryMeasurementTypes` - // drops them so the Postgres `type IN (...)` enum cast only ever sees real - // enum values; each is sourced separately below, exactly like the route. - const behaviourTypes = discoveryMeasurementTypes( - DISCOVERY_BEHAVIOURS, - ) as MeasurementType[]; - const outcomeTypes = discoveryMeasurementTypes( - DISCOVERY_OUTCOMES, - ) as MeasurementType[]; - - // v1.30.3 (QA F1) — the fetch + desc/cap/resort discipline (a dense - // account's cap must fall on the OLDEST rows, never the newest — the - // emerging-correlations pass below is entirely about the recent window) - // now lives in `fetchMeasurementWindowSeries` / `fetchMoodWindowSeries` - // (`correlation-channel-series.ts`), shared with the route, the - // per-metric card, and the period narrative. - const [ - { byType, measurementsCapped }, - { moodDaily, moodCapped }, - coincidentDerived, - priorityJson, - ] = await Promise.all([ - fetchMeasurementWindowSeries(userId, since, [ - ...behaviourTypes, - ...outcomeTypes, - ]), - fetchMoodWindowSeries(userId, tz, since), + // The channel set is the shared one, so the Coach scans exactly what the + // correlations page scans. The measurement read stays `"raw"` (the route's + // rollup read-swap is a read-cost choice, not a channel-set one). + // + // v1.22 — the lab draws feed a separate point-vs-window pass, not the + // matrix, so they are fetched alongside rather than assembled in. + const [matrix, coincidentDerived, labDraws] = await Promise.all([ + assembleDiscoveryMatrix(userId, { tz, since, fetchMode: "raw" }), // Coincident-deviation is its own derived metric — fail-soft to null so a // baseline hiccup never sinks the whole correlations read. D2-8: pass the // user's tz so the "today" grouping matches the user's calendar day, not // UTC's, before the fired flag is narrated as "out of band TODAY". computeCoincidentDeviation(userId, profile, { tz }).catch(() => null), - loadUserSourcePriority(userId), - ]); - const seriesPoints = (key: string) => - key === "MOOD" - ? moodDaily - : buildMeasurementDailySeries( - key as MeasurementType, - byType.get(key) ?? [], - tz, - priorityJson, - ); - - // The two non-measurement, non-mood channels come from their own sources - // (the dose-history ledger + the illness day-log), folded in below exactly - // like the route. Each degrades to an empty series when the user has no - // data, so the discovery loop drops the channel (it cannot clear the n ≥ 20 - // floor) — it can never surface a fabricated driver. - const [complianceSeries, symptomSeries, labDraws] = await Promise.all([ - fetchComplianceSeries(userId, tz, since), - fetchSymptomSeries(userId, tz, since), - // v1.22 — lab draws for the labs ↔ outcome pass (degrades to absent). fetchLabDraws(userId, tz, since), ]); - - const series: NamedSeries[] = []; - for (const key of DISCOVERY_BEHAVIOURS) { - if (key === MEDICATION_COMPLIANCE_CHANNEL_KEY) { - series.push(complianceSeries); - } else if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "behaviour" }); - } else { - series.push({ key, role: "behaviour", points: seriesPoints(key) }); - } - } - for (const key of DISCOVERY_OUTCOMES) { - if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "outcome" }); - } else { - series.push({ key, role: "outcome", points: seriesPoints(key) }); - } - } + const { series, diagnostics } = matrix; // QA F1 — surfaces when a dense account's window exceeded the read cap, // mirroring the route's identical annotation. The cap now falls on the @@ -276,15 +235,20 @@ export async function readCoachCorrelations( annotate({ action: { name: "coach.correlations.read" }, meta: { - measurements_capped: measurementsCapped, - mood_entries_capped: moodCapped, + measurements_capped: diagnostics.measurementsCapped, + mood_entries_capped: diagnostics.moodCapped, + // The environmental-exposure and custom-metric families + // reach this path for the first time; these say whether they carried + // anything on a given turn. + environment_days: diagnostics.environmentDays, + custom_metric_channels: diagnostics.customMetricChannels, }, }); const discovery = discoverCorrelations(series, { locale }); const drivers: CoachCorrelationDriver[] = discovery.discovered.map((d) => ({ - behaviour: humanise(d.behaviour), - outcome: humanise(d.outcome), + behaviour: humanise(d.behaviour, d.behaviourLabel), + outcome: humanise(d.outcome, d.outcomeLabel), direction: d.r >= 0 ? "higher" : "lower", lagDays: d.lagDays, n: d.n, @@ -306,8 +270,8 @@ export async function readCoachCorrelations( }); const emerging: CoachEmergingDriver[] = emergingResult.emerging.map( (d) => ({ - behaviour: humanise(d.behaviour), - outcome: humanise(d.outcome), + behaviour: humanise(d.behaviour, d.behaviourLabel), + outcome: humanise(d.outcome, d.outcomeLabel), direction: d.r >= 0 ? "higher" : "lower", lagDays: d.lagDays, n: d.n, @@ -323,6 +287,8 @@ export async function readCoachCorrelations( }); const labDrivers: CoachLabCorrelation[] = labResult.discovered.map((d) => ({ lab: d.lab.startsWith("LAB:") ? d.lab.slice("LAB:".length) : d.lab, + // A lab pair's outcome is always a declared channel key, never a dynamic + // one, so there is no per-pair label to prefer here. outcome: humanise(d.outcome), direction: d.r >= 0 ? "higher" : "lower", n: d.n, diff --git a/src/lib/insights/__tests__/metric-correlation-context.test.ts b/src/lib/insights/__tests__/metric-correlation-context.test.ts index 703af6931..5f4f411a5 100644 --- a/src/lib/insights/__tests__/metric-correlation-context.test.ts +++ b/src/lib/insights/__tests__/metric-correlation-context.test.ts @@ -48,22 +48,34 @@ describe("getRelevantCorrelationsForMetric", () => { }); it("surfaces only FDR-surviving pairs that involve the metric's channel", async () => { - // Build two strongly-correlated daily series over 60 days: - // TIME_IN_DAYLIGHT (behaviour) → next-day RESTING_HEART_RATE (outcome), - // anti-correlated, plus uncorrelated noise on other channels. + // Two strong, INDEPENDENT lag-1 relationships over 60 days: + // TIME_IN_DAYLIGHT → next-day RESTING_HEART_RATE (involves the metric) + // BLOOD_GLUCOSE → next-day WEIGHT (does not) + // + // The second pair is what makes this test say anything. With one + // correlated pair in the fixture, "every returned relation mentions the + // metric" is satisfied by a function that returns the whole matrix — which + // is what this one did for four releases while the assertion stayed green. const rows: Array<{ type: string; value: number; measuredAt: Date }> = []; const base = new Date("2026-01-01T12:00:00Z"); for (let d = 0; d < 60; d++) { const day = new Date(base.getTime() + d * 86_400_000); + const next = new Date(base.getTime() + (d + 1) * 86_400_000); const daylight = 30 + (d % 10) * 6; // varies 30..84 rows.push({ type: "TIME_IN_DAYLIGHT", value: daylight, measuredAt: day }); // next-day RHR moves opposite to today's daylight (lag-1 anti-corr). - const next = new Date(base.getTime() + (d + 1) * 86_400_000); rows.push({ type: "RESTING_HEART_RATE", value: 90 - daylight * 0.4, measuredAt: next, }); + const glucose = 80 + (d % 7) * 5; // varies 80..110 + rows.push({ type: "BLOOD_GLUCOSE", value: glucose, measuredAt: day }); + rows.push({ + type: "WEIGHT", + value: 60 + glucose * 0.1, + measuredAt: next, + }); } vi.mocked(prisma.measurement.findMany).mockResolvedValue(rows as never); @@ -79,6 +91,11 @@ describe("getRelevantCorrelationsForMetric", () => { expect(Number.isFinite(c.r)).toBe(true); expect(c.n).toBeGreaterThanOrEqual(20); } + // And the glucose → weight pair, which the same scan certainly found, is + // not in a resting-heart-rate card's grounded context. + for (const c of out) { + expect(c.interpretation.toLowerCase()).not.toContain("weight"); + } }); it("is best-effort: a DB failure resolves to [] rather than throwing", async () => { diff --git a/src/lib/insights/correlation-channel-series.ts b/src/lib/insights/correlation-channel-series.ts index 57ae69b63..7e1be8512 100644 --- a/src/lib/insights/correlation-channel-series.ts +++ b/src/lib/insights/correlation-channel-series.ts @@ -1,14 +1,19 @@ /** - * v1.21.0 (INTEGFIX) — shared data-fetching helpers for the two non-measurement - * FDR correlation channels (medication compliance + symptom severity). + * v1.21.0 (INTEGFIX) — the DB reads behind the correlation-discovery channels. * * The pure series shapers live in `correlation-series-builders.ts`; the FDR - * engine itself is pure over `NamedSeries[]`. These two helpers own the DB reads - * that feed those shapers — the dose-history ledger (compliance) and the illness - * day-log (symptom severity) — so EVERY consumer of the discovery matrix (the - * `/api/insights/correlations` route AND the Coach `get_correlations` tool) - * builds the channels identically rather than re-implementing the queries, the - * tz keying, and the episode-span clamping per call site. + * engine itself is pure over `NamedSeries[]`. These helpers own the reads that + * feed those shapers — the dose-history ledger (compliance), the illness day-log + * (symptom severity), the daily weather rows, the opt-in custom metrics, the + * measurement window (raw and rollup-tiered), the mood window — so the queries, + * the tz keying and the episode-span clamping exist once. + * + * This module owns FETCHING one channel at a time. It does NOT decide which + * channels a surface gets: that is `discovery-matrix.ts`, the one assembler + * every consumer of the matrix calls. Fetch discipline was hoisted here in + * v1.30.3 and the assembly was left behind at four call sites, which is how the + * Coach tool spent v1.25 onward scanning a matrix with no weather in it. Add a + * channel here, admit it there, and every surface has it. * * Each helper degrades to an EMPTY series when the user has no data, so the * channel drops out of discovery (it cannot clear the n ≥ 20 floor) rather than @@ -22,6 +27,7 @@ import { SCHEDULE_COMPLIANCE_SELECT, } from "@/lib/analytics/compliance"; import { + FACTOR_CHANNEL_PREFIX, MEDICATION_COMPLIANCE_CHANNEL_KEY, SYMPTOM_SEVERITY_CHANNEL_KEY, type DailySeriesPoint, @@ -389,11 +395,10 @@ export interface MoodWindowFetch { /** * v1.30.3 (QA F1/F2/F3) — shared mood-window fetch, same desc+cap+resort - * discipline as {@link fetchMeasurementWindowSeries}. Callers that also need - * per-entry factor tag-links (the period narrative's RATED-factor channels) - * read `MoodEntry` themselves for the extra `tagLinks` select, but MUST - * apply the same desc+resort discipline — this helper only covers the - * plain-score case the route / coach tool / per-metric card share. + * discipline as {@link fetchMeasurementWindowSeries}. This helper covers the + * plain-score case; the RATED-factor variant that also reads each entry's + * tag-links is {@link fetchMoodFactorWindowSeries}, which carries the same + * discipline over a wider select. */ export async function fetchMoodWindowSeries( userId: string, @@ -417,6 +422,131 @@ export async function fetchMoodWindowSeries( return { moodDaily, moodCapped }; } +/** One mood-window fetch that also carries the RATED-factor channels. */ +export interface MoodFactorWindowFetch extends MoodWindowFetch { + /** One inverse-flipped daily-mean series per `FACTOR:` channel. */ + factorSeries: Map; +} + +/** + * A RATED mood factor link, carrying the scale + `inverse` flag so the + * documented sign-flip is applied once, at the boundary. + */ +interface FactorLink { + key: string; + rating: number; + scaleMin: number; + scaleMax: number; + inverse: boolean; + at: Date; +} + +/** + * v1.14.0 — collapse RATED-factor links to one inverse-flipped daily-mean + * series per factor, tz-day-keyed exactly like {@link toDailyMeans} so a factor + * channel joins the discovery matrix on the same day grid as every vital. + * An inverse factor's rating `r` maps to `(scaleMin + scaleMax) - r` BEFORE + * averaging so "up" always reads as a better day — the same flip the mood + * aggregates apply, kept in lock-step. Returns one `FACTOR:` series per + * factor the user actually rated. Pure. + */ +function factorDailyMeans( + links: FactorLink[], + tz: string, +): Map { + const byFactor = new Map< + string, + Map + >(); + for (const l of links) { + if (!Number.isFinite(l.rating)) continue; + const value = l.inverse ? l.scaleMin + l.scaleMax - l.rating : l.rating; + const day = tzDayKey(l.at, tz); + const days = + byFactor.get(l.key) ?? new Map(); + const acc = days.get(day) ?? { sum: 0, count: 0 }; + acc.sum += value; + acc.count += 1; + days.set(day, acc); + byFactor.set(l.key, days); + } + const out = new Map(); + for (const [key, days] of byFactor) { + out.set( + `${FACTOR_CHANNEL_PREFIX}${key}`, + [...days.entries()] + .map(([day, acc]) => ({ day, value: acc.sum / acc.count })) + .sort((a, b) => (a.day < b.day ? -1 : 1)), + ); + } + return out; +} + +/** + * Mood-window fetch WITH the per-entry RATED-factor tag-links, for the one + * surface that admits `FACTOR:*` channels into the discovery matrix (see the + * `includeMoodFactors` option on `assembleDiscoveryMatrix` for why that is one + * surface and not four). Same desc+cap+resort discipline as + * {@link fetchMoodWindowSeries} over a wider select — the tag-links ride the + * existing mood read, so this costs no extra round-trip over the plain variant. + * + * BINARY links carry a null `rating` and are excluded at the query. + */ +export async function fetchMoodFactorWindowSeries( + userId: string, + tz: string, + since: Date, +): Promise { + const rowsDesc = await prisma.moodEntry.findMany({ + where: { userId, deletedAt: null, moodLoggedAt: { gte: since } }, + orderBy: { moodLoggedAt: "desc" }, + take: MOOD_READ_CAP, + select: { + score: true, + moodLoggedAt: true, + tagLinks: { + where: { moodTag: { kind: "RATED" }, rating: { not: null } }, + select: { + rating: true, + moodTag: { + select: { + key: true, + scaleMin: true, + scaleMax: true, + inverse: true, + }, + }, + }, + }, + }, + }); + const moodCapped = rowsDesc.length >= MOOD_READ_CAP; + const rows = [...rowsDesc].sort( + (a, b) => a.moodLoggedAt.getTime() - b.moodLoggedAt.getTime(), + ); + const moodDaily = toDailyMeans( + rows.map((e) => ({ value: e.score, at: e.moodLoggedAt })), + tz, + ); + + const links: FactorLink[] = []; + for (const entry of rows) { + for (const link of entry.tagLinks) { + if (link.rating == null) continue; + links.push({ + key: link.moodTag.key, + rating: link.rating, + scaleMin: link.moodTag.scaleMin, + scaleMax: link.moodTag.scaleMax, + inverse: link.moodTag.inverse, + at: entry.moodLoggedAt, + }); + } + } + + return { moodDaily, moodCapped, factorSeries: factorDailyMeans(links, tz) }; +} + /** * v1.29.6 — collapse rows to per-day MEANS keyed in the user's tz. This is * the correct grain for spot metrics (BP, glucose, HRV, resting HR, weight, diff --git a/src/lib/insights/derived/__tests__/coach-read-locale.test.ts b/src/lib/insights/derived/__tests__/coach-read-locale.test.ts index 504d628af..c861ffce9 100644 --- a/src/lib/insights/derived/__tests__/coach-read-locale.test.ts +++ b/src/lib/insights/derived/__tests__/coach-read-locale.test.ts @@ -24,6 +24,8 @@ const intakeEventFindMany = vi.fn(); const illnessEpisodeFindMany = vi.fn(); const illnessDayLogFindMany = vi.fn(); const labResultFindMany = vi.fn(); +const environmentContextFindMany = vi.fn(); +const customMetricFindMany = vi.fn(); vi.mock("@/lib/modules/gate", () => ({ isModuleEnabled: vi.fn(async () => true), @@ -42,6 +44,12 @@ vi.mock("@/lib/db", () => ({ illnessEpisode: { findMany: (a: unknown) => illnessEpisodeFindMany(a) }, illnessDayLog: { findMany: (a: unknown) => illnessDayLogFindMany(a) }, labResult: { findMany: (a: unknown) => labResultFindMany(a) }, + // The reader assembles the shared discovery matrix, which reads the + // environmental-exposure and custom-metric channels too. + environmentContext: { + findMany: (a: unknown) => environmentContextFindMany(a), + }, + customMetric: { findMany: (a: unknown) => customMetricFindMany(a) }, }, })); @@ -129,6 +137,8 @@ describe("buildCoachReadStrip — the pattern line speaks the reader's language" illnessEpisodeFindMany.mockReset().mockResolvedValue([]); illnessDayLogFindMany.mockReset().mockResolvedValue([]); labResultFindMany.mockReset().mockResolvedValue([]); + environmentContextFindMany.mockReset().mockResolvedValue([]); + customMetricFindMany.mockReset().mockResolvedValue([]); seedSymptomDrivesSleep(); }); diff --git a/src/lib/insights/discovery-matrix.ts b/src/lib/insights/discovery-matrix.ts new file mode 100644 index 000000000..d5dc07993 --- /dev/null +++ b/src/lib/insights/discovery-matrix.ts @@ -0,0 +1,326 @@ +/** + * The one assembler for the correlation-discovery matrix. + * + * `discoverCorrelations` is pure over `NamedSeries[]`. Deciding WHICH series go + * in is the part that used to be copied: the `/api/insights/correlations` route, + * the per-metric assessment card, the Coach `get_correlations` tool and the + * period narrative each folded their own channel list, and three of the four + * carried a different one. The Coach's was missing the environmental and + * custom-metric families entirely, so a question about air pressure and sleep + * met a matrix that had never contained the pair — while the tool's own file + * header promised "the SAME full-matrix scan the route runs". v1.30.3 hoisted + * the FETCH discipline into `correlation-channel-series.ts` for exactly this + * reason and left the assembly duplicated behind; this module finishes that. + * + * Every surface that scans the matrix calls {@link assembleDiscoveryMatrix} and + * gets the same channel set. The differences that remain are declared as + * options, each with its reason written at the option — a surface cannot drop a + * channel family by omission any more, only by naming what it is doing. + * + * What this does NOT own: + * + * - the scan. Each caller still runs `discoverCorrelations` (and, where it + * applies, `discoverEmergingCorrelations`) over the returned series, because + * what each does with the result differs — the route persists patterns, the + * card filters to one metric, the Coach shapes driver rows, the narrative + * cites them. Same input, same statistics, different presentation. + * - the labs ↔ outcome pass. `discoverLabOutcomeCorrelations` runs over + * `LabDrawPoint[]` (point-vs-window over sparse draws), not the + * behaviour × outcome day grid this assembles. Its input comes from + * `fetchLabDraws` at the two call sites that want it. + * - the cycle-phase matrix. `discoverPhaseCorrelations` builds a deliberately + * separate, gated matrix (CYCLE_PHASE × outcomes) that must never reach the + * non-gated surfaces; see `src/lib/cycle/phase-crosstab.ts`. + * + * The freeze on all of this is `src/__tests__/discovery-matrix-guard.test.ts`. + */ +import type { MeasurementType } from "@/generated/prisma/client"; +import { + DISCOVERY_BEHAVIOURS, + DISCOVERY_OUTCOMES, + discoveryMeasurementTypes, + MEDICATION_COMPLIANCE_CHANNEL_KEY, + SYMPTOM_SEVERITY_CHANNEL_KEY, + type DailySeriesPoint, + type NamedSeries, +} from "@/lib/insights/correlation-discovery"; +import { + buildMeasurementDailySeries, + fetchComplianceSeries, + fetchCustomMetricBehaviourSeries, + fetchEnvironmentSeries, + fetchMeasurementDailySeriesTiered, + fetchMeasurementWindowSeries, + fetchMoodFactorWindowSeries, + fetchMoodWindowSeries, + fetchSymptomSeries, + type MoodFactorWindowFetch, +} from "@/lib/insights/correlation-channel-series"; +import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; + +/** + * How the measurement channels are read. + * + * - `"tiered"` takes the DAY rollup read-swap where a channel is eligible and + * the user's profile timezone sits in the near-UTC band, falling back to the + * raw path per channel on any miss. Cheaper on a dense account; carries the + * documented ±3 h day-key tolerance and drops the partial oldest day. + * - `"raw"` always reads raw rows and reduces them in process. + * + * This is a read-cost decision, NOT a channel-set decision, so it stays a + * per-caller choice rather than something the assembler picks. The route takes + * the swap because it scans 180 days on every cache miss; the three library + * surfaces stayed raw when the swap landed and are left there, because moving + * them is a value change at the tolerance boundary and belongs in its own + * change with its own parity proof, not folded into a channel-set fix. + */ +export type DiscoveryMatrixFetchMode = "tiered" | "raw"; + +export interface DiscoveryMatrixOptions { + /** The user's display timezone — every day key is minted in it. */ + tz: string; + /** Inclusive start of the read window. */ + since: Date; + /** See {@link DiscoveryMatrixFetchMode}. */ + fetchMode: DiscoveryMatrixFetchMode; + /** + * Admit the user's RATED mood factors as `FACTOR:` channels, in BOTH + * roles. Off everywhere except the period narrative, deliberately: + * + * - `MoodTag.kind = "RATED"` is FROZEN as of v1.37. The capture sheet stopped + * offering factor ratings when the level-A columns landed on `MoodEntry`; + * nothing was deleted, so existing scores keep rendering and keep feeding + * the factor crosstab, but no new ones are written. Widening a frozen + * capture path's reach to three more surfaces is a decision about what to + * ask, not a parity fix — the schema comment on that column says so in as + * many words. + * - The route and the per-metric card PERSIST every surviving pair into + * `CorrelationPattern.factorKey` / `.outcomeKey`, plaintext columns. A + * custom RATED tag's key is `custom:` with its label held encrypted, + * so a factor channel on those two surfaces would write a channel key that + * only resolves through a decrypt — and would narrate as the raw key + * meanwhile. + * + * So this is a real distinction, not an accident of where someone stopped: + * the narrative reads a bounded period and cites factors in prose it does not + * store. Turning it on costs no extra round-trip — the tag-links ride the + * existing mood read (see `fetchMoodFactorWindowSeries`) — but it does widen + * the BH-FDR family, since each factor enters as both lag source and lag + * target. + */ + includeMoodFactors?: boolean; + /** + * Measurement types to read ALONGSIDE the discovery channels, surfaced in + * {@link DiscoveryMatrix.byMetric} but never folded into `series`. Lets a + * caller that needs more per-metric series than the matrix contains (the + * period narrative's delta metrics and banded vitals) get them from the one + * measurement read instead of a second one. + */ + extraMeasurementTypes?: readonly MeasurementType[]; +} + +/** Per-channel reach, for the callers' wide-event annotations. */ +export interface DiscoveryMatrixDiagnostics { + /** The raw measurement read hit its cap — the window may miss OLDER rows. */ + measurementsCapped: boolean; + /** The mood read hit its cap. */ + moodCapped: boolean; + /** Measurement channels served from the DAY rollup tier (tiered mode only). */ + rollupTypes: MeasurementType[]; + complianceDays: number; + symptomDays: number; + /** Stored daily points summed across the environmental-exposure channels. */ + environmentDays: number; + customMetricChannels: number; + customMetricDays: number; + /** `FACTOR:*` channels admitted (0 unless `includeMoodFactors`). */ + moodFactorChannels: number; +} + +export interface DiscoveryMatrix { + /** The behaviour × outcome channel set, ready for `discoverCorrelations`. */ + series: NamedSeries[]; + /** + * Day-keyed series per metric key — every measurement type read (discovery + * channels plus `extraMeasurementTypes`), `MOOD`, and each admitted + * `FACTOR:*` channel. A key with no data is absent; callers read it as + * `byMetric.get(key) ?? []`. + */ + byMetric: Map; + diagnostics: DiscoveryMatrixDiagnostics; +} + +/** + * The mood read, normalised to the factor-carrying shape. With factors off the + * factor map is empty, so the fold below has one code path rather than two. + */ +async function readMood( + userId: string, + tz: string, + since: Date, + withFactors: boolean, +): Promise { + if (withFactors) return fetchMoodFactorWindowSeries(userId, tz, since); + const plain = await fetchMoodWindowSeries(userId, tz, since); + return { ...plain, factorSeries: new Map() }; +} + +/** One measurement read, reduced to per-day series under the chosen mode. */ +async function readMeasurementDaily( + userId: string, + tz: string, + since: Date, + types: MeasurementType[], + mode: DiscoveryMatrixFetchMode, +): Promise<{ + byType: Map; + measurementsCapped: boolean; + rollupTypes: MeasurementType[]; +}> { + if (mode === "tiered") { + const tiered = await fetchMeasurementDailySeriesTiered( + userId, + tz, + since, + types, + ); + return { + byType: tiered.byType, + measurementsCapped: tiered.measurementsCapped, + rollupTypes: tiered.rollupTypes, + }; + } + + const [raw, priorityJson] = await Promise.all([ + fetchMeasurementWindowSeries(userId, since, types), + loadUserSourcePriority(userId), + ]); + const byType = new Map(); + for (const type of types) { + const rows = raw.byType.get(type); + if (!rows || rows.length === 0) continue; + byType.set(type, buildMeasurementDailySeries(type, rows, tz, priorityJson)); + } + return { + byType, + measurementsCapped: raw.measurementsCapped, + rollupTypes: [], + }; +} + +/** + * Build the full discovery matrix for a user over a window. + * + * Reads every channel family in parallel and folds them in one fixed order: + * the declared behaviours, the declared outcomes, the environmental-exposure + * channels, the opt-in custom-metric channels, then (only when asked) the + * RATED mood factors in both roles. A family the user has no data for yields + * an empty series and drops out of the scan at the n ≥ 20 floor — absence is + * never a fabricated constant. + * + * The order is fixed on purpose. `discoverCorrelations` ranks by shrunk effect + * then q, and `Array.prototype.sort` is stable, so two pairs equal on both + * come out in the order they were tested — which is the order the channels + * were folded. A stable fold keeps two runs over the same data ranked the same + * way, and keeps this route's ranking identical to what it was before the + * assembly moved here. + */ +export async function assembleDiscoveryMatrix( + userId: string, + opts: DiscoveryMatrixOptions, +): Promise { + const { tz, since, fetchMode } = opts; + + // The non-MeasurementType channels (MOOD, MEDICATION_COMPLIANCE, + // SYMPTOM_SEVERITY) are backed by other models, so `discoveryMeasurementTypes` + // drops them before the list reaches a Postgres `type IN (...)` enum cast — + // a non-enum string in that list errors the cast. Each is sourced below. + const measurementTypes = Array.from( + new Set([ + ...discoveryMeasurementTypes(DISCOVERY_BEHAVIOURS), + ...discoveryMeasurementTypes(DISCOVERY_OUTCOMES), + ...(opts.extraMeasurementTypes ?? []), + ]), + ) as MeasurementType[]; + + const [ + measurements, + mood, + complianceSeries, + symptomSeries, + environmentSeries, + customMetricSeries, + ] = await Promise.all([ + readMeasurementDaily(userId, tz, since, measurementTypes, fetchMode), + readMood(userId, tz, since, opts.includeMoodFactors === true), + fetchComplianceSeries(userId, tz, since), + fetchSymptomSeries(userId, tz, since), + fetchEnvironmentSeries(userId, since), + fetchCustomMetricBehaviourSeries(userId, tz, since), + ]); + + const { factorSeries } = mood; + + const points = (key: string): DailySeriesPoint[] => + key === "MOOD" ? mood.moodDaily : (measurements.byType.get(key) ?? []); + + const series: NamedSeries[] = []; + for (const key of DISCOVERY_BEHAVIOURS) { + if (key === MEDICATION_COMPLIANCE_CHANNEL_KEY) { + series.push(complianceSeries); + } else if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { + // The symptom builder returns the outcome role; the points are + // role-invariant, so the behaviour arm re-tags the same series. + series.push({ ...symptomSeries, role: "behaviour" }); + } else { + series.push({ key, role: "behaviour", points: points(key) }); + } + } + for (const key of DISCOVERY_OUTCOMES) { + if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { + series.push({ ...symptomSeries, role: "outcome" }); + } else { + series.push({ key, role: "outcome", points: points(key) }); + } + } + // Environmental exposure and custom metrics are lag SOURCES only: they pair + // (D → D+1) against every outcome above, under the unchanged n ≥ 20 / FDR / + // effect-size gates, so a thin weather or custom series degrades to absent. + series.push(...environmentSeries, ...customMetricSeries); + // A factor is plausibly both a lag source ("rated work today → next-day + // sleep") and a lag target ("steps today → next-day rated energy"), so it + // enters as both roles. The engine skips self-pairs — every factor shares + // MOOD's family — and BH-FDR controls the wider family this opens. + for (const [key, factorPoints] of factorSeries) { + series.push({ key, role: "behaviour", points: factorPoints }); + series.push({ key, role: "outcome", points: factorPoints }); + } + + const byMetric = new Map(measurements.byType); + if (mood.moodDaily.length > 0) byMetric.set("MOOD", mood.moodDaily); + for (const [key, factorPoints] of factorSeries) { + byMetric.set(key, factorPoints); + } + + return { + series, + byMetric, + diagnostics: { + measurementsCapped: measurements.measurementsCapped, + moodCapped: mood.moodCapped, + rollupTypes: measurements.rollupTypes, + complianceDays: complianceSeries.points.length, + symptomDays: symptomSeries.points.length, + environmentDays: environmentSeries.reduce( + (sum, s) => sum + s.points.length, + 0, + ), + customMetricChannels: customMetricSeries.length, + customMetricDays: customMetricSeries.reduce( + (sum, s) => sum + s.points.length, + 0, + ), + moodFactorChannels: factorSeries.size, + }, + }; +} diff --git a/src/lib/insights/metric-correlation-context.ts b/src/lib/insights/metric-correlation-context.ts index 10cea2f33..f626b5ef3 100644 --- a/src/lib/insights/metric-correlation-context.ts +++ b/src/lib/insights/metric-correlation-context.ts @@ -13,35 +13,27 @@ * This module runs the SAME full-matrix discovery the * `/api/insights/correlations` route runs (so a card never surfaces a pair * the correlations page wouldn't), then filters to the surviving pairs that - * INVOLVE the current metric's discovery channel. The result is the - * engine's own conservative, descriptive, never-causal `interpretation` - * strings — passed verbatim into the prompt as grounded context. + * INVOLVE the current metric's discovery channel. That parity is structural + * rather than asserted: both take their channel set from the one assembler in + * `src/lib/insights/discovery-matrix.ts`, and + * `src/__tests__/discovery-matrix-guard.test.ts` freezes the assembler as the + * only place a matrix is built. The result is the engine's own conservative, + * descriptive, never-causal `interpretation` strings — passed verbatim into + * the prompt as grounded context. * * Read-only consumption. No new statistics are computed here; the only - * change vs the route is the filter to one metric and a shorter window cap. + * changes vs the route are the filter to one metric and the raw (rather than + * rollup-tiered) measurement read. */ import type { MeasurementType } from "@/generated/prisma/client"; import { prisma } from "@/lib/db"; import type { Locale } from "@/lib/i18n/config"; import { discoverCorrelations, - discoveryMeasurementTypes, DISCOVERY_BEHAVIOURS, DISCOVERY_OUTCOMES, - type NamedSeries, - MEDICATION_COMPLIANCE_CHANNEL_KEY, - SYMPTOM_SEVERITY_CHANNEL_KEY, } from "@/lib/insights/correlation-discovery"; -import { - buildMeasurementDailySeries, - fetchComplianceSeries, - fetchCustomMetricBehaviourSeries, - fetchEnvironmentSeries, - fetchMeasurementWindowSeries, - fetchMoodWindowSeries, - fetchSymptomSeries, -} from "@/lib/insights/correlation-channel-series"; -import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { assembleDiscoveryMatrix } from "@/lib/insights/discovery-matrix"; import { decisionForEvidence, PATTERN_FAMILIES, @@ -102,73 +94,15 @@ export async function getRelevantCorrelationsForMetric( const tz = profile?.timezone ?? "Europe/Berlin"; const since = new Date(Date.now() - WINDOW_DAYS * MS_PER_DAY); - // Non-measurement channels (MOOD, and v1.21.0 MEDICATION_COMPLIANCE / - // SYMPTOM_SEVERITY) are backed by other models — `discoveryMeasurementTypes` - // strips them from the measurement `type IN (...)` query (a non-enum string - // would error the cast). This per-metric-card surface does not populate the - // compliance / symptom channels (they degrade to absent here); the - // canonical `/api/insights/correlations` route builds them. - const behaviourTypes = discoveryMeasurementTypes( - DISCOVERY_BEHAVIOURS, - ) as MeasurementType[]; - const outcomeTypes = discoveryMeasurementTypes( - DISCOVERY_OUTCOMES, - ) as MeasurementType[]; - - // v1.30.3 (QA F1) — the fetch + desc/cap/resort discipline (a dense - // account's cap must fall on the OLDEST rows, never the newest) now - // lives in `fetchMeasurementWindowSeries` / `fetchMoodWindowSeries` - // (`correlation-channel-series.ts`), shared with the route, the Coach - // tool, and the period narrative. - const [ - { byType }, - { moodDaily }, - priorityJson, - complianceSeries, - symptomSeries, - environmentSeries, - customMetricSeries, - ] = await Promise.all([ - fetchMeasurementWindowSeries(userId, since, [ - ...behaviourTypes, - ...outcomeTypes, - ]), - fetchMoodWindowSeries(userId, tz, since), - loadUserSourcePriority(userId), - fetchComplianceSeries(userId, tz, since), - fetchSymptomSeries(userId, tz, since), - fetchEnvironmentSeries(userId, since), - fetchCustomMetricBehaviourSeries(userId, tz, since), - ]); - - const points = (key: string) => - key === "MOOD" - ? moodDaily - : buildMeasurementDailySeries( - key as MeasurementType, - byType.get(key) ?? [], - tz, - priorityJson, - ); - - const series: NamedSeries[] = []; - for (const key of DISCOVERY_BEHAVIOURS) { - if (key === MEDICATION_COMPLIANCE_CHANNEL_KEY) { - series.push(complianceSeries); - } else if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "behaviour" }); - } else { - series.push({ key, role: "behaviour", points: points(key) }); - } - } - for (const key of DISCOVERY_OUTCOMES) { - if (key === SYMPTOM_SEVERITY_CHANNEL_KEY) { - series.push({ ...symptomSeries, role: "outcome" }); - } else { - series.push({ key, role: "outcome", points: points(key) }); - } - } - series.push(...environmentSeries, ...customMetricSeries); + // The channel set is the shared one — every family the correlations page + // scans, so a card can never be missing a pair the page would show. The + // measurement read stays `"raw"` (the route's rollup read-swap is a + // read-cost choice, not a channel-set one; see the option's doc comment). + const { series } = await assembleDiscoveryMatrix(userId, { + tz, + since, + fetchMode: "raw", + }); const result = discoverCorrelations(series, { locale }); const decisions = await syncAcceptedPatterns({ @@ -184,16 +118,26 @@ export async function getRelevantCorrelationsForMetric( qValue: pattern.qValue, })), }); - return result.discovered - .filter((d) => { - const decision = decisionForEvidence(decisions, { - factorKey: d.behaviour, - outcomeKey: d.outcome, - lagDays: d.lagDays, - }); - return decision?.dismissed !== true; - }) - .map((d) => ({ interpretation: d.interpretation, n: d.n, r: d.r })); + return ( + result.discovered + // The pair has to INVOLVE this metric. This filter is the difference + // between the function's name and a full dump of the matrix: without + // it, a WEIGHT card's prompt is handed "your daylight goes with your + // sleep" as grounded context for a paragraph about weight. It was here + // from the start and was lost when the pattern-dismissal filter was + // written in its place rather than after it; the docstring, the name + // and the `channelKeyForType` guard above all kept saying otherwise. + .filter((d) => d.behaviour === channel || d.outcome === channel) + .filter((d) => { + const decision = decisionForEvidence(decisions, { + factorKey: d.behaviour, + outcomeKey: d.outcome, + lagDays: d.lagDays, + }); + return decision?.dismissed !== true; + }) + .map((d) => ({ interpretation: d.interpretation, n: d.n, r: d.r })) + ); } catch { return []; } diff --git a/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts b/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts index 2415fde27..2b24fcada 100644 --- a/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts +++ b/src/lib/insights/narrative/__tests__/period-narrative-db.test.ts @@ -18,11 +18,31 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; const userFindUnique = vi.fn(); const measurementFindMany = vi.fn(); const moodFindMany = vi.fn(); +// The wrapper takes its channel set from the shared discovery +// assembler, so the four channel families that used to be missing here (the +// dose-history ledger, the illness day-log, the daily weather rows, the opt-in +// custom metrics) are read on this path too. Default every one empty: this +// file is about the measurement read's ORDER and GRAIN, and an empty channel +// degrades to absent without touching either. +const medicationFindMany = vi.fn(); +const intakeEventFindMany = vi.fn(); +const illnessEpisodeFindMany = vi.fn(); +const illnessDayLogFindMany = vi.fn(); +const environmentContextFindMany = vi.fn(); +const customMetricFindMany = vi.fn(); vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: (a: unknown) => userFindUnique(a) }, measurement: { findMany: (a: unknown) => measurementFindMany(a) }, moodEntry: { findMany: (a: unknown) => moodFindMany(a) }, + medication: { findMany: (a: unknown) => medicationFindMany(a) }, + medicationIntakeEvent: { findMany: (a: unknown) => intakeEventFindMany(a) }, + illnessEpisode: { findMany: (a: unknown) => illnessEpisodeFindMany(a) }, + illnessDayLog: { findMany: (a: unknown) => illnessDayLogFindMany(a) }, + environmentContext: { + findMany: (a: unknown) => environmentContextFindMany(a), + }, + customMetric: { findMany: (a: unknown) => customMetricFindMany(a) }, }, })); @@ -37,6 +57,12 @@ beforeEach(() => { }); measurementFindMany.mockReset(); moodFindMany.mockReset().mockResolvedValue([]); + medicationFindMany.mockReset().mockResolvedValue([]); + intakeEventFindMany.mockReset().mockResolvedValue([]); + illnessEpisodeFindMany.mockReset().mockResolvedValue([]); + illnessDayLogFindMany.mockReset().mockResolvedValue([]); + environmentContextFindMany.mockReset().mockResolvedValue([]); + customMetricFindMany.mockReset().mockResolvedValue([]); }); describe("buildPeriodNarrativeContext — grain (QA F3)", () => { @@ -135,5 +161,53 @@ describe("buildPeriodNarrativeContext — grain (QA F3)", () => { take: 20000, }), ); + // The mood read carries the same discipline, and it is the one that also + // pulls the RATED-factor tag-links this surface admits. + expect(moodFindMany).toHaveBeenCalledWith( + expect.objectContaining({ + orderBy: { moodLoggedAt: "desc" }, + take: 5000, + }), + ); + }); + + // The narrative used to fold in only the measurement channels, + // MOOD and the RATED mood factors: the compliance, symptom, environmental + // and custom-metric families had arrived at the correlations route and never + // reached this surface. Nothing explained the gap — the comment that looked + // like it did was the same sentence a sibling file carried while doing the + // opposite. It scans the shared matrix now. + it("reads every discovery channel family, not just the measurement ones", async () => { + const now = new Date("2026-06-30T12:00:00.000Z"); + measurementFindMany.mockResolvedValue([]); + + await buildPeriodNarrativeContext("u1", { + period: "month", + now, + locale: "en", + }); + + expect(medicationFindMany).toHaveBeenCalledTimes(1); + expect(illnessEpisodeFindMany).toHaveBeenCalledTimes(1); + expect(environmentContextFindMany).toHaveBeenCalledTimes(1); + expect(customMetricFindMany).toHaveBeenCalledTimes(1); + }); + + // The one declared divergence: this surface — and only this surface — admits + // the user's RATED mood factors. See `includeMoodFactors` for why. + it("pulls the RATED-factor tag-links the other three surfaces do not", async () => { + const now = new Date("2026-06-30T12:00:00.000Z"); + measurementFindMany.mockResolvedValue([]); + + await buildPeriodNarrativeContext("u1", { + period: "month", + now, + locale: "en", + }); + + const select = moodFindMany.mock.calls[0]?.[0]?.select as { + tagLinks?: { where?: { moodTag?: { kind?: string } } }; + }; + expect(select.tagLinks?.where?.moodTag?.kind).toBe("RATED"); }); }); diff --git a/src/lib/insights/narrative/period-narrative.ts b/src/lib/insights/narrative/period-narrative.ts index 2a160b63e..34a600fd1 100644 --- a/src/lib/insights/narrative/period-narrative.ts +++ b/src/lib/insights/narrative/period-narrative.ts @@ -12,7 +12,19 @@ * The honesty contract carries through verbatim from the layers it reuses: * - **Drivers** are ONLY the BH-FDR-surviving pairs from `discoverCorrelations` * (`benjaminiHochberg` already enforces descriptive-never-causal); each - * keeps its conservative `interpretation` string unchanged. + * keeps its conservative `interpretation` string unchanged. The channels + * scanned come from the one shared assembler in + * `src/lib/insights/discovery-matrix.ts`, so this surface sees the same + * families the correlations page and the Coach do. It differs in exactly + * one declared way — it admits the user's RATED mood factors as + * `FACTOR:` channels, which the three persisting surfaces do not; the + * reason is written at the `includeMoodFactors` option. + * + * It also silently lacked, until the assembler landed, the + * medication-compliance, symptom-severity, environmental-exposure and + * custom-metric families, which had arrived at the route and never + * here. Nothing explained that; the comment that appeared to was the + * same sentence a sibling file carried while doing the opposite. * - **Band transitions** are a personal-baseline (median ± k·MAD, Hampel/Leys) * comparison of the current period against the band established over the * PRIOR period — never an invented threshold. @@ -36,25 +48,14 @@ import { annotate } from "@/lib/logging/context"; import { wallClockInTz } from "@/lib/tz/wall-clock"; import { discoverCorrelations, - discoveryMeasurementTypes, - DISCOVERY_BEHAVIOURS, - DISCOVERY_OUTCOMES, - FACTOR_CHANNEL_PREFIX, type DailySeriesPoint, type NamedSeries, } from "@/lib/insights/correlation-discovery"; import { buildBaselineBand, median } from "@/lib/insights/derived/baseline"; import { VITALS_BASELINE_TYPES } from "@/lib/insights/derived/registry"; -import { - buildMeasurementDailySeries, - fetchMeasurementWindowSeries, - toDailyMeans, -} from "@/lib/insights/correlation-channel-series"; -import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; +import { assembleDiscoveryMatrix } from "@/lib/insights/discovery-matrix"; const MS_PER_DAY = 24 * 60 * 60 * 1000; -/** Cap on the mood-entry window read, mirroring the route / Coach tool. */ -const MOOD_READ_CAP = 5000; /** The two supported narrative periods and their length in days. */ export const PERIOD_DAYS = { week: 7, month: 30 } as const; @@ -191,61 +192,6 @@ function tzDayKey(at: Date, tz: string): string { return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`; } -/** - * A RATED mood factor link the period read pulls alongside the score. - * Carries the scale + `inverse` so the series build can apply the - * documented sign-flip once, at the boundary. - */ -interface FactorLink { - key: string; - rating: number; - scaleMin: number; - scaleMax: number; - inverse: boolean; - at: Date; -} - -/** - * v1.14.0 — collapse RATED-factor links to one inverse-flipped daily-mean - * series per factor, tz-day-keyed exactly like `toDailyMeans` so a factor - * channel joins the discovery matrix on the same day grid as every vital. - * An inverse factor's rating `r` maps to `(scaleMin + scaleMax) - r` BEFORE - * averaging so "up" always reads as a better day — the same flip the mood - * aggregates apply, kept in lock-step. Returns one `FACTOR:` series per - * factor the user actually rated. Pure. - */ -function factorDailyMeans( - links: FactorLink[], - tz: string, -): Map { - const byFactor = new Map< - string, - Map - >(); - for (const l of links) { - if (!Number.isFinite(l.rating)) continue; - const value = l.inverse ? l.scaleMin + l.scaleMax - l.rating : l.rating; - const day = tzDayKey(l.at, tz); - const days = - byFactor.get(l.key) ?? new Map(); - const acc = days.get(day) ?? { sum: 0, count: 0 }; - acc.sum += value; - acc.count += 1; - days.set(day, acc); - byFactor.set(l.key, days); - } - const out = new Map(); - for (const [key, days] of byFactor) { - out.set( - `${FACTOR_CHANNEL_PREFIX}${key}`, - [...days.entries()] - .map(([day, acc]) => ({ day, value: acc.sum / acc.count })) - .sort((a, b) => (a.day < b.day ? -1 : 1)), - ); - } - return out; -} - /** Round to 2 decimals. */ function round2(n: number): number { return Math.round(n * 100) / 100; @@ -508,8 +454,15 @@ export interface BuildPeriodNarrativeContextOpts { * Fetch + day-key + assemble. Reads a single bounded window covering the * current AND prior period (2× the period length plus one extra day so the * day-1 lag join in discovery has its source), day-keys in the user's tz, - * and delegates to the pure core. No LLM, no migration, no new heavy query — - * one measurement read + one mood read, both bounded. + * and delegates to the pure core. No LLM, no migration — every read is the + * shared assembler's, bounded, and the delta / band beats ride the same + * measurement query as the discovery channels. + * + * A note on what the `week` period can produce: its window is 15 days, and + * `discoverCorrelations` needs 20 lag-joined pairs before it will test one, so + * a weekly narrative's `drivers` list is empty by arithmetic, not by data. + * That predates this file's current shape and is left as it is; the deltas, + * band transitions and coincident flags are what a weekly narrative rests on. */ export async function buildPeriodNarrativeContext( userId: string, @@ -537,77 +490,40 @@ export async function buildPeriodNarrativeContext( tz, ); - // The full set of types any beat needs: delta metrics ∪ banded vitals ∪ - // discovery channels (minus the NON-measurement channels: MOOD is mood-entry - // backed; v1.21.0 MEDICATION_COMPLIANCE / SYMPTOM_SEVERITY are - // ledger / illness-day-log backed — none are MeasurementType enum values, so - // `discoveryMeasurementTypes` drops them from the `type IN (...)` query). - // Those two channels are not populated on the period-narrative surface (they - // degrade to absent here); the canonical `/api/insights/correlations` route - // builds them. - const measurementTypes = Array.from( - new Set([ - ...DELTA_METRICS.map((m) => m.type), - ...VITALS_BASELINE_TYPES, - ...discoveryMeasurementTypes(DISCOVERY_BEHAVIOURS), - ...discoveryMeasurementTypes(DISCOVERY_OUTCOMES), - ]), - ) as MeasurementType[]; - + // One read for both jobs. The discovery channels come from the shared + // assembler; `extraMeasurementTypes` widens the SAME measurement query with + // the types only this surface's other beats need (the period deltas and the + // banded vitals), so the delta beat costs no second round-trip. + // // v1.30.3 (QA F2/F3) — the fetch + desc/cap/resort discipline AND the // per-type grain resolution (source-collapse sum for cumulative types, - // per-night reconstruction for sleep — not a blind per-row MEAN) now - // route through the shared `fetchMeasurementWindowSeries` + - // `buildMeasurementDailySeries` (`correlation-channel-series.ts`), the - // same helpers the route / Coach tool / per-metric card share. Before - // this fix the local `toDailyMeans` twin folded EVERY type through a - // blind per-row mean: a month's current-vs-prior period comparison for - // `SLEEP_DURATION` averaged per-stage segment durations (~45 min) - // instead of summing a night's total time asleep, and `ACTIVITY_STEPS` - // blended per-sample chunk means with drained daily totals — and the - // `asc, take 20000` cap dropped the CURRENT period first on a dense - // account, the worst possible direction for a current-vs-prior surface. - const [{ byType, measurementsCapped }, moodEntriesDesc, priorityJson] = - await Promise.all([ - fetchMeasurementWindowSeries(userId, since, measurementTypes), - prisma.moodEntry.findMany({ - where: { userId, deletedAt: null, moodLoggedAt: { gte: since } }, - orderBy: { moodLoggedAt: "desc" }, - take: MOOD_READ_CAP, - select: { - score: true, - moodLoggedAt: true, - // v1.14.0 — pull RATED-factor links so each factor the user scores - // (work / sleep-quality / stress …) joins the discovery matrix as a - // `FACTOR:` channel. One extra select on the existing mood read - // — no new round-trip. BINARY links carry a null `rating` and are - // dropped below. - tagLinks: { - where: { moodTag: { kind: "RATED" }, rating: { not: null } }, - select: { - rating: true, - moodTag: { - select: { - key: true, - scaleMin: true, - scaleMax: true, - inverse: true, - }, - }, - }, - }, - }, - }), - loadUserSourcePriority(userId), - ]); - // This surface reads mood alongside its RATED-factor tag-links (a select - // the shared `fetchMoodWindowSeries` helper does not carry), so the - // desc+cap+resort discipline is applied in place here rather than - // through that helper — same discipline, bespoke select. - const moodCapped = moodEntriesDesc.length >= MOOD_READ_CAP; - const moodEntries = [...moodEntriesDesc].sort( - (a, b) => a.moodLoggedAt.getTime() - b.moodLoggedAt.getTime(), - ); + // per-night reconstruction for sleep — not a blind per-row MEAN) live in + // `correlation-channel-series.ts`, which the assembler reads through. Before + // that fix the local `toDailyMeans` twin folded EVERY type through a blind + // per-row mean: a month's current-vs-prior comparison for `SLEEP_DURATION` + // averaged per-stage segment durations (~45 min) instead of summing a + // night's total time asleep, and `ACTIVITY_STEPS` blended per-sample chunk + // means with drained daily totals — and the `asc, take 20000` cap dropped + // the CURRENT period first on a dense account, the worst possible direction + // for a current-vs-prior surface. + const { + series: discoverySeries, + byMetric: seriesByMetric, + diagnostics, + } = await assembleDiscoveryMatrix(userId, { + tz, + since, + fetchMode: "raw", + // The one surface that admits RATED mood factors — see the option's doc + // comment for why it is one and not four. + includeMoodFactors: true, + extraMeasurementTypes: Array.from( + new Set([ + ...DELTA_METRICS.map((m) => m.type), + ...VITALS_BASELINE_TYPES, + ]), + ), + }); // QA F2 — surfaces when a dense account's window exceeded the read cap, // mirroring the route's identical annotation. The cap now falls on the @@ -617,71 +533,12 @@ export async function buildPeriodNarrativeContext( action: { name: "insights.period-narrative.read" }, meta: { period, - measurements_capped: measurementsCapped, - mood_entries_capped: moodCapped, + measurements_capped: diagnostics.measurementsCapped, + mood_entries_capped: diagnostics.moodCapped, + mood_factor_channels: diagnostics.moodFactorChannels, }, }); - const seriesByMetric = new Map(); - for (const type of measurementTypes) { - const rows = byType.get(type); - if (!rows || rows.length === 0) continue; - seriesByMetric.set( - type, - buildMeasurementDailySeries(type, rows, tz, priorityJson), - ); - } - const moodPoints = toDailyMeans( - moodEntries.map((e) => ({ value: e.score, at: e.moodLoggedAt })), - tz, - ); - if (moodPoints.length > 0) seriesByMetric.set("MOOD", moodPoints); - - // v1.14.0 — RATED-factor channels. Flatten every entry's RATED links into - // one inverse-flipped daily-mean series per factor, keyed `FACTOR:`. - const factorLinks: FactorLink[] = []; - for (const e of moodEntries) { - for (const link of e.tagLinks) { - if (link.rating == null) continue; - factorLinks.push({ - key: link.moodTag.key, - rating: link.rating, - scaleMin: link.moodTag.scaleMin, - scaleMax: link.moodTag.scaleMax, - inverse: link.moodTag.inverse, - at: e.moodLoggedAt, - }); - } - } - const factorSeries = factorDailyMeans(factorLinks, tz); - for (const [key, points] of factorSeries) seriesByMetric.set(key, points); - - // Discovery matrix over the same window. - const discoverySeries: NamedSeries[] = []; - for (const key of DISCOVERY_BEHAVIOURS) { - discoverySeries.push({ - key, - role: "behaviour", - points: seriesByMetric.get(key) ?? [], - }); - } - for (const key of DISCOVERY_OUTCOMES) { - discoverySeries.push({ - key, - role: "outcome", - points: seriesByMetric.get(key) ?? [], - }); - } - // A factor is plausibly both a lag source ("rated work today → next-day - // sleep") and a lag target ("steps today → next-day rated energy"), so it - // enters as both roles. The engine skips self-pairs and BH-FDR-controls the - // wider family this opens. Factors the user rated on < the paired-day floor - // simply never get tested — no new statistical machinery, same honesty. - for (const [key, points] of factorSeries) { - discoverySeries.push({ key, role: "behaviour", points }); - discoverySeries.push({ key, role: "outcome", points }); - } - return assemblePeriodNarrativeContext({ period, currentFrom,