diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 0005a232d..0e1b9250f 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -18631,6 +18631,14 @@ components: minimum: -9007199254740991 maximum: 9007199254740991 description: Minimum paired-day count enforced per pair. + emerging: + description: v1.22 — rolling early-detection pass over the trailing window; emerging pairs not yet established + retrospectively (no double-count). + $ref: "#/components/schemas/EmergingCorrelationResult" + labCorrelations: + description: v1.22 — labs ↔ outcome associations (each draw vs the contemporaneous outcome window-mean), FDR-controlled; + absent-degrading on sparse draws. + $ref: "#/components/schemas/LabCorrelationResult" required: - discovered - pairsTested @@ -18670,6 +18678,16 @@ components: minimum: -9007199254740991 maximum: 9007199254740991 description: Lag in days applied (1). + window: + description: v1.22 — which window surfaced the pair. Absent on the 180-day scan (retrospective default); `recent` for an + emerging early-detection pair. + type: string + enum: + - retrospective + - recent + provisional: + description: "v1.22 — true for an emerging recent-window pair: fewer days, hedged as provisional rather than established." + type: boolean required: - behaviour - outcome @@ -18680,6 +18698,107 @@ components: - interpretation - lagDays additionalProperties: false + EmergingCorrelationResult: + type: object + properties: + emerging: + type: array + items: + $ref: "#/components/schemas/DiscoveredCorrelation" + description: Recent-window pairs NOT already established retrospectively — the emerging signals (provisional, hedged). + windowDays: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Trailing window (days) the early pass scanned. + minPairs: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Paired-day floor enforced for the early pass. + fdrQ: + type: number + description: FDR target the early pass used (tighter than the main scan). + pairsTested: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Pairs tested in the early window. + required: + - emerging + - windowDays + - minPairs + - fdrQ + - pairsTested + additionalProperties: false + LabCorrelationResult: + type: object + properties: + discovered: + type: array + items: + $ref: "#/components/schemas/DiscoveredLabCorrelation" + description: Lab ↔ outcome associations surviving the per-pair floor + BH-FDR. + pairsTested: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Lab × outcome pairs assessed. + fdrQ: + type: number + description: The FDR target the pass used. + minDraws: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Minimum paired-draw count enforced per pair. + required: + - discovered + - pairsTested + - fdrQ + - minDraws + additionalProperties: false + DiscoveredLabCorrelation: + type: object + properties: + lab: + type: string + description: "`LAB:` channel key (display strips the prefix)." + outcome: + type: string + description: Outcome channel the marker tracks with (WEIGHT, BLOOD_GLUCOSE, BLOOD_PRESSURE_SYS). + n: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Draws paired with a usable contemporaneous outcome window. + r: + type: number + description: Pearson r over (draw value, contemporaneous outcome window-mean). + pValue: + type: number + description: Two-sided exact Student-t p-value (< 0.05). + qValue: + type: number + description: Benjamini-Hochberg FDR-adjusted q-value (≤ the surface threshold). + windowDays: + type: integer + minimum: -9007199254740991 + maximum: 9007199254740991 + description: Trailing days each draw's outcome window spanned. + interpretation: + type: string + description: Conservative, descriptive interpretation — never causal. + required: + - lab + - outcome + - n + - r + - pValue + - qValue + - windowDays + - interpretation + additionalProperties: false MoodTagTreeEnvelope: type: object properties: diff --git a/src/app/api/insights/correlations/__tests__/route.test.ts b/src/app/api/insights/correlations/__tests__/route.test.ts index ea5d940a5..a4d8b711f 100644 --- a/src/app/api/insights/correlations/__tests__/route.test.ts +++ b/src/app/api/insights/correlations/__tests__/route.test.ts @@ -18,6 +18,8 @@ vi.mock("@/lib/db", () => ({ medicationIntakeEvent: { findMany: vi.fn().mockResolvedValue([]) }, illnessEpisode: { findMany: vi.fn().mockResolvedValue([]) }, illnessDayLog: { findMany: vi.fn().mockResolvedValue([]) }, + // v1.22 — labs ↔ outcome pass fetches lab draws; default empty. + labResult: { findMany: vi.fn().mockResolvedValue([]) }, }, })); @@ -109,6 +111,8 @@ beforeEach(() => { (prisma.illnessDayLog.findMany as ReturnType).mockResolvedValue( [], ); + // v1.22 — labs ↔ outcome pass reads lab draws; default empty. + (prisma.labResult.findMany as ReturnType).mockResolvedValue([]); }); const callGet = GET as unknown as (req: NextRequest) => Promise; diff --git a/src/app/api/insights/correlations/route.ts b/src/app/api/insights/correlations/route.ts index e9b511687..33cf3329a 100644 --- a/src/app/api/insights/correlations/route.ts +++ b/src/app/api/insights/correlations/route.ts @@ -26,9 +26,12 @@ import { wallClockInTz } from "@/lib/tz/wall-clock"; import type { MeasurementType } from "@/generated/prisma/client"; import { discoverCorrelations, + discoverEmergingCorrelations, + discoverLabOutcomeCorrelations, discoveryMeasurementTypes, DISCOVERY_BEHAVIOURS, DISCOVERY_OUTCOMES, + EARLY_WINDOW_DAYS, MEDICATION_COMPLIANCE_CHANNEL_KEY, SYMPTOM_SEVERITY_CHANNEL_KEY, type DailySeriesPoint, @@ -36,6 +39,7 @@ import { } from "@/lib/insights/correlation-discovery"; import { fetchComplianceSeries, + fetchLabDraws, fetchSymptomSeries, } from "@/lib/insights/correlation-channel-series"; @@ -147,8 +151,12 @@ export const GET = apiHandler(async () => { // v1.21.0 (FDREXTEND) — build the two non-measurement, non-mood channels from // their own sources. Each degrades to an empty series when the user has no // data, so the discovery loop drops the channel (it cannot clear n ≥ 20). - const complianceSeries = await fetchComplianceSeries(user.id, tz, since); - const symptomSeries = await fetchSymptomSeries(user.id, tz, since); + // v1.22 — lab draws (for the labs ↔ outcome pass) fetch alongside. + const [complianceSeries, symptomSeries, labDraws] = await Promise.all([ + fetchComplianceSeries(user.id, tz, since), + fetchSymptomSeries(user.id, tz, since), + fetchLabDraws(user.id, tz, since), + ]); const points = (key: string): DailySeriesPoint[] => key === "MOOD" @@ -175,6 +183,21 @@ export const GET = apiHandler(async () => { const result = discoverCorrelations(series); + // v1.22 — rolling early-detection pass over the trailing window, re-using + // the already-built series (no extra DB read). Emerging pairs exclude anything + // the retrospective scan already established (no double-count). + const recentFromDayKey = tzDayKey( + new Date(now.getTime() - EARLY_WINDOW_DAYS * MS_PER_DAY), + tz, + ); + const emerging = discoverEmergingCorrelations(series, result, { + recentFromDayKey, + }); + + // v1.22 — labs ↔ outcome pass (point-vs-window over sparse draws). Degrades + // to absent when the user has too few draws to clear the per-pair floor. + const labCorrelations = discoverLabOutcomeCorrelations(labDraws, series); + annotate({ action: { name: "insights.correlations.discover" }, meta: { @@ -185,8 +208,13 @@ export const GET = apiHandler(async () => { // two sparse new channels reached the n ≥ 20 floor or degraded to absent. compliance_days: complianceSeries.points.length, symptom_days: symptomSeries.points.length, + // v1.22 — early-detection + labs reach. + emerging: emerging.emerging.length, + emerging_window_days: emerging.windowDays, + lab_draws: labDraws.length, + lab_correlations: labCorrelations.discovered.length, }, }); - return apiSuccess(result); + return apiSuccess({ ...result, emerging, labCorrelations }); }); diff --git a/src/lib/ai/coach/__tests__/correlations-read.test.ts b/src/lib/ai/coach/__tests__/correlations-read.test.ts index fcda4fbed..2ab4f2cb6 100644 --- a/src/lib/ai/coach/__tests__/correlations-read.test.ts +++ b/src/lib/ai/coach/__tests__/correlations-read.test.ts @@ -12,6 +12,7 @@ const medicationFindMany = vi.fn(); const intakeEventFindMany = vi.fn(); const illnessEpisodeFindMany = vi.fn(); const illnessDayLogFindMany = vi.fn(); +const labResultFindMany = vi.fn(); vi.mock("@/lib/db", () => ({ prisma: { measurement: { findMany: (a: unknown) => measurementFindMany(a) }, @@ -21,6 +22,7 @@ vi.mock("@/lib/db", () => ({ medicationIntakeEvent: { findMany: (a: unknown) => intakeEventFindMany(a) }, illnessEpisode: { findMany: (a: unknown) => illnessEpisodeFindMany(a) }, illnessDayLog: { findMany: (a: unknown) => illnessDayLogFindMany(a) }, + labResult: { findMany: (a: unknown) => labResultFindMany(a) }, }, })); @@ -59,6 +61,7 @@ describe("readCoachCorrelations", () => { intakeEventFindMany.mockReset().mockResolvedValue([]); illnessEpisodeFindMany.mockReset().mockResolvedValue([]); illnessDayLogFindMany.mockReset().mockResolvedValue([]); + labResultFindMany.mockReset().mockResolvedValue([]); loadBaselineProfile .mockReset() .mockResolvedValue({ ageYears: 40, sex: "MALE", heightCm: 180 }); diff --git a/src/lib/ai/coach/tools/correlations-read.ts b/src/lib/ai/coach/tools/correlations-read.ts index 997360678..ebeb3e2d5 100644 --- a/src/lib/ai/coach/tools/correlations-read.ts +++ b/src/lib/ai/coach/tools/correlations-read.ts @@ -27,9 +27,12 @@ import { prisma } from "@/lib/db"; import { wallClockInTz } from "@/lib/tz/wall-clock"; import { discoverCorrelations, + discoverEmergingCorrelations, + discoverLabOutcomeCorrelations, discoveryMeasurementTypes, DISCOVERY_BEHAVIOURS, DISCOVERY_OUTCOMES, + EARLY_WINDOW_DAYS, MEDICATION_COMPLIANCE_CHANNEL_KEY, SYMPTOM_SEVERITY_CHANNEL_KEY, type DailySeriesPoint, @@ -37,6 +40,7 @@ import { } from "@/lib/insights/correlation-discovery"; import { fetchComplianceSeries, + fetchLabDraws, fetchSymptomSeries, } from "@/lib/insights/correlation-channel-series"; import { @@ -77,9 +81,38 @@ export interface CoachCoincidentFlag { illnessExplained: boolean; } +/** One emerging (recent-window) driver — provisional, hedged. */ +export interface CoachEmergingDriver extends CoachCorrelationDriver { + /** Always true here — a recent-window signal on fewer days. */ + provisional: true; +} + +/** One labs ↔ outcome association — descriptive, never causal. */ +export interface CoachLabCorrelation { + /** Display analyte name (LAB: prefix stripped). */ + lab: string; + /** The outcome it tracks with. */ + outcome: string; + direction: "higher" | "lower"; + /** Paired draws. */ + n: number; + r: number; + note: string; +} + export interface CoachCorrelationsResult { present: boolean; drivers?: CoachCorrelationDriver[]; + /** + * v1.22 — emerging recent-window drivers NOT yet established by the 180-day + * scan: early-detection signals the Coach narrates as provisional. + */ + emerging?: CoachEmergingDriver[]; + /** + * v1.22 — labs ↔ outcome associations (each draw vs the contemporaneous + * outcome window-mean), FDR-controlled. Descriptive, never causal. + */ + labDrivers?: CoachLabCorrelation[]; coincident?: CoachCoincidentFlag; /** How many behaviour×outcome pairs were tested (honest footer). */ pairsTested?: number; @@ -194,8 +227,12 @@ export async function readCoachCorrelations( // like the route. Each degrades to an empty series when the user has no // data, so the discovery loop drops the channel (it cannot clear the n ≥ 20 // floor) — it can never surface a fabricated driver. - const complianceSeries = await fetchComplianceSeries(userId, tz, since); - const symptomSeries = await fetchSymptomSeries(userId, tz, since); + const [complianceSeries, symptomSeries, labDraws] = await Promise.all([ + fetchComplianceSeries(userId, tz, since), + fetchSymptomSeries(userId, tz, since), + // v1.22 — lab draws for the labs ↔ outcome pass (degrades to absent). + fetchLabDraws(userId, tz, since), + ]); const series: NamedSeries[] = []; for (const key of DISCOVERY_BEHAVIOURS) { @@ -230,17 +267,59 @@ export async function readCoachCorrelations( note: d.interpretation, })); + // v1.22 — rolling early-detection pass over the trailing window (re-uses + // the already-built series). Emerging pairs exclude anything the 180-day + // scan already established, so the Coach never narrates the same pattern as + // both "established" and "emerging". + const recentFromDayKey = tzDayKey( + new Date(Date.now() - EARLY_WINDOW_DAYS * MS_PER_DAY), + tz, + ); + const emergingResult = discoverEmergingCorrelations(series, discovery, { + recentFromDayKey, + }); + const emerging: CoachEmergingDriver[] = emergingResult.emerging.map( + (d) => ({ + behaviour: humanise(d.behaviour), + outcome: humanise(d.outcome), + direction: d.r >= 0 ? "higher" : "lower", + lagDays: d.lagDays, + n: d.n, + r: Math.round(d.r * 100) / 100, + note: d.interpretation, + provisional: true, + }), + ); + + // v1.22 — labs ↔ outcome pass (point-vs-window over sparse draws). + const labResult = discoverLabOutcomeCorrelations(labDraws, series); + const labDrivers: CoachLabCorrelation[] = labResult.discovered.map((d) => ({ + lab: d.lab.startsWith("LAB:") ? d.lab.slice("LAB:".length) : d.lab, + outcome: humanise(d.outcome), + direction: d.r >= 0 ? "higher" : "lower", + n: d.n, + r: Math.round(d.r * 100) / 100, + note: d.interpretation, + })); + const coincident = buildCoincidentFlag(coincidentDerived); - // Nothing to say: no surviving driver AND the coincident flag is either - // insufficient or quiet (not fired). Report a clean miss. - if (drivers.length === 0 && (!coincident || !coincident.fired)) { + // Nothing to say: no surviving driver of any kind AND the coincident flag is + // either insufficient or quiet (not fired). Report a clean miss. + if ( + drivers.length === 0 && + emerging.length === 0 && + labDrivers.length === 0 && + (!coincident || !coincident.fired) + ) { return { present: false, reason: "no_significant_pattern" }; } return { present: true, ...(drivers.length > 0 ? { drivers } : {}), + ...(emerging.length > 0 ? { emerging } : {}), + ...(labDrivers.length > 0 ? { labDrivers } : {}), ...(coincident ? { coincident } : {}), pairsTested: discovery.pairsTested, windowDays: WINDOW_DAYS, diff --git a/src/lib/insights/__tests__/correlation-discovery.test.ts b/src/lib/insights/__tests__/correlation-discovery.test.ts index 8eb2bff08..02c7c9060 100644 --- a/src/lib/insights/__tests__/correlation-discovery.test.ts +++ b/src/lib/insights/__tests__/correlation-discovery.test.ts @@ -529,3 +529,253 @@ describe("compliance + symptom channels (FDREXTEND)", () => { expect(pair!.r).toBeLessThan(0); // higher adherence → lower next-day RHR }); }); + +// ─── v1.22 — new linkages + early detection + labs ──────────────────────── + +import { + DISCOVERY_BEHAVIOURS as MATRIX_BEHAVIOURS, + DISCOVERY_OUTCOMES as MATRIX_OUTCOMES, + discoverEmergingCorrelations, + discoverLabOutcomeCorrelations, + filterSeriesToWindow, + EARLY_WINDOW_DAYS, + type LabDrawPoint, +} from "../correlation-discovery"; + +/** Contiguous daily series starting `startISO` (YYYY-MM-DD). */ +function seriesFrom(startISO: string, values: number[]): DailySeriesPoint[] { + const [y, m, d] = startISO.split("-").map(Number); + const start = Date.UTC(y, m - 1, d); + return values.map((value, i) => { + const dt = new Date(start + i * 24 * 60 * 60 * 1000); + return { day: dt.toISOString().slice(0, 10), value }; + }); +} + +describe("v1.22 — new curated linkages", () => { + it("adds sleep as a behaviour (was outcome-only)", () => { + expect(MATRIX_BEHAVIOURS).toContain("SLEEP_DURATION"); + }); + + it("adds blood pressure (both components) as outcomes", () => { + expect(MATRIX_OUTCOMES).toContain("BLOOD_PRESSURE_SYS"); + expect(MATRIX_OUTCOMES).toContain("BLOOD_PRESSURE_DIA"); + }); + + it("surfaces compliance→BP_SYS but skips the same-family BP→BP self-cross", () => { + const n = 80; + const adherence = Array.from({ length: n }, (_, i) => + i % 2 === 0 ? 100 : 50, + ); + // Higher adherence today → lower next-day systolic. + const sys = [ + 130, + ...adherence + .slice(0, n - 1) + .map((a, i) => 160 - a / 4 + (i % 2 ? 0.2 : -0.2)), + ]; + const result = discoverCorrelations([ + { + key: MEDICATION_COMPLIANCE_CHANNEL_KEY, + role: "behaviour", + points: longSeries(adherence), + }, + // BP is both a behaviour and an outcome in the real matrix; the + // same-family guard must keep BP_SYS(behaviour)→BP_SYS(outcome) out. + { key: "BLOOD_PRESSURE_SYS", role: "behaviour", points: longSeries(sys) }, + { key: "BLOOD_PRESSURE_SYS", role: "outcome", points: longSeries(sys) }, + ]); + expect( + result.discovered.find( + (p) => + p.behaviour === "BLOOD_PRESSURE_SYS" && + p.outcome === "BLOOD_PRESSURE_SYS", + ), + ).toBeUndefined(); + const link = result.discovered.find( + (p) => + p.behaviour === MEDICATION_COMPLIANCE_CHANNEL_KEY && + p.outcome === "BLOOD_PRESSURE_SYS", + ); + expect(link).toBeDefined(); + expect(link!.r).toBeLessThan(0); + }); +}); + +describe("discoverEmergingCorrelations (early detection)", () => { + it("filterSeriesToWindow keeps only on/after the cutoff", () => { + const s: NamedSeries[] = [ + { + key: "MOOD", + role: "behaviour", + points: seriesFrom("2026-03-01", [1, 2, 3, 4, 5]), + }, + ]; + const filtered = filterSeriesToWindow(s, "2026-03-03"); + expect(filtered[0].points.map((p) => p.day)).toEqual([ + "2026-03-03", + "2026-03-04", + "2026-03-05", + ]); + }); + + it("surfaces a fortnight-long signal the retrospective n≥20 floor cannot yet see", () => { + // Only ~18 days of data: a clean behaviour→next-day-outcome coupling. The + // retrospective scan needs ≥ 20 lagged pairs, so it sees NOTHING; the early + // window (floor 10) catches it — exactly the early-detection mandate. + const len = 18; + const beh = Array.from({ length: len }, (_, i) => (i % 2 ? 2 : 9)); + const out = beh.map((b, i) => 40 + b * 3 + (i % 2 ? 0.3 : -0.3)); + const series: NamedSeries[] = [ + { key: "MOOD", role: "behaviour", points: longSeries(beh) }, + { + key: "HEART_RATE_VARIABILITY", + role: "outcome", + points: longSeries(out), + }, + ]; + const retrospective = discoverCorrelations(series); + expect(retrospective.discovered).toHaveLength(0); // n too low for the floor + + const cutoff = longSeries(beh)[0].day; // whole (short) window + const { emerging } = discoverEmergingCorrelations(series, retrospective, { + recentFromDayKey: cutoff, + minPairs: 10, + }); + const pair = emerging.find( + (p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY", + ); + expect(pair).toBeDefined(); + expect(pair!.window).toBe("recent"); + expect(pair!.provisional).toBe(true); + }); + + it("does not double-count a pair already established retrospectively", () => { + // A strong relationship across the WHOLE 80-day window → retrospective finds + // it; the emerging pass must NOT re-surface it. + const n = 80; + const beh = Array.from({ length: n }, (_, i) => (i % 2 ? 2 : 9)); + const out = beh.map((b, i) => 40 + b * 3 + (i % 2 ? 0.3 : -0.3)); + const series: NamedSeries[] = [ + { key: "MOOD", role: "behaviour", points: longSeries(beh) }, + { + key: "HEART_RATE_VARIABILITY", + role: "outcome", + points: longSeries(out), + }, + ]; + const retrospective = discoverCorrelations(series); + expect( + retrospective.discovered.some( + (p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY", + ), + ).toBe(true); + const cutoff = new Date( + Date.UTC(2026, 0, 1) + (n - EARLY_WINDOW_DAYS) * 24 * 60 * 60 * 1000, + ) + .toISOString() + .slice(0, 10); + const { emerging } = discoverEmergingCorrelations(series, retrospective, { + recentFromDayKey: cutoff, + minPairs: 10, + }); + expect( + emerging.find( + (p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY", + ), + ).toBeUndefined(); + }); + + it("returns nothing on pure noise (FDR + high bar hold in the short window)", () => { + const n = 30; + const beh = Array.from({ length: n }, (_, i) => (i * 7) % 11); + const out = Array.from({ length: n }, (_, i) => (i * 3) % 5); + const series: NamedSeries[] = [ + { key: "MOOD", role: "behaviour", points: longSeries(beh) }, + { + key: "HEART_RATE_VARIABILITY", + role: "outcome", + points: longSeries(out), + }, + ]; + const retrospective = discoverCorrelations(series); + const cutoff = longSeries(beh)[Math.max(0, n - EARLY_WINDOW_DAYS)].day; + const { emerging } = discoverEmergingCorrelations(series, retrospective, { + recentFromDayKey: cutoff, + minPairs: 10, + }); + expect(emerging).toHaveLength(0); + }); +}); + +describe("discoverLabOutcomeCorrelations (labs ↔ outcomes)", () => { + function weightSeries(): NamedSeries { + // Daily weight that drifts up over ~6 months. + const values = Array.from({ length: 180 }, (_, i) => 80 + i * 0.05); + return { + key: "WEIGHT", + role: "outcome", + points: seriesFrom("2026-01-01", values), + }; + } + + it("surfaces a lab that tracks the contemporaneous outcome window-mean", () => { + const weight = weightSeries(); + // 8 monthly-ish HbA1c-like draws that rise with weight. + const draws: LabDrawPoint[] = []; + for (let i = 0; i < 8; i++) { + const dayIdx = 30 + i * 18; // spread draws across the window + const day = weight.points[dayIdx].day; + // Value tracks the period's weight mean (with tiny jitter). + const v = 5.0 + dayIdx * 0.01 + (i % 2 ? 0.02 : -0.02); + draws.push({ key: "LAB:HbA1c", day, value: v }); + } + const result = discoverLabOutcomeCorrelations(draws, [weight], { + minDraws: 5, + minWindowPoints: 5, + }); + const pair = result.discovered.find( + (d) => d.lab === "LAB:HbA1c" && d.outcome === "WEIGHT", + ); + expect(pair).toBeDefined(); + expect(pair!.n).toBeGreaterThanOrEqual(5); + expect(pair!.r).toBeGreaterThan(0); + expect(pair!.qValue).toBeLessThanOrEqual(result.fdrQ); + expect(pair!.interpretation).toMatch(/never a cause/i); + }); + + it("degrades to absent when the marker has too few draws", () => { + const weight = weightSeries(); + const draws: LabDrawPoint[] = [0, 1, 2].map((i) => ({ + key: "LAB:HbA1c", + day: weight.points[30 + i * 18].day, + value: 5 + i, + })); + const result = discoverLabOutcomeCorrelations(draws, [weight], { + minDraws: 5, + }); + expect(result.discovered).toHaveLength(0); + expect(result.pairsTested).toBe(0); + }); + + it("ignores outcomes outside the curated lab-target set", () => { + const hrv: NamedSeries = { + key: "HEART_RATE_VARIABILITY", + role: "outcome", + points: seriesFrom( + "2026-01-01", + Array.from({ length: 180 }, (_, i) => 50 + i * 0.1), + ), + }; + const draws: LabDrawPoint[] = Array.from({ length: 8 }, (_, i) => ({ + key: "LAB:HbA1c", + day: hrv.points[30 + i * 18].day, + value: 5 + i, + })); + const result = discoverLabOutcomeCorrelations(draws, [hrv], { + minDraws: 5, + }); + // HRV is not a curated lab-outcome target → no pair tested. + expect(result.pairsTested).toBe(0); + }); +}); diff --git a/src/lib/insights/__tests__/features.test.ts b/src/lib/insights/__tests__/features.test.ts index 1572b4b3e..8104af687 100644 --- a/src/lib/insights/__tests__/features.test.ts +++ b/src/lib/insights/__tests__/features.test.ts @@ -3,12 +3,16 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; vi.mock("@/lib/db", () => ({ prisma: { user: { findUnique: vi.fn() }, - measurement: { findMany: vi.fn() }, + measurement: { findMany: vi.fn(), groupBy: vi.fn() }, measurementRollup: { findMany: vi.fn() }, moodEntry: { findMany: vi.fn(), findFirst: vi.fn() }, moodEntryRollup: { findMany: vi.fn(), findFirst: vi.fn() }, medication: { findMany: vi.fn() }, medicationIntakeEvent: { findMany: vi.fn() }, + // v1.22 — cross-signal integration blocks. + labResult: { findMany: vi.fn() }, + measurementReminder: { findMany: vi.fn() }, + workout: { findMany: vi.fn() }, $queryRawUnsafe: vi.fn(), }, })); @@ -41,7 +45,10 @@ import { const prismaMock = prisma as unknown as { user: { findUnique: ReturnType }; - measurement: { findMany: ReturnType }; + measurement: { + findMany: ReturnType; + groupBy: ReturnType; + }; measurementRollup: { findMany: ReturnType }; moodEntry: { findMany: ReturnType; @@ -53,6 +60,9 @@ const prismaMock = prisma as unknown as { }; medication: { findMany: ReturnType }; medicationIntakeEvent: { findMany: ReturnType }; + labResult: { findMany: ReturnType }; + measurementReminder: { findMany: ReturnType }; + workout: { findMany: ReturnType }; }; const dayMs = 24 * 60 * 60 * 1000; @@ -79,6 +89,7 @@ beforeEach(() => { gender: "MALE", }); prismaMock.measurement.findMany.mockResolvedValue([]); + prismaMock.measurement.groupBy.mockResolvedValue([]); prismaMock.measurementRollup.findMany.mockResolvedValue([]); prismaMock.moodEntry.findMany.mockResolvedValue([]); prismaMock.moodEntry.findFirst.mockResolvedValue(null); @@ -86,6 +97,9 @@ beforeEach(() => { prismaMock.moodEntryRollup.findFirst.mockResolvedValue(null); prismaMock.medication.findMany.mockResolvedValue([]); prismaMock.medicationIntakeEvent.findMany.mockResolvedValue([]); + prismaMock.labResult.findMany.mockResolvedValue([]); + prismaMock.measurementReminder.findMany.mockResolvedValue([]); + prismaMock.workout.findMany.mockResolvedValue([]); }); describe("extractFeatures — v1.4.36 W3 bucketed payload", () => { @@ -187,3 +201,199 @@ describe("extractFeatures — v1.4.36 W3 bucketed payload", () => { expect(FEATURES_MAX_BYTES).toBe(5 * 1024 * 1024); }); }); + +// ─── v1.22 — cross-signal integration blocks ────────────────────────────── + +const DAY_MS = 24 * 60 * 60 * 1000; + +function measurementRow(type: string, value: number, daysAgo: number) { + return { + type, + value, + measuredAt: new Date(Date.now() - daysAgo * DAY_MS), + sleepStage: null, + source: "MANUAL", + deviceType: null, + }; +} + +function labRow(opts: { + analyte: string; + value: number | null; + referenceLow: number | null; + referenceHigh: number | null; + daysAgo: number; + hidden?: boolean; + valueText?: string | null; +}) { + return { + analyte: opts.analyte, + panel: null, + unit: "mg/dL", + value: opts.value, + valueText: opts.valueText ?? null, + referenceLow: opts.referenceLow, + referenceHigh: opts.referenceHigh, + takenAt: new Date(Date.now() - opts.daysAgo * DAY_MS), + biomarkerId: opts.hidden === undefined ? null : "bm-" + opts.analyte, + biomarker: + opts.hidden === undefined + ? null + : { + id: "bm-" + opts.analyte, + name: opts.analyte, + unit: "mg/dL", + lowerBound: opts.referenceLow, + upperBound: opts.referenceHigh, + panel: null, + hidden: opts.hidden, + }, + }; +} + +describe("extractFeatures — v1.22 glucose aggregate block", () => { + it("emits a glucose block from BLOOD_GLUCOSE readings", async () => { + prismaMock.measurement.findMany.mockResolvedValue([ + measurementRow("BLOOD_GLUCOSE", 95, 1), + measurementRow("BLOOD_GLUCOSE", 110, 10), + measurementRow("BLOOD_GLUCOSE", 105, 40), + ]); + const f = await extractFeatures("user-1", false); + expect(f.glucose).toBeDefined(); + expect(f.glucose?.latest).not.toBeNull(); + expect(f.glucose?.coverage.count).toBe(3); + }); + + it("omits the glucose block when there are no readings", async () => { + const f = await extractFeatures("user-1", false); + expect(f.glucose).toBeUndefined(); + }); +}); + +describe("extractFeatures — v1.22 labs briefing block", () => { + it("surfaces an abnormal (out-of-range) biomarker", async () => { + prismaMock.labResult.findMany.mockResolvedValue([ + labRow({ + analyte: "LDL", + value: 200, + referenceLow: 0, + referenceHigh: 130, + daysAgo: 5, + }), + ]); + const f = await extractFeatures("user-1", false); + expect(f.labs).toBeDefined(); + expect(f.labs?.flagged[0]?.analyte).toBe("LDL"); + expect(f.labs?.flagged[0]?.rangeStatus).toBe("above"); + }); + + it("excludes a HIDDEN biomarker even when it is abnormal", async () => { + prismaMock.labResult.findMany.mockResolvedValue([ + labRow({ + analyte: "Ferritin", + value: 500, + referenceLow: 30, + referenceHigh: 300, + daysAgo: 3, + hidden: true, + }), + ]); + const f = await extractFeatures("user-1", false); + // The only marker is hidden + abnormal → block omitted entirely. + expect(f.labs).toBeUndefined(); + }); + + it("does not surface an in-range, non-trending marker", async () => { + prismaMock.labResult.findMany.mockResolvedValue([ + labRow({ + analyte: "TSH", + value: 2, + referenceLow: 0.4, + referenceHigh: 4, + daysAgo: 4, + }), + ]); + const f = await extractFeatures("user-1", false); + expect(f.labs).toBeUndefined(); + }); + + it("surfaces an in-range but trending marker (rising vs prior reading)", async () => { + prismaMock.labResult.findMany.mockResolvedValue([ + // Newest first (route orders desc); both in range, but rising. + labRow({ + analyte: "Glucose", + value: 99, + referenceLow: 70, + referenceHigh: 100, + daysAgo: 2, + }), + labRow({ + analyte: "Glucose", + value: 80, + referenceLow: 70, + referenceHigh: 100, + daysAgo: 60, + }), + ]); + const f = await extractFeatures("user-1", false); + expect(f.labs?.flagged[0]?.trend).toBe("rising"); + }); +}); + +describe("extractFeatures — v1.22 preventive-care block", () => { + it("buckets reminders into overdue and due", async () => { + prismaMock.measurementReminder.findMany.mockResolvedValue([ + { label: "Blutbild", nextDueAt: new Date(Date.now() - 5 * DAY_MS) }, + { label: "Augenarzt", nextDueAt: new Date(Date.now() + 7 * DAY_MS) }, + ]); + const f = await extractFeatures("user-1", false); + expect(f.preventiveCare?.overdue[0]?.label).toBe("Blutbild"); + expect(f.preventiveCare?.overdue[0]?.daysOverdue).toBeGreaterThanOrEqual(4); + expect(f.preventiveCare?.due[0]?.label).toBe("Augenarzt"); + }); + + it("omits the block when nothing is due or overdue", async () => { + const f = await extractFeatures("user-1", false); + expect(f.preventiveCare).toBeUndefined(); + }); +}); + +describe("extractFeatures — v1.22 workouts block", () => { + it("aggregates counts + distance over the windows", async () => { + prismaMock.workout.findMany.mockResolvedValue([ + { + sportType: "running", + startedAt: new Date(Date.now() - 1 * DAY_MS), + durationSec: 1800, + totalDistanceM: 5000, + }, + { + sportType: "cycling", + startedAt: new Date(Date.now() - 20 * DAY_MS), + durationSec: 3600, + totalDistanceM: 20000, + }, + ]); + const f = await extractFeatures("user-1", false); + expect(f.workouts?.last7.count).toBe(1); + expect(f.workouts?.last30.count).toBe(2); + expect(f.workouts?.last7.totalDistanceKm).toBe(5); + expect(f.workouts?.latest?.sportType).toBe("running"); + }); +}); + +describe("extractFeatures — v1.22 integration blocks are briefing-scoped", () => { + it("skips the extra reads on the tight Coach-snapshot window (sinceDays: 90)", async () => { + await extractFeatures("user-1", false, { sinceDays: 90 }); + expect(prismaMock.labResult.findMany).not.toHaveBeenCalled(); + expect(prismaMock.measurementReminder.findMany).not.toHaveBeenCalled(); + expect(prismaMock.workout.findMany).not.toHaveBeenCalled(); + }); + + it("runs the extra reads on the wide briefing window (sinceDays: 400)", async () => { + await extractFeatures("user-1", false, { sinceDays: 400 }); + expect(prismaMock.labResult.findMany).toHaveBeenCalled(); + expect(prismaMock.measurementReminder.findMany).toHaveBeenCalled(); + expect(prismaMock.workout.findMany).toHaveBeenCalled(); + }); +}); diff --git a/src/lib/insights/correlation-channel-series.ts b/src/lib/insights/correlation-channel-series.ts index 7d2c65783..2911716a4 100644 --- a/src/lib/insights/correlation-channel-series.ts +++ b/src/lib/insights/correlation-channel-series.ts @@ -24,8 +24,11 @@ import { import { MEDICATION_COMPLIANCE_CHANNEL_KEY, SYMPTOM_SEVERITY_CHANNEL_KEY, + type LabDrawPoint, type NamedSeries, } from "@/lib/insights/correlation-discovery"; +import { resolveLabFields } from "@/lib/labs/serialise"; +import { wallClockInTz } from "@/lib/tz/wall-clock"; import { buildComplianceDailySeries, buildSymptomSeverityDailySeries, @@ -177,3 +180,76 @@ export async function fetchSymptomSeries( role: "outcome", }); } + +/** Day key (YYYY-MM-DD) for an instant in the user's display timezone. */ +function tzDayKey(at: Date, tz: string): string { + const { year, month, day } = wallClockInTz(at, tz); + return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`; +} + +/** + * v1.22 — build the user's lab draws for the labs ↔ outcome correlation pass. + * + * One {@link LabDrawPoint} per QUANTITATIVE reading in the window, keyed + * `LAB:` (the resolved name, so two spellings of one marker + * collapse). HIDDEN biomarkers are excluded (the W3 catalog `hidden` flag — a + * marker the user retired from the active list must not silently re-enter an + * analysis surface). Qualitative readings (no numeric `value`) and rows whose + * resolved value is non-finite are dropped — there is nothing to correlate. + * The encrypted note column is never selected. + * + * Returns an EMPTY array when the user has no usable readings, so the discovery + * pass degrades to absent rather than fabricating a link. + */ +export async function fetchLabDraws( + userId: string, + tz: string, + since: Date, +): Promise { + const now = new Date(); + const rows = await prisma.labResult.findMany({ + where: { + userId, + deletedAt: null, + value: { not: null }, + takenAt: { gte: since, lte: now }, + }, + orderBy: { takenAt: "asc" }, + take: 5000, + select: { + analyte: true, + unit: true, + referenceLow: true, + referenceHigh: true, + panel: true, + value: true, + takenAt: true, + biomarkerId: true, + biomarker: { + select: { + id: true, + name: true, + unit: true, + lowerBound: true, + upperBound: true, + panel: true, + hidden: true, + }, + }, + }, + }); + + const draws: LabDrawPoint[] = []; + for (const row of rows) { + // Exclude retired markers (W3 hidden flag); unlinked rows cannot be hidden. + if (row.biomarker?.hidden) continue; + if (row.value === null || !Number.isFinite(row.value)) continue; + const resolved = resolveLabFields(row, row.biomarker); + draws.push({ + key: `LAB:${resolved.analyte}`, + day: tzDayKey(row.takenAt, tz), + value: row.value, + }); + } + return draws; +} diff --git a/src/lib/insights/correlation-discovery.ts b/src/lib/insights/correlation-discovery.ts index 54c4d3f05..e76699982 100644 --- a/src/lib/insights/correlation-discovery.ts +++ b/src/lib/insights/correlation-discovery.ts @@ -179,6 +179,22 @@ export interface DiscoveredCorrelation { interpretation: string; /** Lag in days applied (always 1 here). */ lagDays: number; + /** + * v1.22 — which window surfaced the pair. `retrospective` = the standard + * 180-day scan (the established pattern). `recent` = the rolling + * early-detection pass over the trailing {@link EARLY_WINDOW_DAYS} days, + * surfaced ONLY when the pair is not already established retrospectively. + * Absent on the legacy path so existing consumers read it as the + * retrospective default. + */ + window?: "retrospective" | "recent"; + /** + * v1.22 — true for a `recent`-window emerging pair: it cleared a tighter + * FDR and a no-faint effect bar over a short window, but rests on fewer days + * than the retrospective scan, so the narration must hedge it as emerging / + * provisional rather than established. + */ + provisional?: boolean; } export interface CorrelationDiscoveryResult { @@ -470,6 +486,13 @@ export const DISCOVERY_BEHAVIOURS = [ "BLOOD_GLUCOSE", "BLOOD_PRESSURE_SYS", "ACTIVITY_STEPS", + // v1.22 — sleep duration as a lag SOURCE too (it was outcome-only). This + // opens the clinically-obvious "poor sleep last night → next-day mood / HRV / + // a vital drifting" direction. The same-family guard collapses only the + // sleep→sleep self-lag (the key is its own family), so sleep stays free to + // pair cross-domain. The daily series is the same per-day collapse the outcome + // role already uses, so neither role sees a fabricated value. + "SLEEP_DURATION", // v1.21.0 (FDREXTEND) — daily adherence rate as a lag source: the high-value // "adherence dip today → a worse outcome tomorrow" direction. MEDICATION_COMPLIANCE_CHANNEL_KEY, @@ -483,6 +506,16 @@ export const DISCOVERY_OUTCOMES = [ "HEART_RATE_VARIABILITY", "RESTING_HEART_RATE", "WEIGHT", + // v1.22 — blood pressure as an OUTCOME (it was behaviour-only). This makes + // the clinically-obvious "medication adherence → next-day BP control" link + // discoverable (compliance reached HRV / RHR / weight but never BP). Both + // components are added so diastolic — previously never a candidate channel at + // all — can correlate too. The same-family guard skips every + // BP-behaviour → BP-outcome self/cross-component lag (serial autocorrelation), + // so only genuine cross-domain behaviours (compliance, mood, daylight, …) → + // BP survive. BH-FDR controls the wider pair family this opens up. + "BLOOD_PRESSURE_SYS", + "BLOOD_PRESSURE_DIA", // v1.11.5 (F3) — mood is now an OUTCOME channel too, not only a behaviour. // The discovery loop skips the MOOD→MOOD self-pair (`b.key === o.key`), so // promoting it lets the FDR scan surface "behaviour today → next-day mood" @@ -504,3 +537,364 @@ export const DISCOVERY_OUTCOMES = [ // documented here so the omission is intentional, not a gap (same posture // as the medication-compliance omission above). ] as const; + +// ─── v1.22 — rolling early-detection window ─────────────────────────────── + +/** + * v1.22 — trailing window (days) for the rolling EARLY-detection pass. + * + * The standard discovery scan is retrospective: it needs n ≥ 20 paired days, so + * a pattern only surfaces once ~3 weeks of overlap have accrued INSIDE the + * 180-day window — it explains the past well but reacts slowly to an emerging + * shift (an illness pulling HRV + sleep down together over the last fortnight). + * This shorter window re-runs the SAME engine over only the trailing days so a + * fresh, concentrated pattern can surface before the long window dilutes it. + */ +export const EARLY_WINDOW_DAYS = 21; + +/** + * v1.22 — paired-day floor for the early window. Lower than the retrospective + * `MIN_PAIRED_N` (20) because 21 days cannot reach 20 lagged pairs, but still + * high enough that Pearson is defensible. The shorter, noisier series is + * counter-balanced by a TIGHTER FDR ({@link EARLY_FDR_Q}) and a no-faint effect + * bar (see {@link discoverEmergingCorrelations}), so the early pass keeps a HIGH + * firing bar despite the smaller n. + */ +export const EARLY_MIN_PAIRS = 12; + +/** + * v1.22 — tighter Benjamini-Hochberg target for the early window (vs the + * retrospective {@link FDR_Q} = 0.10). A short window is noisier, so the early + * pass demands a stricter false-discovery rate before it will narrate an + * emerging pattern. + */ +export const EARLY_FDR_Q = 0.05; + +/** Pure: keep only the points on/after `fromDayKey` (inclusive). */ +export function filterSeriesToWindow( + series: NamedSeries[], + fromDayKey: string, +): NamedSeries[] { + return series.map((s) => ({ + ...s, + points: s.points.filter((p) => p.day >= fromDayKey), + })); +} + +/** Stable pair identity for double-count exclusion. */ +function pairKey(behaviour: string, outcome: string): string { + return `${behaviour}${outcome}`; +} + +export interface EmergingCorrelationResult { + /** + * Recent-window pairs that are NOT already established by the retrospective + * scan — the genuinely emerging signals. Each is flagged `window: "recent"` + * and `provisional: true`. + */ + emerging: DiscoveredCorrelation[]; + /** Trailing window (days) the early pass scanned. */ + windowDays: number; + /** Paired-day floor enforced for the early pass. */ + minPairs: number; + /** FDR target the early pass used. */ + fdrQ: number; + /** How many pairs were tested in the early window (honest footer). */ + pairsTested: number; +} + +/** + * v1.22 — rolling early-detection pass. + * + * Re-runs {@link discoverCorrelations} over only the trailing + * {@link EARLY_WINDOW_DAYS} days (a lower n floor, a tighter FDR), then keeps + * ONLY pairs that: + * 1. are not already in the retrospective `discovered` set (no double-count — + * an established pattern is reported once, retrospectively), and + * 2. clear a no-`faint` effect bar — over a short, noisy window a faint signal + * is too likely to be transient noise to narrate as "emerging". + * + * The early window runs its OWN Benjamini-Hochberg control across its OWN tested + * family, so FDR discipline holds independently in each pass. Pure — the caller + * passes the full series and the day key of the early-window start. + */ +export function discoverEmergingCorrelations( + fullSeries: NamedSeries[], + retrospective: CorrelationDiscoveryResult, + opts: { + recentFromDayKey: string; + windowDays?: number; + lagDays?: number; + minPairs?: number; + fdrQ?: number; + }, +): EmergingCorrelationResult { + const windowDays = opts.windowDays ?? EARLY_WINDOW_DAYS; + const minPairs = opts.minPairs ?? EARLY_MIN_PAIRS; + const fdrQ = opts.fdrQ ?? EARLY_FDR_Q; + + const recentSeries = filterSeriesToWindow(fullSeries, opts.recentFromDayKey); + const recent = discoverCorrelations(recentSeries, { + lagDays: opts.lagDays, + minPairs, + fdrQ, + }); + + const established = new Set( + retrospective.discovered.map((d) => pairKey(d.behaviour, d.outcome)), + ); + + const emerging = recent.discovered + // High firing bar for the noisier short window: drop faint-tier signals. + .filter((d) => d.tier !== "faint") + // No double-count: an already-established retrospective pair is reported + // once (retrospectively), never re-surfaced as "emerging". + .filter((d) => !established.has(pairKey(d.behaviour, d.outcome))) + .map((d) => ({ + ...d, + window: "recent" as const, + provisional: true, + })); + + return { + emerging, + windowDays, + minPairs, + fdrQ, + pairsTested: recent.pairsTested, + }; +} + +// ─── v1.22 — labs ↔ outcome correlation ─────────────────────────────────── + +/** + * v1.22 — why labs are NOT a daily-lag channel in the matrix above. + * + * Lab draws are sparse and irregular (a panel every few months), so a day-D → + * day-D+1 lagged Pearson with an n ≥ 20 paired-DAY floor is structurally + * unreachable for a biomarker — folding `LAB:*` into `DISCOVERY_*` would only + * add channels that always degrade to absent (dead candidates). The clinically + * meaningful object for a sparse marker is different: each draw vs the + * CONTEMPORANEOUS windowed mean of an outcome the marker plausibly reflects + * (HbA1c ↔ the period's mean glucose; lipids ↔ weight). That is a point-vs-window + * statistic over the DRAWS (n = number of draws), not a daily lag — so it gets + * its own FDR-controlled pass here rather than a forced, never-firing channel. + */ + +/** A single lab draw, day-keyed in the user's tz. */ +export interface LabDrawPoint { + /** Stable channel key — `LAB:`. */ + key: string; + /** Day key YYYY-MM-DD of the draw. */ + day: string; + /** Numeric reading (qualitative rows are excluded by the caller). */ + value: number; +} + +/** One discovered lab ↔ outcome association — descriptive, never causal. */ +export interface DiscoveredLabCorrelation { + /** `LAB:` channel key (display strips the prefix). */ + lab: string; + /** Outcome channel key it tracks with. */ + outcome: string; + /** Number of draws paired with a usable contemporaneous outcome window. */ + n: number; + /** Pearson r across (draw value, contemporaneous outcome window-mean). */ + r: number; + pValue: number; + qValue: number; + /** Trailing days each draw's outcome window spanned. */ + windowDays: number; + /** Conservative, descriptive interpretation — never causal. */ + interpretation: string; +} + +export interface LabCorrelationResult { + discovered: DiscoveredLabCorrelation[]; + /** Lab × outcome pairs assessed (honest footer). */ + pairsTested: number; + fdrQ: number; + /** Minimum paired-draw count enforced per pair. */ + minDraws: number; +} + +/** + * v1.22 — curated outcome channels a lab biomarker is paired against. Kept + * small and physiologically broad (the flagship clinical wins the integration + * audit names) so the lab × outcome family stays tight and BH-FDR stays honest: + * a biomarker that tracks the period's mean weight, glucose, or systolic BP. + */ +export const LAB_OUTCOME_TARGETS: readonly string[] = [ + "WEIGHT", + "BLOOD_GLUCOSE", + "BLOOD_PRESSURE_SYS", +]; + +/** Default trailing window (days) each draw's outcome mean is taken over. */ +export const LAB_OUTCOME_WINDOW_DAYS = 28; + +/** Default minimum draws (paired with a usable outcome window) per lab pair. */ +export const LAB_MIN_DRAWS = 5; + +/** Default minimum outcome points inside a draw's window for it to count. */ +export const LAB_MIN_WINDOW_POINTS = 5; + +/** YYYY-MM-DD `daysBack` days before `day` (inclusive lower bound). */ +function shiftDayBack(day: string, daysBack: number): string { + return shiftDay(day, -daysBack); +} + +/** + * Mean of an outcome series over `(drawDay - windowDays, drawDay]` — the period + * the marker physiologically reflects. Returns `null` when too few outcome + * points fall in the window for a defensible mean. Pure. + */ +function outcomeWindowMean( + outcome: DailySeriesPoint[], + drawDay: string, + windowDays: number, + minPoints: number, +): number | null { + const lower = shiftDayBack(drawDay, windowDays); + let sum = 0; + let count = 0; + for (const p of outcome) { + if (p.day > lower && p.day <= drawDay && Number.isFinite(p.value)) { + sum += p.value; + count += 1; + } + } + return count >= minPoints ? sum / count : null; +} + +/** + * v1.22 — FDR-controlled labs ↔ outcome discovery. + * + * For each biomarker (grouped by `LabDrawPoint.key`) and each + * {@link LAB_OUTCOME_TARGETS} outcome, pair every draw value with the + * contemporaneous trailing-window mean of that outcome, then run Pearson + the + * exact p-value across the draws. Benjamini-Hochberg controls the lab × outcome + * family; the same effect-size floor + James-Stein shrinkage as the daily engine + * gate the survivors. A sparse marker simply fails the `minDraws` floor and + * degrades to absent — it never fabricates a link. Pure over already-fetched + * draws + outcome series. + */ +export function discoverLabOutcomeCorrelations( + labDraws: LabDrawPoint[], + series: NamedSeries[], + opts: { + windowDays?: number; + minDraws?: number; + minWindowPoints?: number; + fdrQ?: number; + } = {}, +): LabCorrelationResult { + const windowDays = opts.windowDays ?? LAB_OUTCOME_WINDOW_DAYS; + const minDraws = opts.minDraws ?? LAB_MIN_DRAWS; + const minWindowPoints = opts.minWindowPoints ?? LAB_MIN_WINDOW_POINTS; + const fdrQ = opts.fdrQ ?? FDR_Q; + + // Group draws by biomarker key (newest-irrelevant; Pearson is order-free). + const drawsByLab = new Map(); + for (const d of labDraws) { + if (!Number.isFinite(d.value)) continue; + const list = drawsByLab.get(d.key) ?? []; + list.push(d); + drawsByLab.set(d.key, list); + } + + // Index the curated outcome series by key (last-write keeps a single series + // per key even when a channel appears in both roles). + const outcomeByKey = new Map(); + for (const s of series) { + if (LAB_OUTCOME_TARGETS.includes(s.key)) outcomeByKey.set(s.key, s.points); + } + + interface RawLabPair { + lab: string; + outcome: string; + n: number; + r: number; + pValue: number; + } + const tested: RawLabPair[] = []; + for (const [lab, draws] of drawsByLab) { + for (const outcomeKey of LAB_OUTCOME_TARGETS) { + const outcome = outcomeByKey.get(outcomeKey); + if (!outcome || outcome.length === 0) continue; + const xs: number[] = []; + const ys: number[] = []; + for (const draw of draws) { + const mean = outcomeWindowMean( + outcome, + draw.day, + windowDays, + minWindowPoints, + ); + if (mean === null) continue; + xs.push(draw.value); + ys.push(mean); + } + if (xs.length < minDraws) continue; + const result = pearson({ xs, ys, minPairs: minDraws }); + if (result.status !== "ok") continue; + tested.push({ + lab, + outcome: outcomeKey, + n: result.n, + r: result.r, + pValue: result.pValue, + }); + } + } + + const pairsTested = tested.length; + if (pairsTested === 0) { + return { discovered: [], pairsTested: 0, fdrQ, minDraws }; + } + + const qValues = benjaminiHochberg(tested.map((t) => t.pValue)); + const discovered: DiscoveredLabCorrelation[] = tested + .map((t, i) => ({ ...t, qValue: qValues[i] })) + .filter((t) => t.pValue < 0.05 && t.qValue <= fdrQ) + .map((t) => { + const shrunkR = shrinkEstimate(t.r, t.n); + const tier = confidenceTier(shrunkR, t.n); + return { ...t, shrunkR, tier }; + }) + .filter( + ( + t, + ): t is RawLabPair & { + qValue: number; + shrunkR: number; + tier: ConfidenceTier; + } => t.tier !== null, + ) + .map((t) => ({ + lab: t.lab, + outcome: t.outcome, + n: t.n, + r: t.r, + pValue: t.pValue, + qValue: Math.round(t.qValue * 1000) / 1000, + windowDays, + interpretation: interpretLab(t.lab, t.outcome, t.r), + })) + .sort((a, b) => Math.abs(b.r) - Math.abs(a.r) || a.qValue - b.qValue); + + return { discovered, pairsTested, fdrQ, minDraws }; +} + +/** Strip the `LAB:` prefix for display. */ +function humaniseLab(key: string): string { + return key.startsWith("LAB:") ? key.slice("LAB:".length) : key; +} + +/** Descriptive, never-causal interpretation for a lab ↔ outcome pair. */ +function interpretLab(lab: string, outcome: string, r: number): string { + const l = humaniseLab(lab); + const o = humanise(outcome); + const dir = r < 0 ? "lower" : "higher"; + return `Higher ${l} readings line up with ${dir} ${o} over the same periods in your data — an association worth watching with your clinician, never a cause.`; +} diff --git a/src/lib/insights/features.ts b/src/lib/insights/features.ts index efe3fe3ba..5d4c54872 100644 --- a/src/lib/insights/features.ts +++ b/src/lib/insights/features.ts @@ -26,6 +26,8 @@ import { type CorrelationResult, } from "@/lib/analytics/correlations"; import { getMedicationCategories } from "@/lib/medication-category"; +import { classifyReferenceRange } from "@/lib/labs/reference-range"; +import { resolveLabFields } from "@/lib/labs/serialise"; import { readRollupBuckets } from "@/lib/rollups/measurement-rollups"; import { ensureUserMoodRollupsFresh, @@ -197,6 +199,77 @@ export interface AggregatedFeatures { * forces the briefing to regenerate. See `computeSignalsOfDay`. */ signalsOfDay?: SignalOfDay[]; + /** + * v1.22 — glucose aggregate block. Glucose previously reached the briefing + * only as a single `signalOfDay` row; this is the present-and-trend block the + * other vitals already carry (7 / 30 / 90-day means + a 30-day slope), from + * the canonical stored value. Omitted when the user has no glucose readings. + */ + glucose?: { + avg7: number | null; + avg30: number | null; + avg90: number | null; + latest: number | null; + latestDaysAgo: number | null; + slope30: number | null; + coverage: DataCoverage; + }; + /** + * v1.22 — recent FLAGGED biomarkers (abnormal or trending). Closes the + * biggest siloed domain: labs reached the Coach snapshot but never the + * briefing. Hidden biomarkers (the W3 catalog `hidden` flag) are excluded; + * qualitative rows carry a neutral `unknown` status (no fabricated verdict). + * Only abnormal / trending markers surface (the briefing-relevant ones) and + * the list is bounded. Omitted when nothing is flagged. + */ + labs?: { + flagged: Array<{ + analyte: string; + value: number | null; + valueText: string | null; + unit: string; + rangeStatus: "in-range" | "below" | "above" | "unknown"; + /** Direction vs the immediately prior reading; null without a prior. */ + trend: "rising" | "falling" | "flat" | null; + takenAt: string; + daysAgo: number; + }>; + /** Count of flagged markers (may exceed the bounded `flagged` length). */ + flaggedCount: number; + }; + /** + * v1.22 — preventive-care (Vorsorge) read-side: due + overdue items. The + * early-nudge use case ("your screening is overdue"). Previously fully siloed + * — captured + scheduled but read by no AI surface. Labels are sanitised + * (user free-text). Omitted when nothing is due or overdue. + */ + preventiveCare?: { + overdue: Array<{ label: string; daysOverdue: number }>; + due: Array<{ label: string; daysUntil: number }>; + }; + /** + * v1.22 — workout aggregate. The briefing previously saw activity only as + * ACTIVITY_STEPS; the `Workout` table never reached it. Provider-agnostic + * counts + load over the trailing windows. Omitted when no workouts logged. + */ + workouts?: { + last7: { + count: number; + totalDurationMin: number; + totalDistanceKm: number | null; + }; + last30: { + count: number; + totalDurationMin: number; + totalDistanceKm: number | null; + }; + latest: { + sportType: string; + daysAgo: number; + durationMin: number; + distanceKm: number | null; + } | null; + }; } /** @@ -649,6 +722,267 @@ function computeSignalsOfDay( .slice(0, 3); } +// ─── v1.22 — cross-signal integration blocks (briefing-scoped) ──────────── + +/** + * v1.22 — the integration blocks (labs / preventive-care / workouts) carry + * extra DB reads. They are BRIEFING-scoped: the daily briefing reads a wide + * window (`BRIEFING_FEATURE_WINDOW_DAYS`) and narrates long trends, while the + * Coach snapshot path calls `extractFeatures` with a tight ~90-day window and + * already carries its own labs / illness / workout context. Gating on the read + * window keeps these extra reads off every Coach turn without a new caller flag: + * the briefing window (400) clears it, the Coach window (90) does not. + */ +const INTEGRATION_BLOCK_MIN_WINDOW_DAYS = 180; + +/** Days a preventive-care item must be due within to surface as "due soon". */ +const PREVENTIVE_DUE_HORIZON_DAYS = 21; + +/** Cap on items surfaced per preventive-care bucket. */ +const PREVENTIVE_MAX_PER_BUCKET = 5; + +/** Cap on flagged biomarkers surfaced to the briefing. */ +const LABS_MAX_FLAGGED = 8; + +/** Only lab readings within this many months are considered "recent". */ +const LABS_LOOKBACK_MONTHS = 12; + +/** Trailing window (days) for the workout aggregate. */ +const WORKOUT_WINDOW_DAYS = 90; + +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +/** + * Strip control chars + collapse whitespace, then bound the length, before a + * user-supplied label can reach the briefing prompt. Mirrors the labs / + * illness snapshot label handling — a self-scoped prompt-injection surface. + */ +function sanitizeLabel(text: string, max = 80): string { + let out = ""; + for (const ch of text) { + const code = ch.codePointAt(0) ?? 0; + out += code < 0x20 || (code >= 0x7f && code <= 0x9f) ? " " : ch; + } + return out.replace(/\s+/g, " ").trim().slice(0, max); +} + +/** + * v1.22 — recent FLAGGED biomarkers (abnormal or trending) for the briefing. + * Most-recent reading per biomarker over the lookback window; hidden markers + * excluded; qualitative rows neutral. Only abnormal (below/above) OR trending + * markers surface, bounded. Returns `undefined` when nothing is flagged so the + * block is omitted rather than emitting an empty shape. + */ +async function readLabsBriefingBlock( + userId: string, + now: number, +): Promise { + const nowDate = new Date(now); + const cutoff = new Date(nowDate); + cutoff.setMonth(cutoff.getMonth() - LABS_LOOKBACK_MONTHS); + + const rows = await prisma.labResult.findMany({ + where: { userId, deletedAt: null, takenAt: { gte: cutoff, lte: nowDate } }, + orderBy: { takenAt: "desc" }, + take: LABS_MAX_FLAGGED * 16, + select: { + analyte: true, + panel: true, + unit: true, + value: true, + valueText: true, + referenceLow: true, + referenceHigh: true, + takenAt: true, + biomarkerId: true, + biomarker: { + select: { + id: true, + name: true, + unit: true, + lowerBound: true, + upperBound: true, + panel: true, + hidden: true, + }, + }, + }, + }); + if (rows.length === 0) return undefined; + + // Group rows per biomarker identity (linked id, else lower-cased analyte), + // newest-first, so we can read the latest reading + the immediately prior one + // for a trend. Hidden markers are dropped entirely. + const byMarker = new Map(); + for (const row of rows) { + if (row.biomarker?.hidden) continue; + const resolved = resolveLabFields(row, row.biomarker); + const key = row.biomarkerId ?? `analyte:${resolved.analyte.toLowerCase()}`; + const list = byMarker.get(key) ?? []; + list.push(row); + byMarker.set(key, list); + } + + const flagged: NonNullable["flagged"] = []; + for (const list of byMarker.values()) { + const latest = list[0]; + const resolved = resolveLabFields(latest, latest.biomarker); + const rangeStatus = + latest.value === null + ? ("unknown" as const) + : classifyReferenceRange( + latest.value, + resolved.referenceLow, + resolved.referenceHigh, + ); + + // Trend = latest numeric reading vs the immediately prior numeric reading. + let trend: "rising" | "falling" | "flat" | null = null; + if (latest.value !== null) { + const prior = list.find((r, i) => i > 0 && r.value !== null); + if (prior?.value != null) { + const delta = latest.value - prior.value; + const eps = Math.max(Math.abs(prior.value) * 0.02, 1e-9); + trend = delta > eps ? "rising" : delta < -eps ? "falling" : "flat"; + } + } + + const isAbnormal = rangeStatus === "below" || rangeStatus === "above"; + const isTrending = trend === "rising" || trend === "falling"; + if (!isAbnormal && !isTrending) continue; + + flagged.push({ + analyte: resolved.analyte, + value: latest.value, + valueText: latest.valueText ? sanitizeLabel(latest.valueText, 60) : null, + unit: resolved.unit, + rangeStatus, + trend, + takenAt: latest.takenAt.toISOString(), + daysAgo: Math.round((now - latest.takenAt.getTime()) / MS_PER_DAY), + }); + } + + if (flagged.length === 0) return undefined; + // Abnormal markers lead, then most-recent first. + flagged.sort((a, b) => { + const abn = (s: typeof a.rangeStatus) => + s === "below" || s === "above" ? 0 : 1; + const d = abn(a.rangeStatus) - abn(b.rangeStatus); + return d !== 0 ? d : a.daysAgo - b.daysAgo; + }); + return { + flagged: flagged.slice(0, LABS_MAX_FLAGGED), + flaggedCount: flagged.length, + }; +} + +/** + * v1.22 — preventive-care (Vorsorge) due + overdue read-side. Reads the + * user's enabled, live reminders and buckets by the server-authoritative + * `nextDueAt`. Returns `undefined` when nothing is due or overdue. + */ +async function readPreventiveCareBlock( + userId: string, + now: number, +): Promise { + const horizon = new Date(now + PREVENTIVE_DUE_HORIZON_DAYS * MS_PER_DAY); + const rows = await prisma.measurementReminder.findMany({ + where: { + userId, + deletedAt: null, + enabled: true, + nextDueAt: { not: null, lte: horizon }, + }, + orderBy: { nextDueAt: "asc" }, + take: (PREVENTIVE_MAX_PER_BUCKET + 1) * 4, + select: { label: true, nextDueAt: true }, + }); + if (rows.length === 0) return undefined; + + const overdue: NonNullable["overdue"] = + []; + const due: NonNullable["due"] = []; + for (const r of rows) { + if (!r.nextDueAt) continue; + const label = sanitizeLabel(r.label); + if (!label) continue; + const diffMs = r.nextDueAt.getTime() - now; + if (diffMs < 0) { + overdue.push({ label, daysOverdue: Math.round(-diffMs / MS_PER_DAY) }); + } else { + due.push({ label, daysUntil: Math.round(diffMs / MS_PER_DAY) }); + } + } + if (overdue.length === 0 && due.length === 0) return undefined; + return { + overdue: overdue.slice(0, PREVENTIVE_MAX_PER_BUCKET), + due: due.slice(0, PREVENTIVE_MAX_PER_BUCKET), + }; +} + +/** + * v1.22 — workout aggregate over the trailing window. Provider-agnostic: + * counts + summed duration + summed distance (km, when any source reported it) + * over 7 / 30 days, plus the latest workout. Returns `undefined` when no + * workouts fall in the window. + */ +async function readWorkoutsBlock( + userId: string, + now: number, +): Promise { + const since = new Date(now - WORKOUT_WINDOW_DAYS * MS_PER_DAY); + const rows = await prisma.workout.findMany({ + where: { userId, startedAt: { gte: since } }, + orderBy: { startedAt: "desc" }, + take: 2000, + select: { + sportType: true, + startedAt: true, + durationSec: true, + totalDistanceM: true, + }, + }); + if (rows.length === 0) return undefined; + + const tally = (windowDays: number) => { + const cutoff = now - windowDays * MS_PER_DAY; + let count = 0; + let durationSec = 0; + let distanceM = 0; + let anyDistance = false; + for (const w of rows) { + if (w.startedAt.getTime() < cutoff) continue; + count += 1; + durationSec += w.durationSec; + if (w.totalDistanceM != null) { + distanceM += w.totalDistanceM; + anyDistance = true; + } + } + return { + count, + totalDurationMin: Math.round(durationSec / 60), + totalDistanceKm: anyDistance + ? Math.round((distanceM / 1000) * 10) / 10 + : null, + }; + }; + + const newest = rows[0]; + const latest = { + sportType: sanitizeLabel(newest.sportType, 40), + daysAgo: Math.round((now - newest.startedAt.getTime()) / MS_PER_DAY), + durationMin: Math.round(newest.durationSec / 60), + distanceKm: + newest.totalDistanceM != null + ? Math.round((newest.totalDistanceM / 1000) * 10) / 10 + : null, + }; + + return { last7: tally(7), last30: tally(30), latest }; +} + export async function extractFeatures( userId: string, includeRaw: boolean, @@ -1389,6 +1723,46 @@ export async function extractFeatures( features.signalsOfDay = signalsOfDay; } + // v1.22 — glucose aggregate block. Computed from the already-fetched + // measurement set (no extra DB read), so it is always cheap to emit. Glucose + // previously reached the briefing only as one `signalOfDay` row. + const glucoseData = byType("BLOOD_GLUCOSE"); + if (glucoseData.length > 0) { + const summary = summarize(toDataPoints(glucoseData)); + const newest = glucoseData[glucoseData.length - 1]; + features.glucose = { + avg7: summary.avg7, + avg30: summary.avg30, + avg90: avgInWindow(glucoseData, now, 90), + latest: summary.latest, + latestDaysAgo: Math.round( + (now - newest.measuredAt.getTime()) / (24 * 60 * 60 * 1000), + ), + slope30: summary.slope30?.slope ?? null, + coverage: computeCoverage(glucoseData, now), + }; + } + + // v1.22 — cross-signal integration blocks (labs / preventive-care / + // workouts). Briefing-scoped: only the wide briefing window pays the extra + // reads; the tight Coach-snapshot window (which carries its own labs / illness + // / workout context) skips them. Each block is omitted when its domain is + // empty (no fabricated zeros). Fetched in parallel — independent reads. + const includeIntegrations = + sinceCutoff === null || + (typeof sinceDays === "number" && + sinceDays >= INTEGRATION_BLOCK_MIN_WINDOW_DAYS); + if (includeIntegrations) { + const [labs, preventiveCare, workouts] = await Promise.all([ + readLabsBriefingBlock(userId, now), + readPreventiveCareBlock(userId, now), + readWorkoutsBlock(userId, now), + ]); + if (labs) features.labs = labs; + if (preventiveCare) features.preventiveCare = preventiveCare; + if (workouts) features.workouts = workouts; + } + // v1.4.36 W3 T1 — "raw" mode no longer dumps every measurement row // into the prompt. Instead we attach DAY / WEEK / MONTH bucket means // from `measurement_rollups` so the model gets the same temporal diff --git a/src/lib/openapi/routes/insights/schemas.ts b/src/lib/openapi/routes/insights/schemas.ts index d5ee83614..82971e74f 100644 --- a/src/lib/openapi/routes/insights/schemas.ts +++ b/src/lib/openapi/routes/insights/schemas.ts @@ -296,9 +296,97 @@ export const discoveredCorrelation = z .string() .describe("Conservative, descriptive interpretation — never causal."), lagDays: z.number().int().describe("Lag in days applied (1)."), + window: z + .enum(["retrospective", "recent"]) + .optional() + .describe( + "v1.22 — which window surfaced the pair. Absent on the 180-day scan (retrospective default); `recent` for an emerging early-detection pair.", + ), + provisional: z + .boolean() + .optional() + .describe( + "v1.22 — true for an emerging recent-window pair: fewer days, hedged as provisional rather than established.", + ), }) .meta({ id: "DiscoveredCorrelation" }); +// v1.22 — one labs ↔ outcome association (point-vs-window over sparse draws). +export const discoveredLabCorrelation = z + .object({ + lab: z + .string() + .describe("`LAB:` channel key (display strips the prefix)."), + outcome: z + .string() + .describe( + "Outcome channel the marker tracks with (WEIGHT, BLOOD_GLUCOSE, BLOOD_PRESSURE_SYS).", + ), + n: z + .number() + .int() + .describe("Draws paired with a usable contemporaneous outcome window."), + r: z + .number() + .describe( + "Pearson r over (draw value, contemporaneous outcome window-mean).", + ), + pValue: z.number().describe("Two-sided exact Student-t p-value (< 0.05)."), + qValue: z + .number() + .describe( + "Benjamini-Hochberg FDR-adjusted q-value (≤ the surface threshold).", + ), + windowDays: z + .number() + .int() + .describe("Trailing days each draw's outcome window spanned."), + interpretation: z + .string() + .describe("Conservative, descriptive interpretation — never causal."), + }) + .meta({ id: "DiscoveredLabCorrelation" }); + +// v1.22 — rolling early-detection result (recent-window emerging pairs). +export const emergingCorrelationResult = z + .object({ + emerging: z + .array(discoveredCorrelation) + .describe( + "Recent-window pairs NOT already established retrospectively — the emerging signals (provisional, hedged).", + ), + windowDays: z + .number() + .int() + .describe("Trailing window (days) the early pass scanned."), + minPairs: z + .number() + .int() + .describe("Paired-day floor enforced for the early pass."), + fdrQ: z + .number() + .describe("FDR target the early pass used (tighter than the main scan)."), + pairsTested: z.number().int().describe("Pairs tested in the early window."), + }) + .meta({ id: "EmergingCorrelationResult" }); + +// v1.22 — labs ↔ outcome pass result. +export const labCorrelationResult = z + .object({ + discovered: z + .array(discoveredLabCorrelation) + .describe( + "Lab ↔ outcome associations surviving the per-pair floor + BH-FDR.", + ), + pairsTested: z.number().int().describe("Lab × outcome pairs assessed."), + fdrQ: z.number().describe("The FDR target the pass used."), + minDraws: z + .number() + .int() + .describe("Minimum paired-draw count enforced per pair."), + }) + .meta({ id: "LabCorrelationResult" }); + export const correlationDiscoveryResponse = z .object({ discovered: z @@ -313,6 +401,16 @@ export const correlationDiscoveryResponse = z .number() .int() .describe("Minimum paired-day count enforced per pair."), + emerging: emergingCorrelationResult + .optional() + .describe( + "v1.22 — rolling early-detection pass over the trailing window; emerging pairs not yet established retrospectively (no double-count).", + ), + labCorrelations: labCorrelationResult + .optional() + .describe( + "v1.22 — labs ↔ outcome associations (each draw vs the contemporaneous outcome window-mean), FDR-controlled; absent-degrading on sparse draws.", + ), }) .meta({ id: "CorrelationDiscoveryResponse",