From 138211b46a38c3c2807a4ae1bb66a1a93b0f7d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 27 Jun 2026 13:34:30 +0200 Subject: [PATCH] feat(insights): early-detection window and new correlation linkages Add a rolling 14-21 day early-detection pass to the FDR correlation engine so an emerging shift (illness pulling HRV and sleep down together) surfaces before the 180-day retrospective scan accrues the n>=20 it needs. The recent pass re-runs the same engine over the trailing window with a lower paired-day floor, a tighter Benjamini-Hochberg target, and a no-faint effect bar, then drops any pair the retrospective scan already established so a pattern is never reported as both established and emerging. Extend the curated behaviour x outcome matrix with three clinically meaningful links: medication adherence -> next-day blood pressure (systolic and diastolic, the latter previously never a candidate channel), sleep as a behaviour (poor sleep -> next-day mood / HRV), and a separate point-vs-window labs -> outcome pass (each draw against the contemporaneous outcome mean) since sparse lab draws cannot meet a daily-lag floor. Every pass keeps FDR control, the effect-size floor, and shrinkage; thin data degrades to absent, never a fabricated link. All associations stay descriptive, never causal. --- .../__tests__/correlation-discovery.test.ts | 250 +++++++++++ .../insights/correlation-channel-series.ts | 76 ++++ src/lib/insights/correlation-discovery.ts | 394 ++++++++++++++++++ 3 files changed, 720 insertions(+) diff --git a/src/lib/insights/__tests__/correlation-discovery.test.ts b/src/lib/insights/__tests__/correlation-discovery.test.ts index 8eb2bff08..02c7c9060 100644 --- a/src/lib/insights/__tests__/correlation-discovery.test.ts +++ b/src/lib/insights/__tests__/correlation-discovery.test.ts @@ -529,3 +529,253 @@ describe("compliance + symptom channels (FDREXTEND)", () => { expect(pair!.r).toBeLessThan(0); // higher adherence → lower next-day RHR }); }); + +// ─── v1.22 — new linkages + early detection + labs ──────────────────────── + +import { + DISCOVERY_BEHAVIOURS as MATRIX_BEHAVIOURS, + DISCOVERY_OUTCOMES as MATRIX_OUTCOMES, + discoverEmergingCorrelations, + discoverLabOutcomeCorrelations, + filterSeriesToWindow, + EARLY_WINDOW_DAYS, + type LabDrawPoint, +} from "../correlation-discovery"; + +/** Contiguous daily series starting `startISO` (YYYY-MM-DD). */ +function seriesFrom(startISO: string, values: number[]): DailySeriesPoint[] { + const [y, m, d] = startISO.split("-").map(Number); + const start = Date.UTC(y, m - 1, d); + return values.map((value, i) => { + const dt = new Date(start + i * 24 * 60 * 60 * 1000); + return { day: dt.toISOString().slice(0, 10), value }; + }); +} + +describe("v1.22 — new curated linkages", () => { + it("adds sleep as a behaviour (was outcome-only)", () => { + expect(MATRIX_BEHAVIOURS).toContain("SLEEP_DURATION"); + }); + + it("adds blood pressure (both components) as outcomes", () => { + expect(MATRIX_OUTCOMES).toContain("BLOOD_PRESSURE_SYS"); + expect(MATRIX_OUTCOMES).toContain("BLOOD_PRESSURE_DIA"); + }); + + it("surfaces compliance→BP_SYS but skips the same-family BP→BP self-cross", () => { + const n = 80; + const adherence = Array.from({ length: n }, (_, i) => + i % 2 === 0 ? 100 : 50, + ); + // Higher adherence today → lower next-day systolic. + const sys = [ + 130, + ...adherence + .slice(0, n - 1) + .map((a, i) => 160 - a / 4 + (i % 2 ? 0.2 : -0.2)), + ]; + const result = discoverCorrelations([ + { + key: MEDICATION_COMPLIANCE_CHANNEL_KEY, + role: "behaviour", + points: longSeries(adherence), + }, + // BP is both a behaviour and an outcome in the real matrix; the + // same-family guard must keep BP_SYS(behaviour)→BP_SYS(outcome) out. + { key: "BLOOD_PRESSURE_SYS", role: "behaviour", points: longSeries(sys) }, + { key: "BLOOD_PRESSURE_SYS", role: "outcome", points: longSeries(sys) }, + ]); + expect( + result.discovered.find( + (p) => + p.behaviour === "BLOOD_PRESSURE_SYS" && + p.outcome === "BLOOD_PRESSURE_SYS", + ), + ).toBeUndefined(); + const link = result.discovered.find( + (p) => + p.behaviour === MEDICATION_COMPLIANCE_CHANNEL_KEY && + p.outcome === "BLOOD_PRESSURE_SYS", + ); + expect(link).toBeDefined(); + expect(link!.r).toBeLessThan(0); + }); +}); + +describe("discoverEmergingCorrelations (early detection)", () => { + it("filterSeriesToWindow keeps only on/after the cutoff", () => { + const s: NamedSeries[] = [ + { + key: "MOOD", + role: "behaviour", + points: seriesFrom("2026-03-01", [1, 2, 3, 4, 5]), + }, + ]; + const filtered = filterSeriesToWindow(s, "2026-03-03"); + expect(filtered[0].points.map((p) => p.day)).toEqual([ + "2026-03-03", + "2026-03-04", + "2026-03-05", + ]); + }); + + it("surfaces a fortnight-long signal the retrospective n≥20 floor cannot yet see", () => { + // Only ~18 days of data: a clean behaviour→next-day-outcome coupling. The + // retrospective scan needs ≥ 20 lagged pairs, so it sees NOTHING; the early + // window (floor 10) catches it — exactly the early-detection mandate. + const len = 18; + const beh = Array.from({ length: len }, (_, i) => (i % 2 ? 2 : 9)); + const out = beh.map((b, i) => 40 + b * 3 + (i % 2 ? 0.3 : -0.3)); + const series: NamedSeries[] = [ + { key: "MOOD", role: "behaviour", points: longSeries(beh) }, + { + key: "HEART_RATE_VARIABILITY", + role: "outcome", + points: longSeries(out), + }, + ]; + const retrospective = discoverCorrelations(series); + expect(retrospective.discovered).toHaveLength(0); // n too low for the floor + + const cutoff = longSeries(beh)[0].day; // whole (short) window + const { emerging } = discoverEmergingCorrelations(series, retrospective, { + recentFromDayKey: cutoff, + minPairs: 10, + }); + const pair = emerging.find( + (p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY", + ); + expect(pair).toBeDefined(); + expect(pair!.window).toBe("recent"); + expect(pair!.provisional).toBe(true); + }); + + it("does not double-count a pair already established retrospectively", () => { + // A strong relationship across the WHOLE 80-day window → retrospective finds + // it; the emerging pass must NOT re-surface it. + const n = 80; + const beh = Array.from({ length: n }, (_, i) => (i % 2 ? 2 : 9)); + const out = beh.map((b, i) => 40 + b * 3 + (i % 2 ? 0.3 : -0.3)); + const series: NamedSeries[] = [ + { key: "MOOD", role: "behaviour", points: longSeries(beh) }, + { + key: "HEART_RATE_VARIABILITY", + role: "outcome", + points: longSeries(out), + }, + ]; + const retrospective = discoverCorrelations(series); + expect( + retrospective.discovered.some( + (p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY", + ), + ).toBe(true); + const cutoff = new Date( + Date.UTC(2026, 0, 1) + (n - EARLY_WINDOW_DAYS) * 24 * 60 * 60 * 1000, + ) + .toISOString() + .slice(0, 10); + const { emerging } = discoverEmergingCorrelations(series, retrospective, { + recentFromDayKey: cutoff, + minPairs: 10, + }); + expect( + emerging.find( + (p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY", + ), + ).toBeUndefined(); + }); + + it("returns nothing on pure noise (FDR + high bar hold in the short window)", () => { + const n = 30; + const beh = Array.from({ length: n }, (_, i) => (i * 7) % 11); + const out = Array.from({ length: n }, (_, i) => (i * 3) % 5); + const series: NamedSeries[] = [ + { key: "MOOD", role: "behaviour", points: longSeries(beh) }, + { + key: "HEART_RATE_VARIABILITY", + role: "outcome", + points: longSeries(out), + }, + ]; + const retrospective = discoverCorrelations(series); + const cutoff = longSeries(beh)[Math.max(0, n - EARLY_WINDOW_DAYS)].day; + const { emerging } = discoverEmergingCorrelations(series, retrospective, { + recentFromDayKey: cutoff, + minPairs: 10, + }); + expect(emerging).toHaveLength(0); + }); +}); + +describe("discoverLabOutcomeCorrelations (labs ↔ outcomes)", () => { + function weightSeries(): NamedSeries { + // Daily weight that drifts up over ~6 months. + const values = Array.from({ length: 180 }, (_, i) => 80 + i * 0.05); + return { + key: "WEIGHT", + role: "outcome", + points: seriesFrom("2026-01-01", values), + }; + } + + it("surfaces a lab that tracks the contemporaneous outcome window-mean", () => { + const weight = weightSeries(); + // 8 monthly-ish HbA1c-like draws that rise with weight. + const draws: LabDrawPoint[] = []; + for (let i = 0; i < 8; i++) { + const dayIdx = 30 + i * 18; // spread draws across the window + const day = weight.points[dayIdx].day; + // Value tracks the period's weight mean (with tiny jitter). + const v = 5.0 + dayIdx * 0.01 + (i % 2 ? 0.02 : -0.02); + draws.push({ key: "LAB:HbA1c", day, value: v }); + } + const result = discoverLabOutcomeCorrelations(draws, [weight], { + minDraws: 5, + minWindowPoints: 5, + }); + const pair = result.discovered.find( + (d) => d.lab === "LAB:HbA1c" && d.outcome === "WEIGHT", + ); + expect(pair).toBeDefined(); + expect(pair!.n).toBeGreaterThanOrEqual(5); + expect(pair!.r).toBeGreaterThan(0); + expect(pair!.qValue).toBeLessThanOrEqual(result.fdrQ); + expect(pair!.interpretation).toMatch(/never a cause/i); + }); + + it("degrades to absent when the marker has too few draws", () => { + const weight = weightSeries(); + const draws: LabDrawPoint[] = [0, 1, 2].map((i) => ({ + key: "LAB:HbA1c", + day: weight.points[30 + i * 18].day, + value: 5 + i, + })); + const result = discoverLabOutcomeCorrelations(draws, [weight], { + minDraws: 5, + }); + expect(result.discovered).toHaveLength(0); + expect(result.pairsTested).toBe(0); + }); + + it("ignores outcomes outside the curated lab-target set", () => { + const hrv: NamedSeries = { + key: "HEART_RATE_VARIABILITY", + role: "outcome", + points: seriesFrom( + "2026-01-01", + Array.from({ length: 180 }, (_, i) => 50 + i * 0.1), + ), + }; + const draws: LabDrawPoint[] = Array.from({ length: 8 }, (_, i) => ({ + key: "LAB:HbA1c", + day: hrv.points[30 + i * 18].day, + value: 5 + i, + })); + const result = discoverLabOutcomeCorrelations(draws, [hrv], { + minDraws: 5, + }); + // HRV is not a curated lab-outcome target → no pair tested. + expect(result.pairsTested).toBe(0); + }); +}); diff --git a/src/lib/insights/correlation-channel-series.ts b/src/lib/insights/correlation-channel-series.ts index 7d2c65783..2911716a4 100644 --- a/src/lib/insights/correlation-channel-series.ts +++ b/src/lib/insights/correlation-channel-series.ts @@ -24,8 +24,11 @@ import { import { MEDICATION_COMPLIANCE_CHANNEL_KEY, SYMPTOM_SEVERITY_CHANNEL_KEY, + type LabDrawPoint, type NamedSeries, } from "@/lib/insights/correlation-discovery"; +import { resolveLabFields } from "@/lib/labs/serialise"; +import { wallClockInTz } from "@/lib/tz/wall-clock"; import { buildComplianceDailySeries, buildSymptomSeverityDailySeries, @@ -177,3 +180,76 @@ export async function fetchSymptomSeries( role: "outcome", }); } + +/** Day key (YYYY-MM-DD) for an instant in the user's display timezone. */ +function tzDayKey(at: Date, tz: string): string { + const { year, month, day } = wallClockInTz(at, tz); + return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`; +} + +/** + * v1.22 — build the user's lab draws for the labs ↔ outcome correlation pass. + * + * One {@link LabDrawPoint} per QUANTITATIVE reading in the window, keyed + * `LAB:` (the resolved name, so two spellings of one marker + * collapse). HIDDEN biomarkers are excluded (the W3 catalog `hidden` flag — a + * marker the user retired from the active list must not silently re-enter an + * analysis surface). Qualitative readings (no numeric `value`) and rows whose + * resolved value is non-finite are dropped — there is nothing to correlate. + * The encrypted note column is never selected. + * + * Returns an EMPTY array when the user has no usable readings, so the discovery + * pass degrades to absent rather than fabricating a link. + */ +export async function fetchLabDraws( + userId: string, + tz: string, + since: Date, +): Promise { + const now = new Date(); + const rows = await prisma.labResult.findMany({ + where: { + userId, + deletedAt: null, + value: { not: null }, + takenAt: { gte: since, lte: now }, + }, + orderBy: { takenAt: "asc" }, + take: 5000, + select: { + analyte: true, + unit: true, + referenceLow: true, + referenceHigh: true, + panel: true, + value: true, + takenAt: true, + biomarkerId: true, + biomarker: { + select: { + id: true, + name: true, + unit: true, + lowerBound: true, + upperBound: true, + panel: true, + hidden: true, + }, + }, + }, + }); + + const draws: LabDrawPoint[] = []; + for (const row of rows) { + // Exclude retired markers (W3 hidden flag); unlinked rows cannot be hidden. + if (row.biomarker?.hidden) continue; + if (row.value === null || !Number.isFinite(row.value)) continue; + const resolved = resolveLabFields(row, row.biomarker); + draws.push({ + key: `LAB:${resolved.analyte}`, + day: tzDayKey(row.takenAt, tz), + value: row.value, + }); + } + return draws; +} diff --git a/src/lib/insights/correlation-discovery.ts b/src/lib/insights/correlation-discovery.ts index 54c4d3f05..e76699982 100644 --- a/src/lib/insights/correlation-discovery.ts +++ b/src/lib/insights/correlation-discovery.ts @@ -179,6 +179,22 @@ export interface DiscoveredCorrelation { interpretation: string; /** Lag in days applied (always 1 here). */ lagDays: number; + /** + * v1.22 — which window surfaced the pair. `retrospective` = the standard + * 180-day scan (the established pattern). `recent` = the rolling + * early-detection pass over the trailing {@link EARLY_WINDOW_DAYS} days, + * surfaced ONLY when the pair is not already established retrospectively. + * Absent on the legacy path so existing consumers read it as the + * retrospective default. + */ + window?: "retrospective" | "recent"; + /** + * v1.22 — true for a `recent`-window emerging pair: it cleared a tighter + * FDR and a no-faint effect bar over a short window, but rests on fewer days + * than the retrospective scan, so the narration must hedge it as emerging / + * provisional rather than established. + */ + provisional?: boolean; } export interface CorrelationDiscoveryResult { @@ -470,6 +486,13 @@ export const DISCOVERY_BEHAVIOURS = [ "BLOOD_GLUCOSE", "BLOOD_PRESSURE_SYS", "ACTIVITY_STEPS", + // v1.22 — sleep duration as a lag SOURCE too (it was outcome-only). This + // opens the clinically-obvious "poor sleep last night → next-day mood / HRV / + // a vital drifting" direction. The same-family guard collapses only the + // sleep→sleep self-lag (the key is its own family), so sleep stays free to + // pair cross-domain. The daily series is the same per-day collapse the outcome + // role already uses, so neither role sees a fabricated value. + "SLEEP_DURATION", // v1.21.0 (FDREXTEND) — daily adherence rate as a lag source: the high-value // "adherence dip today → a worse outcome tomorrow" direction. MEDICATION_COMPLIANCE_CHANNEL_KEY, @@ -483,6 +506,16 @@ export const DISCOVERY_OUTCOMES = [ "HEART_RATE_VARIABILITY", "RESTING_HEART_RATE", "WEIGHT", + // v1.22 — blood pressure as an OUTCOME (it was behaviour-only). This makes + // the clinically-obvious "medication adherence → next-day BP control" link + // discoverable (compliance reached HRV / RHR / weight but never BP). Both + // components are added so diastolic — previously never a candidate channel at + // all — can correlate too. The same-family guard skips every + // BP-behaviour → BP-outcome self/cross-component lag (serial autocorrelation), + // so only genuine cross-domain behaviours (compliance, mood, daylight, …) → + // BP survive. BH-FDR controls the wider pair family this opens up. + "BLOOD_PRESSURE_SYS", + "BLOOD_PRESSURE_DIA", // v1.11.5 (F3) — mood is now an OUTCOME channel too, not only a behaviour. // The discovery loop skips the MOOD→MOOD self-pair (`b.key === o.key`), so // promoting it lets the FDR scan surface "behaviour today → next-day mood" @@ -504,3 +537,364 @@ export const DISCOVERY_OUTCOMES = [ // documented here so the omission is intentional, not a gap (same posture // as the medication-compliance omission above). ] as const; + +// ─── v1.22 — rolling early-detection window ─────────────────────────────── + +/** + * v1.22 — trailing window (days) for the rolling EARLY-detection pass. + * + * The standard discovery scan is retrospective: it needs n ≥ 20 paired days, so + * a pattern only surfaces once ~3 weeks of overlap have accrued INSIDE the + * 180-day window — it explains the past well but reacts slowly to an emerging + * shift (an illness pulling HRV + sleep down together over the last fortnight). + * This shorter window re-runs the SAME engine over only the trailing days so a + * fresh, concentrated pattern can surface before the long window dilutes it. + */ +export const EARLY_WINDOW_DAYS = 21; + +/** + * v1.22 — paired-day floor for the early window. Lower than the retrospective + * `MIN_PAIRED_N` (20) because 21 days cannot reach 20 lagged pairs, but still + * high enough that Pearson is defensible. The shorter, noisier series is + * counter-balanced by a TIGHTER FDR ({@link EARLY_FDR_Q}) and a no-faint effect + * bar (see {@link discoverEmergingCorrelations}), so the early pass keeps a HIGH + * firing bar despite the smaller n. + */ +export const EARLY_MIN_PAIRS = 12; + +/** + * v1.22 — tighter Benjamini-Hochberg target for the early window (vs the + * retrospective {@link FDR_Q} = 0.10). A short window is noisier, so the early + * pass demands a stricter false-discovery rate before it will narrate an + * emerging pattern. + */ +export const EARLY_FDR_Q = 0.05; + +/** Pure: keep only the points on/after `fromDayKey` (inclusive). */ +export function filterSeriesToWindow( + series: NamedSeries[], + fromDayKey: string, +): NamedSeries[] { + return series.map((s) => ({ + ...s, + points: s.points.filter((p) => p.day >= fromDayKey), + })); +} + +/** Stable pair identity for double-count exclusion. */ +function pairKey(behaviour: string, outcome: string): string { + return `${behaviour}${outcome}`; +} + +export interface EmergingCorrelationResult { + /** + * Recent-window pairs that are NOT already established by the retrospective + * scan — the genuinely emerging signals. Each is flagged `window: "recent"` + * and `provisional: true`. + */ + emerging: DiscoveredCorrelation[]; + /** Trailing window (days) the early pass scanned. */ + windowDays: number; + /** Paired-day floor enforced for the early pass. */ + minPairs: number; + /** FDR target the early pass used. */ + fdrQ: number; + /** How many pairs were tested in the early window (honest footer). */ + pairsTested: number; +} + +/** + * v1.22 — rolling early-detection pass. + * + * Re-runs {@link discoverCorrelations} over only the trailing + * {@link EARLY_WINDOW_DAYS} days (a lower n floor, a tighter FDR), then keeps + * ONLY pairs that: + * 1. are not already in the retrospective `discovered` set (no double-count — + * an established pattern is reported once, retrospectively), and + * 2. clear a no-`faint` effect bar — over a short, noisy window a faint signal + * is too likely to be transient noise to narrate as "emerging". + * + * The early window runs its OWN Benjamini-Hochberg control across its OWN tested + * family, so FDR discipline holds independently in each pass. Pure — the caller + * passes the full series and the day key of the early-window start. + */ +export function discoverEmergingCorrelations( + fullSeries: NamedSeries[], + retrospective: CorrelationDiscoveryResult, + opts: { + recentFromDayKey: string; + windowDays?: number; + lagDays?: number; + minPairs?: number; + fdrQ?: number; + }, +): EmergingCorrelationResult { + const windowDays = opts.windowDays ?? EARLY_WINDOW_DAYS; + const minPairs = opts.minPairs ?? EARLY_MIN_PAIRS; + const fdrQ = opts.fdrQ ?? EARLY_FDR_Q; + + const recentSeries = filterSeriesToWindow(fullSeries, opts.recentFromDayKey); + const recent = discoverCorrelations(recentSeries, { + lagDays: opts.lagDays, + minPairs, + fdrQ, + }); + + const established = new Set( + retrospective.discovered.map((d) => pairKey(d.behaviour, d.outcome)), + ); + + const emerging = recent.discovered + // High firing bar for the noisier short window: drop faint-tier signals. + .filter((d) => d.tier !== "faint") + // No double-count: an already-established retrospective pair is reported + // once (retrospectively), never re-surfaced as "emerging". + .filter((d) => !established.has(pairKey(d.behaviour, d.outcome))) + .map((d) => ({ + ...d, + window: "recent" as const, + provisional: true, + })); + + return { + emerging, + windowDays, + minPairs, + fdrQ, + pairsTested: recent.pairsTested, + }; +} + +// ─── v1.22 — labs ↔ outcome correlation ─────────────────────────────────── + +/** + * v1.22 — why labs are NOT a daily-lag channel in the matrix above. + * + * Lab draws are sparse and irregular (a panel every few months), so a day-D → + * day-D+1 lagged Pearson with an n ≥ 20 paired-DAY floor is structurally + * unreachable for a biomarker — folding `LAB:*` into `DISCOVERY_*` would only + * add channels that always degrade to absent (dead candidates). The clinically + * meaningful object for a sparse marker is different: each draw vs the + * CONTEMPORANEOUS windowed mean of an outcome the marker plausibly reflects + * (HbA1c ↔ the period's mean glucose; lipids ↔ weight). That is a point-vs-window + * statistic over the DRAWS (n = number of draws), not a daily lag — so it gets + * its own FDR-controlled pass here rather than a forced, never-firing channel. + */ + +/** A single lab draw, day-keyed in the user's tz. */ +export interface LabDrawPoint { + /** Stable channel key — `LAB:`. */ + key: string; + /** Day key YYYY-MM-DD of the draw. */ + day: string; + /** Numeric reading (qualitative rows are excluded by the caller). */ + value: number; +} + +/** One discovered lab ↔ outcome association — descriptive, never causal. */ +export interface DiscoveredLabCorrelation { + /** `LAB:` channel key (display strips the prefix). */ + lab: string; + /** Outcome channel key it tracks with. */ + outcome: string; + /** Number of draws paired with a usable contemporaneous outcome window. */ + n: number; + /** Pearson r across (draw value, contemporaneous outcome window-mean). */ + r: number; + pValue: number; + qValue: number; + /** Trailing days each draw's outcome window spanned. */ + windowDays: number; + /** Conservative, descriptive interpretation — never causal. */ + interpretation: string; +} + +export interface LabCorrelationResult { + discovered: DiscoveredLabCorrelation[]; + /** Lab × outcome pairs assessed (honest footer). */ + pairsTested: number; + fdrQ: number; + /** Minimum paired-draw count enforced per pair. */ + minDraws: number; +} + +/** + * v1.22 — curated outcome channels a lab biomarker is paired against. Kept + * small and physiologically broad (the flagship clinical wins the integration + * audit names) so the lab × outcome family stays tight and BH-FDR stays honest: + * a biomarker that tracks the period's mean weight, glucose, or systolic BP. + */ +export const LAB_OUTCOME_TARGETS: readonly string[] = [ + "WEIGHT", + "BLOOD_GLUCOSE", + "BLOOD_PRESSURE_SYS", +]; + +/** Default trailing window (days) each draw's outcome mean is taken over. */ +export const LAB_OUTCOME_WINDOW_DAYS = 28; + +/** Default minimum draws (paired with a usable outcome window) per lab pair. */ +export const LAB_MIN_DRAWS = 5; + +/** Default minimum outcome points inside a draw's window for it to count. */ +export const LAB_MIN_WINDOW_POINTS = 5; + +/** YYYY-MM-DD `daysBack` days before `day` (inclusive lower bound). */ +function shiftDayBack(day: string, daysBack: number): string { + return shiftDay(day, -daysBack); +} + +/** + * Mean of an outcome series over `(drawDay - windowDays, drawDay]` — the period + * the marker physiologically reflects. Returns `null` when too few outcome + * points fall in the window for a defensible mean. Pure. + */ +function outcomeWindowMean( + outcome: DailySeriesPoint[], + drawDay: string, + windowDays: number, + minPoints: number, +): number | null { + const lower = shiftDayBack(drawDay, windowDays); + let sum = 0; + let count = 0; + for (const p of outcome) { + if (p.day > lower && p.day <= drawDay && Number.isFinite(p.value)) { + sum += p.value; + count += 1; + } + } + return count >= minPoints ? sum / count : null; +} + +/** + * v1.22 — FDR-controlled labs ↔ outcome discovery. + * + * For each biomarker (grouped by `LabDrawPoint.key`) and each + * {@link LAB_OUTCOME_TARGETS} outcome, pair every draw value with the + * contemporaneous trailing-window mean of that outcome, then run Pearson + the + * exact p-value across the draws. Benjamini-Hochberg controls the lab × outcome + * family; the same effect-size floor + James-Stein shrinkage as the daily engine + * gate the survivors. A sparse marker simply fails the `minDraws` floor and + * degrades to absent — it never fabricates a link. Pure over already-fetched + * draws + outcome series. + */ +export function discoverLabOutcomeCorrelations( + labDraws: LabDrawPoint[], + series: NamedSeries[], + opts: { + windowDays?: number; + minDraws?: number; + minWindowPoints?: number; + fdrQ?: number; + } = {}, +): LabCorrelationResult { + const windowDays = opts.windowDays ?? LAB_OUTCOME_WINDOW_DAYS; + const minDraws = opts.minDraws ?? LAB_MIN_DRAWS; + const minWindowPoints = opts.minWindowPoints ?? LAB_MIN_WINDOW_POINTS; + const fdrQ = opts.fdrQ ?? FDR_Q; + + // Group draws by biomarker key (newest-irrelevant; Pearson is order-free). + const drawsByLab = new Map(); + for (const d of labDraws) { + if (!Number.isFinite(d.value)) continue; + const list = drawsByLab.get(d.key) ?? []; + list.push(d); + drawsByLab.set(d.key, list); + } + + // Index the curated outcome series by key (last-write keeps a single series + // per key even when a channel appears in both roles). + const outcomeByKey = new Map(); + for (const s of series) { + if (LAB_OUTCOME_TARGETS.includes(s.key)) outcomeByKey.set(s.key, s.points); + } + + interface RawLabPair { + lab: string; + outcome: string; + n: number; + r: number; + pValue: number; + } + const tested: RawLabPair[] = []; + for (const [lab, draws] of drawsByLab) { + for (const outcomeKey of LAB_OUTCOME_TARGETS) { + const outcome = outcomeByKey.get(outcomeKey); + if (!outcome || outcome.length === 0) continue; + const xs: number[] = []; + const ys: number[] = []; + for (const draw of draws) { + const mean = outcomeWindowMean( + outcome, + draw.day, + windowDays, + minWindowPoints, + ); + if (mean === null) continue; + xs.push(draw.value); + ys.push(mean); + } + if (xs.length < minDraws) continue; + const result = pearson({ xs, ys, minPairs: minDraws }); + if (result.status !== "ok") continue; + tested.push({ + lab, + outcome: outcomeKey, + n: result.n, + r: result.r, + pValue: result.pValue, + }); + } + } + + const pairsTested = tested.length; + if (pairsTested === 0) { + return { discovered: [], pairsTested: 0, fdrQ, minDraws }; + } + + const qValues = benjaminiHochberg(tested.map((t) => t.pValue)); + const discovered: DiscoveredLabCorrelation[] = tested + .map((t, i) => ({ ...t, qValue: qValues[i] })) + .filter((t) => t.pValue < 0.05 && t.qValue <= fdrQ) + .map((t) => { + const shrunkR = shrinkEstimate(t.r, t.n); + const tier = confidenceTier(shrunkR, t.n); + return { ...t, shrunkR, tier }; + }) + .filter( + ( + t, + ): t is RawLabPair & { + qValue: number; + shrunkR: number; + tier: ConfidenceTier; + } => t.tier !== null, + ) + .map((t) => ({ + lab: t.lab, + outcome: t.outcome, + n: t.n, + r: t.r, + pValue: t.pValue, + qValue: Math.round(t.qValue * 1000) / 1000, + windowDays, + interpretation: interpretLab(t.lab, t.outcome, t.r), + })) + .sort((a, b) => Math.abs(b.r) - Math.abs(a.r) || a.qValue - b.qValue); + + return { discovered, pairsTested, fdrQ, minDraws }; +} + +/** Strip the `LAB:` prefix for display. */ +function humaniseLab(key: string): string { + return key.startsWith("LAB:") ? key.slice("LAB:".length) : key; +} + +/** Descriptive, never-causal interpretation for a lab ↔ outcome pair. */ +function interpretLab(lab: string, outcome: string, r: number): string { + const l = humaniseLab(lab); + const o = humanise(outcome); + const dir = r < 0 ? "lower" : "higher"; + return `Higher ${l} readings line up with ${dir} ${o} over the same periods in your data — an association worth watching with your clinician, never a cause.`; +}