feat(insights): early-detection window and new correlation linkages

Add a rolling 14-21 day early-detection pass to the FDR correlation engine
so an emerging shift (illness pulling HRV and sleep down together) surfaces
before the 180-day retrospective scan accrues the n>=20 it needs. The recent
pass re-runs the same engine over the trailing window with a lower paired-day
floor, a tighter Benjamini-Hochberg target, and a no-faint effect bar, then
drops any pair the retrospective scan already established so a pattern is never
reported as both established and emerging.

Extend the curated behaviour x outcome matrix with three clinically meaningful
links: medication adherence -> next-day blood pressure (systolic and diastolic,
the latter previously never a candidate channel), sleep as a behaviour
(poor sleep -> next-day mood / HRV), and a separate point-vs-window labs ->
outcome pass (each draw against the contemporaneous outcome mean) since sparse
lab draws cannot meet a daily-lag floor. Every pass keeps FDR control, the
effect-size floor, and shrinkage; thin data degrades to absent, never a
fabricated link. All associations stay descriptive, never causal.
This commit is contained in:
Marc-André Bombeck committed 2026-06-27 13:34:30 +02:00
1 parent 7d3c528157
commit 138211b46a
3 files changed
+720

No files matched your search

@@ -529,3 +529,253 @@ describe("compliance + symptom channels (FDREXTEND)", () => {
expect(pair!.r).toBeLessThan(0); // higher adherence → lower next-day RHR
});
});
// ─── v1.22 — new linkages + early detection + labs ────────────────────────
import {
DISCOVERY_BEHAVIOURS as MATRIX_BEHAVIOURS,
DISCOVERY_OUTCOMES as MATRIX_OUTCOMES,
discoverEmergingCorrelations,
discoverLabOutcomeCorrelations,
filterSeriesToWindow,
EARLY_WINDOW_DAYS,
type LabDrawPoint,
} from "../correlation-discovery";
/** Contiguous daily series starting `startISO` (YYYY-MM-DD). */
function seriesFrom(startISO: string, values: number[]): DailySeriesPoint[] {
const [y, m, d] = startISO.split("-").map(Number);
const start = Date.UTC(y, m - 1, d);
return values.map((value, i) => {
const dt = new Date(start + i * 24 * 60 * 60 * 1000);
return { day: dt.toISOString().slice(0, 10), value };
});
}
describe("v1.22 — new curated linkages", () => {
it("adds sleep as a behaviour (was outcome-only)", () => {
expect(MATRIX_BEHAVIOURS).toContain("SLEEP_DURATION");
});
it("adds blood pressure (both components) as outcomes", () => {
expect(MATRIX_OUTCOMES).toContain("BLOOD_PRESSURE_SYS");
expect(MATRIX_OUTCOMES).toContain("BLOOD_PRESSURE_DIA");
});
it("surfaces compliance→BP_SYS but skips the same-family BP→BP self-cross", () => {
const n = 80;
const adherence = Array.from({ length: n }, (_, i) =>
i % 2 === 0 ? 100 : 50,
);
// Higher adherence today → lower next-day systolic.
const sys = [
130,
...adherence
.slice(0, n - 1)
.map((a, i) => 160 - a / 4 + (i % 2 ? 0.2 : -0.2)),
];
const result = discoverCorrelations([
{
key: MEDICATION_COMPLIANCE_CHANNEL_KEY,
role: "behaviour",
points: longSeries(adherence),
},
// BP is both a behaviour and an outcome in the real matrix; the
// same-family guard must keep BP_SYS(behaviour)→BP_SYS(outcome) out.
{ key: "BLOOD_PRESSURE_SYS", role: "behaviour", points: longSeries(sys) },
{ key: "BLOOD_PRESSURE_SYS", role: "outcome", points: longSeries(sys) },
]);
expect(
result.discovered.find(
(p) =>
p.behaviour === "BLOOD_PRESSURE_SYS" &&
p.outcome === "BLOOD_PRESSURE_SYS",
),
).toBeUndefined();
const link = result.discovered.find(
(p) =>
p.behaviour === MEDICATION_COMPLIANCE_CHANNEL_KEY &&
p.outcome === "BLOOD_PRESSURE_SYS",
);
expect(link).toBeDefined();
expect(link!.r).toBeLessThan(0);
});
});
describe("discoverEmergingCorrelations (early detection)", () => {
it("filterSeriesToWindow keeps only on/after the cutoff", () => {
const s: NamedSeries[] = [
{
key: "MOOD",
role: "behaviour",
points: seriesFrom("2026-03-01", [1, 2, 3, 4, 5]),
},
];
const filtered = filterSeriesToWindow(s, "2026-03-03");
expect(filtered[0].points.map((p) => p.day)).toEqual([
"2026-03-03",
"2026-03-04",
"2026-03-05",
]);
});
it("surfaces a fortnight-long signal the retrospective n≥20 floor cannot yet see", () => {
// Only ~18 days of data: a clean behaviour→next-day-outcome coupling. The
// retrospective scan needs ≥ 20 lagged pairs, so it sees NOTHING; the early
// window (floor 10) catches it — exactly the early-detection mandate.
const len = 18;
const beh = Array.from({ length: len }, (_, i) => (i % 2 ? 2 : 9));
const out = beh.map((b, i) => 40 + b * 3 + (i % 2 ? 0.3 : -0.3));
const series: NamedSeries[] = [
{ key: "MOOD", role: "behaviour", points: longSeries(beh) },
{
key: "HEART_RATE_VARIABILITY",
role: "outcome",
points: longSeries(out),
},
];
const retrospective = discoverCorrelations(series);
expect(retrospective.discovered).toHaveLength(0); // n too low for the floor
const cutoff = longSeries(beh)[0].day; // whole (short) window
const { emerging } = discoverEmergingCorrelations(series, retrospective, {
recentFromDayKey: cutoff,
minPairs: 10,
});
const pair = emerging.find(
(p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY",
);
expect(pair).toBeDefined();
expect(pair!.window).toBe("recent");
expect(pair!.provisional).toBe(true);
});
it("does not double-count a pair already established retrospectively", () => {
// A strong relationship across the WHOLE 80-day window → retrospective finds
// it; the emerging pass must NOT re-surface it.
const n = 80;
const beh = Array.from({ length: n }, (_, i) => (i % 2 ? 2 : 9));
const out = beh.map((b, i) => 40 + b * 3 + (i % 2 ? 0.3 : -0.3));
const series: NamedSeries[] = [
{ key: "MOOD", role: "behaviour", points: longSeries(beh) },
{
key: "HEART_RATE_VARIABILITY",
role: "outcome",
points: longSeries(out),
},
];
const retrospective = discoverCorrelations(series);
expect(
retrospective.discovered.some(
(p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY",
),
).toBe(true);
const cutoff = new Date(
Date.UTC(2026, 0, 1) + (n - EARLY_WINDOW_DAYS) * 24 * 60 * 60 * 1000,
)
.toISOString()
.slice(0, 10);
const { emerging } = discoverEmergingCorrelations(series, retrospective, {
recentFromDayKey: cutoff,
minPairs: 10,
});
expect(
emerging.find(
(p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY",
),
).toBeUndefined();
});
it("returns nothing on pure noise (FDR + high bar hold in the short window)", () => {
const n = 30;
const beh = Array.from({ length: n }, (_, i) => (i * 7) % 11);
const out = Array.from({ length: n }, (_, i) => (i * 3) % 5);
const series: NamedSeries[] = [
{ key: "MOOD", role: "behaviour", points: longSeries(beh) },
{
key: "HEART_RATE_VARIABILITY",
role: "outcome",
points: longSeries(out),
},
];
const retrospective = discoverCorrelations(series);
const cutoff = longSeries(beh)[Math.max(0, n - EARLY_WINDOW_DAYS)].day;
const { emerging } = discoverEmergingCorrelations(series, retrospective, {
recentFromDayKey: cutoff,
minPairs: 10,
});
expect(emerging).toHaveLength(0);
});
});
describe("discoverLabOutcomeCorrelations (labs ↔ outcomes)", () => {
function weightSeries(): NamedSeries {
// Daily weight that drifts up over ~6 months.
const values = Array.from({ length: 180 }, (_, i) => 80 + i * 0.05);
return {
key: "WEIGHT",
role: "outcome",
points: seriesFrom("2026-01-01", values),
};
}
it("surfaces a lab that tracks the contemporaneous outcome window-mean", () => {
const weight = weightSeries();
// 8 monthly-ish HbA1c-like draws that rise with weight.
const draws: LabDrawPoint[] = [];
for (let i = 0; i < 8; i++) {
const dayIdx = 30 + i * 18; // spread draws across the window
const day = weight.points[dayIdx].day;
// Value tracks the period's weight mean (with tiny jitter).
const v = 5.0 + dayIdx * 0.01 + (i % 2 ? 0.02 : -0.02);
draws.push({ key: "LAB:HbA1c", day, value: v });
}
const result = discoverLabOutcomeCorrelations(draws, [weight], {
minDraws: 5,
minWindowPoints: 5,
});
const pair = result.discovered.find(
(d) => d.lab === "LAB:HbA1c" && d.outcome === "WEIGHT",
);
expect(pair).toBeDefined();
expect(pair!.n).toBeGreaterThanOrEqual(5);
expect(pair!.r).toBeGreaterThan(0);
expect(pair!.qValue).toBeLessThanOrEqual(result.fdrQ);
expect(pair!.interpretation).toMatch(/never a cause/i);
});
it("degrades to absent when the marker has too few draws", () => {
const weight = weightSeries();
const draws: LabDrawPoint[] = [0, 1, 2].map((i) => ({
key: "LAB:HbA1c",
day: weight.points[30 + i * 18].day,
value: 5 + i,
}));
const result = discoverLabOutcomeCorrelations(draws, [weight], {
minDraws: 5,
});
expect(result.discovered).toHaveLength(0);
expect(result.pairsTested).toBe(0);
});
it("ignores outcomes outside the curated lab-target set", () => {
const hrv: NamedSeries = {
key: "HEART_RATE_VARIABILITY",
role: "outcome",
points: seriesFrom(
"2026-01-01",
Array.from({ length: 180 }, (_, i) => 50 + i * 0.1),
),
};
const draws: LabDrawPoint[] = Array.from({ length: 8 }, (_, i) => ({
key: "LAB:HbA1c",
day: hrv.points[30 + i * 18].day,
value: 5 + i,
}));
const result = discoverLabOutcomeCorrelations(draws, [hrv], {
minDraws: 5,
});
// HRV is not a curated lab-outcome target → no pair tested.
expect(result.pairsTested).toBe(0);
});
});
@@ -24,8 +24,11 @@ import {
import {
MEDICATION_COMPLIANCE_CHANNEL_KEY,
SYMPTOM_SEVERITY_CHANNEL_KEY,
type LabDrawPoint,
type NamedSeries,
} from "@/lib/insights/correlation-discovery";
import { resolveLabFields } from "@/lib/labs/serialise";
import { wallClockInTz } from "@/lib/tz/wall-clock";
import {
buildComplianceDailySeries,
buildSymptomSeverityDailySeries,
@@ -177,3 +180,76 @@ export async function fetchSymptomSeries(
role: "outcome",
});
}
/** Day key (YYYY-MM-DD) for an instant in the user's display timezone. */
function tzDayKey(at: Date, tz: string): string {
const { year, month, day } = wallClockInTz(at, tz);
return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`;
}
/**
* v1.22 — build the user's lab draws for the labs ↔ outcome correlation pass.
*
* One {@link LabDrawPoint} per QUANTITATIVE reading in the window, keyed
* `LAB:<canonical analyte>` (the resolved name, so two spellings of one marker
* collapse). HIDDEN biomarkers are excluded (the W3 catalog `hidden` flag — a
* marker the user retired from the active list must not silently re-enter an
* analysis surface). Qualitative readings (no numeric `value`) and rows whose
* resolved value is non-finite are dropped — there is nothing to correlate.
* The encrypted note column is never selected.
*
* Returns an EMPTY array when the user has no usable readings, so the discovery
* pass degrades to absent rather than fabricating a link.
*/
export async function fetchLabDraws(
userId: string,
tz: string,
since: Date,
): Promise<LabDrawPoint[]> {
const now = new Date();
const rows = await prisma.labResult.findMany({
where: {
userId,
deletedAt: null,
value: { not: null },
takenAt: { gte: since, lte: now },
},
orderBy: { takenAt: "asc" },
take: 5000,
select: {
analyte: true,
unit: true,
referenceLow: true,
referenceHigh: true,
panel: true,
value: true,
takenAt: true,
biomarkerId: true,
biomarker: {
select: {
id: true,
name: true,
unit: true,
lowerBound: true,
upperBound: true,
panel: true,
hidden: true,
},
},
},
});
const draws: LabDrawPoint[] = [];
for (const row of rows) {
// Exclude retired markers (W3 hidden flag); unlinked rows cannot be hidden.
if (row.biomarker?.hidden) continue;
if (row.value === null || !Number.isFinite(row.value)) continue;
const resolved = resolveLabFields(row, row.biomarker);
draws.push({
key: `LAB:${resolved.analyte}`,
day: tzDayKey(row.takenAt, tz),
value: row.value,
});
}
return draws;
}
+394
View File
@@ -179,6 +179,22 @@ export interface DiscoveredCorrelation {
interpretation: string;
/** Lag in days applied (always 1 here). */
lagDays: number;
/**
* v1.22 — which window surfaced the pair. `retrospective` = the standard
* 180-day scan (the established pattern). `recent` = the rolling
* early-detection pass over the trailing {@link EARLY_WINDOW_DAYS} days,
* surfaced ONLY when the pair is not already established retrospectively.
* Absent on the legacy path so existing consumers read it as the
* retrospective default.
*/
window?: "retrospective" | "recent";
/**
* v1.22 — true for a `recent`-window emerging pair: it cleared a tighter
* FDR and a no-faint effect bar over a short window, but rests on fewer days
* than the retrospective scan, so the narration must hedge it as emerging /
* provisional rather than established.
*/
provisional?: boolean;
}
export interface CorrelationDiscoveryResult {
@@ -470,6 +486,13 @@ export const DISCOVERY_BEHAVIOURS = [
"BLOOD_GLUCOSE",
"BLOOD_PRESSURE_SYS",
"ACTIVITY_STEPS",
// v1.22 — sleep duration as a lag SOURCE too (it was outcome-only). This
// opens the clinically-obvious "poor sleep last night → next-day mood / HRV /
// a vital drifting" direction. The same-family guard collapses only the
// sleep→sleep self-lag (the key is its own family), so sleep stays free to
// pair cross-domain. The daily series is the same per-day collapse the outcome
// role already uses, so neither role sees a fabricated value.
"SLEEP_DURATION",
// v1.21.0 (FDREXTEND) — daily adherence rate as a lag source: the high-value
// "adherence dip today → a worse outcome tomorrow" direction.
MEDICATION_COMPLIANCE_CHANNEL_KEY,
@@ -483,6 +506,16 @@ export const DISCOVERY_OUTCOMES = [
"HEART_RATE_VARIABILITY",
"RESTING_HEART_RATE",
"WEIGHT",
// v1.22 — blood pressure as an OUTCOME (it was behaviour-only). This makes
// the clinically-obvious "medication adherence → next-day BP control" link
// discoverable (compliance reached HRV / RHR / weight but never BP). Both
// components are added so diastolic — previously never a candidate channel at
// all — can correlate too. The same-family guard skips every
// BP-behaviour → BP-outcome self/cross-component lag (serial autocorrelation),
// so only genuine cross-domain behaviours (compliance, mood, daylight, …) →
// BP survive. BH-FDR controls the wider pair family this opens up.
"BLOOD_PRESSURE_SYS",
"BLOOD_PRESSURE_DIA",
// v1.11.5 (F3) — mood is now an OUTCOME channel too, not only a behaviour.
// The discovery loop skips the MOOD→MOOD self-pair (`b.key === o.key`), so
// promoting it lets the FDR scan surface "behaviour today → next-day mood"
@@ -504,3 +537,364 @@ export const DISCOVERY_OUTCOMES = [
// documented here so the omission is intentional, not a gap (same posture
// as the medication-compliance omission above).
] as const;
// ─── v1.22 — rolling early-detection window ───────────────────────────────
/**
* v1.22 — trailing window (days) for the rolling EARLY-detection pass.
*
* The standard discovery scan is retrospective: it needs n ≥ 20 paired days, so
* a pattern only surfaces once ~3 weeks of overlap have accrued INSIDE the
* 180-day window — it explains the past well but reacts slowly to an emerging
* shift (an illness pulling HRV + sleep down together over the last fortnight).
* This shorter window re-runs the SAME engine over only the trailing days so a
* fresh, concentrated pattern can surface before the long window dilutes it.
*/
export const EARLY_WINDOW_DAYS = 21;
/**
* v1.22 — paired-day floor for the early window. Lower than the retrospective
* `MIN_PAIRED_N` (20) because 21 days cannot reach 20 lagged pairs, but still
* high enough that Pearson is defensible. The shorter, noisier series is
* counter-balanced by a TIGHTER FDR ({@link EARLY_FDR_Q}) and a no-faint effect
* bar (see {@link discoverEmergingCorrelations}), so the early pass keeps a HIGH
* firing bar despite the smaller n.
*/
export const EARLY_MIN_PAIRS = 12;
/**
* v1.22 — tighter Benjamini-Hochberg target for the early window (vs the
* retrospective {@link FDR_Q} = 0.10). A short window is noisier, so the early
* pass demands a stricter false-discovery rate before it will narrate an
* emerging pattern.
*/
export const EARLY_FDR_Q = 0.05;
/** Pure: keep only the points on/after `fromDayKey` (inclusive). */
export function filterSeriesToWindow(
series: NamedSeries[],
fromDayKey: string,
): NamedSeries[] {
return series.map((s) => ({
...s,
points: s.points.filter((p) => p.day >= fromDayKey),
}));
}
/** Stable pair identity for double-count exclusion. */
function pairKey(behaviour: string, outcome: string): string {
return `${behaviour}${outcome}`;
}
export interface EmergingCorrelationResult {
/**
* Recent-window pairs that are NOT already established by the retrospective
* scan — the genuinely emerging signals. Each is flagged `window: "recent"`
* and `provisional: true`.
*/
emerging: DiscoveredCorrelation[];
/** Trailing window (days) the early pass scanned. */
windowDays: number;
/** Paired-day floor enforced for the early pass. */
minPairs: number;
/** FDR target the early pass used. */
fdrQ: number;
/** How many pairs were tested in the early window (honest footer). */
pairsTested: number;
}
/**
* v1.22 — rolling early-detection pass.
*
* Re-runs {@link discoverCorrelations} over only the trailing
* {@link EARLY_WINDOW_DAYS} days (a lower n floor, a tighter FDR), then keeps
* ONLY pairs that:
* 1. are not already in the retrospective `discovered` set (no double-count —
* an established pattern is reported once, retrospectively), and
* 2. clear a no-`faint` effect bar — over a short, noisy window a faint signal
* is too likely to be transient noise to narrate as "emerging".
*
* The early window runs its OWN Benjamini-Hochberg control across its OWN tested
* family, so FDR discipline holds independently in each pass. Pure — the caller
* passes the full series and the day key of the early-window start.
*/
export function discoverEmergingCorrelations(
fullSeries: NamedSeries[],
retrospective: CorrelationDiscoveryResult,
opts: {
recentFromDayKey: string;
windowDays?: number;
lagDays?: number;
minPairs?: number;
fdrQ?: number;
},
): EmergingCorrelationResult {
const windowDays = opts.windowDays ?? EARLY_WINDOW_DAYS;
const minPairs = opts.minPairs ?? EARLY_MIN_PAIRS;
const fdrQ = opts.fdrQ ?? EARLY_FDR_Q;
const recentSeries = filterSeriesToWindow(fullSeries, opts.recentFromDayKey);
const recent = discoverCorrelations(recentSeries, {
lagDays: opts.lagDays,
minPairs,
fdrQ,
});
const established = new Set(
retrospective.discovered.map((d) => pairKey(d.behaviour, d.outcome)),
);
const emerging = recent.discovered
// High firing bar for the noisier short window: drop faint-tier signals.
.filter((d) => d.tier !== "faint")
// No double-count: an already-established retrospective pair is reported
// once (retrospectively), never re-surfaced as "emerging".
.filter((d) => !established.has(pairKey(d.behaviour, d.outcome)))
.map((d) => ({
...d,
window: "recent" as const,
provisional: true,
}));
return {
emerging,
windowDays,
minPairs,
fdrQ,
pairsTested: recent.pairsTested,
};
}
// ─── v1.22 — labs ↔ outcome correlation ───────────────────────────────────
/**
* v1.22 — why labs are NOT a daily-lag channel in the matrix above.
*
* Lab draws are sparse and irregular (a panel every few months), so a day-D →
* day-D+1 lagged Pearson with an n ≥ 20 paired-DAY floor is structurally
* unreachable for a biomarker — folding `LAB:*` into `DISCOVERY_*` would only
* add channels that always degrade to absent (dead candidates). The clinically
* meaningful object for a sparse marker is different: each draw vs the
* CONTEMPORANEOUS windowed mean of an outcome the marker plausibly reflects
* (HbA1c ↔ the period's mean glucose; lipids ↔ weight). That is a point-vs-window
* statistic over the DRAWS (n = number of draws), not a daily lag — so it gets
* its own FDR-controlled pass here rather than a forced, never-firing channel.
*/
/** A single lab draw, day-keyed in the user's tz. */
export interface LabDrawPoint {
/** Stable channel key — `LAB:<analyte>`. */
key: string;
/** Day key YYYY-MM-DD of the draw. */
day: string;
/** Numeric reading (qualitative rows are excluded by the caller). */
value: number;
}
/** One discovered lab ↔ outcome association — descriptive, never causal. */
export interface DiscoveredLabCorrelation {
/** `LAB:<analyte>` channel key (display strips the prefix). */
lab: string;
/** Outcome channel key it tracks with. */
outcome: string;
/** Number of draws paired with a usable contemporaneous outcome window. */
n: number;
/** Pearson r across (draw value, contemporaneous outcome window-mean). */
r: number;
pValue: number;
qValue: number;
/** Trailing days each draw's outcome window spanned. */
windowDays: number;
/** Conservative, descriptive interpretation — never causal. */
interpretation: string;
}
export interface LabCorrelationResult {
discovered: DiscoveredLabCorrelation[];
/** Lab × outcome pairs assessed (honest footer). */
pairsTested: number;
fdrQ: number;
/** Minimum paired-draw count enforced per pair. */
minDraws: number;
}
/**
* v1.22 — curated outcome channels a lab biomarker is paired against. Kept
* small and physiologically broad (the flagship clinical wins the integration
* audit names) so the lab × outcome family stays tight and BH-FDR stays honest:
* a biomarker that tracks the period's mean weight, glucose, or systolic BP.
*/
export const LAB_OUTCOME_TARGETS: readonly string[] = [
"WEIGHT",
"BLOOD_GLUCOSE",
"BLOOD_PRESSURE_SYS",
];
/** Default trailing window (days) each draw's outcome mean is taken over. */
export const LAB_OUTCOME_WINDOW_DAYS = 28;
/** Default minimum draws (paired with a usable outcome window) per lab pair. */
export const LAB_MIN_DRAWS = 5;
/** Default minimum outcome points inside a draw's window for it to count. */
export const LAB_MIN_WINDOW_POINTS = 5;
/** YYYY-MM-DD `daysBack` days before `day` (inclusive lower bound). */
function shiftDayBack(day: string, daysBack: number): string {
return shiftDay(day, -daysBack);
}
/**
* Mean of an outcome series over `(drawDay - windowDays, drawDay]` — the period
* the marker physiologically reflects. Returns `null` when too few outcome
* points fall in the window for a defensible mean. Pure.
*/
function outcomeWindowMean(
outcome: DailySeriesPoint[],
drawDay: string,
windowDays: number,
minPoints: number,
): number | null {
const lower = shiftDayBack(drawDay, windowDays);
let sum = 0;
let count = 0;
for (const p of outcome) {
if (p.day > lower && p.day <= drawDay && Number.isFinite(p.value)) {
sum += p.value;
count += 1;
}
}
return count >= minPoints ? sum / count : null;
}
/**
* v1.22 — FDR-controlled labs ↔ outcome discovery.
*
* For each biomarker (grouped by `LabDrawPoint.key`) and each
* {@link LAB_OUTCOME_TARGETS} outcome, pair every draw value with the
* contemporaneous trailing-window mean of that outcome, then run Pearson + the
* exact p-value across the draws. Benjamini-Hochberg controls the lab × outcome
* family; the same effect-size floor + James-Stein shrinkage as the daily engine
* gate the survivors. A sparse marker simply fails the `minDraws` floor and
* degrades to absent — it never fabricates a link. Pure over already-fetched
* draws + outcome series.
*/
export function discoverLabOutcomeCorrelations(
labDraws: LabDrawPoint[],
series: NamedSeries[],
opts: {
windowDays?: number;
minDraws?: number;
minWindowPoints?: number;
fdrQ?: number;
} = {},
): LabCorrelationResult {
const windowDays = opts.windowDays ?? LAB_OUTCOME_WINDOW_DAYS;
const minDraws = opts.minDraws ?? LAB_MIN_DRAWS;
const minWindowPoints = opts.minWindowPoints ?? LAB_MIN_WINDOW_POINTS;
const fdrQ = opts.fdrQ ?? FDR_Q;
// Group draws by biomarker key (newest-irrelevant; Pearson is order-free).
const drawsByLab = new Map<string, LabDrawPoint[]>();
for (const d of labDraws) {
if (!Number.isFinite(d.value)) continue;
const list = drawsByLab.get(d.key) ?? [];
list.push(d);
drawsByLab.set(d.key, list);
}
// Index the curated outcome series by key (last-write keeps a single series
// per key even when a channel appears in both roles).
const outcomeByKey = new Map<string, DailySeriesPoint[]>();
for (const s of series) {
if (LAB_OUTCOME_TARGETS.includes(s.key)) outcomeByKey.set(s.key, s.points);
}
interface RawLabPair {
lab: string;
outcome: string;
n: number;
r: number;
pValue: number;
}
const tested: RawLabPair[] = [];
for (const [lab, draws] of drawsByLab) {
for (const outcomeKey of LAB_OUTCOME_TARGETS) {
const outcome = outcomeByKey.get(outcomeKey);
if (!outcome || outcome.length === 0) continue;
const xs: number[] = [];
const ys: number[] = [];
for (const draw of draws) {
const mean = outcomeWindowMean(
outcome,
draw.day,
windowDays,
minWindowPoints,
);
if (mean === null) continue;
xs.push(draw.value);
ys.push(mean);
}
if (xs.length < minDraws) continue;
const result = pearson({ xs, ys, minPairs: minDraws });
if (result.status !== "ok") continue;
tested.push({
lab,
outcome: outcomeKey,
n: result.n,
r: result.r,
pValue: result.pValue,
});
}
}
const pairsTested = tested.length;
if (pairsTested === 0) {
return { discovered: [], pairsTested: 0, fdrQ, minDraws };
}
const qValues = benjaminiHochberg(tested.map((t) => t.pValue));
const discovered: DiscoveredLabCorrelation[] = tested
.map((t, i) => ({ ...t, qValue: qValues[i] }))
.filter((t) => t.pValue < 0.05 && t.qValue <= fdrQ)
.map((t) => {
const shrunkR = shrinkEstimate(t.r, t.n);
const tier = confidenceTier(shrunkR, t.n);
return { ...t, shrunkR, tier };
})
.filter(
(
t,
): t is RawLabPair & {
qValue: number;
shrunkR: number;
tier: ConfidenceTier;
} => t.tier !== null,
)
.map((t) => ({
lab: t.lab,
outcome: t.outcome,
n: t.n,
r: t.r,
pValue: t.pValue,
qValue: Math.round(t.qValue * 1000) / 1000,
windowDays,
interpretation: interpretLab(t.lab, t.outcome, t.r),
}))
.sort((a, b) => Math.abs(b.r) - Math.abs(a.r) || a.qValue - b.qValue);
return { discovered, pairsTested, fdrQ, minDraws };
}
/** Strip the `LAB:` prefix for display. */
function humaniseLab(key: string): string {
return key.startsWith("LAB:") ? key.slice("LAB:".length) : key;
}
/** Descriptive, never-causal interpretation for a lab ↔ outcome pair. */
function interpretLab(lab: string, outcome: string, r: number): string {
const l = humaniseLab(lab);
const o = humanise(outcome);
const dir = r < 0 ? "lower" : "higher";
return `Higher ${l} readings line up with ${dir} ${o} over the same periods in your data — an association worth watching with your clinician, never a cause.`;
}