mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-11 05:42:19 +00:00
feat(insights): early-detection window and new correlation linkages
Add a rolling 14-21 day early-detection pass to the FDR correlation engine so an emerging shift (illness pulling HRV and sleep down together) surfaces before the 180-day retrospective scan accrues the n>=20 it needs. The recent pass re-runs the same engine over the trailing window with a lower paired-day floor, a tighter Benjamini-Hochberg target, and a no-faint effect bar, then drops any pair the retrospective scan already established so a pattern is never reported as both established and emerging. Extend the curated behaviour x outcome matrix with three clinically meaningful links: medication adherence -> next-day blood pressure (systolic and diastolic, the latter previously never a candidate channel), sleep as a behaviour (poor sleep -> next-day mood / HRV), and a separate point-vs-window labs -> outcome pass (each draw against the contemporaneous outcome mean) since sparse lab draws cannot meet a daily-lag floor. Every pass keeps FDR control, the effect-size floor, and shrinkage; thin data degrades to absent, never a fabricated link. All associations stay descriptive, never causal.
This commit is contained in:
1 parent
7d3c528157
commit
138211b46a
3 files changed
+720
No files matched your search
@@ -529,3 +529,253 @@ describe("compliance + symptom channels (FDREXTEND)", () => {
|
||||
expect(pair!.r).toBeLessThan(0); // higher adherence → lower next-day RHR
|
||||
});
|
||||
});
|
||||
|
||||
// ─── v1.22 — new linkages + early detection + labs ────────────────────────
|
||||
|
||||
import {
|
||||
DISCOVERY_BEHAVIOURS as MATRIX_BEHAVIOURS,
|
||||
DISCOVERY_OUTCOMES as MATRIX_OUTCOMES,
|
||||
discoverEmergingCorrelations,
|
||||
discoverLabOutcomeCorrelations,
|
||||
filterSeriesToWindow,
|
||||
EARLY_WINDOW_DAYS,
|
||||
type LabDrawPoint,
|
||||
} from "../correlation-discovery";
|
||||
|
||||
/** Contiguous daily series starting `startISO` (YYYY-MM-DD). */
|
||||
function seriesFrom(startISO: string, values: number[]): DailySeriesPoint[] {
|
||||
const [y, m, d] = startISO.split("-").map(Number);
|
||||
const start = Date.UTC(y, m - 1, d);
|
||||
return values.map((value, i) => {
|
||||
const dt = new Date(start + i * 24 * 60 * 60 * 1000);
|
||||
return { day: dt.toISOString().slice(0, 10), value };
|
||||
});
|
||||
}
|
||||
|
||||
describe("v1.22 — new curated linkages", () => {
|
||||
it("adds sleep as a behaviour (was outcome-only)", () => {
|
||||
expect(MATRIX_BEHAVIOURS).toContain("SLEEP_DURATION");
|
||||
});
|
||||
|
||||
it("adds blood pressure (both components) as outcomes", () => {
|
||||
expect(MATRIX_OUTCOMES).toContain("BLOOD_PRESSURE_SYS");
|
||||
expect(MATRIX_OUTCOMES).toContain("BLOOD_PRESSURE_DIA");
|
||||
});
|
||||
|
||||
it("surfaces compliance→BP_SYS but skips the same-family BP→BP self-cross", () => {
|
||||
const n = 80;
|
||||
const adherence = Array.from({ length: n }, (_, i) =>
|
||||
i % 2 === 0 ? 100 : 50,
|
||||
);
|
||||
// Higher adherence today → lower next-day systolic.
|
||||
const sys = [
|
||||
130,
|
||||
...adherence
|
||||
.slice(0, n - 1)
|
||||
.map((a, i) => 160 - a / 4 + (i % 2 ? 0.2 : -0.2)),
|
||||
];
|
||||
const result = discoverCorrelations([
|
||||
{
|
||||
key: MEDICATION_COMPLIANCE_CHANNEL_KEY,
|
||||
role: "behaviour",
|
||||
points: longSeries(adherence),
|
||||
},
|
||||
// BP is both a behaviour and an outcome in the real matrix; the
|
||||
// same-family guard must keep BP_SYS(behaviour)→BP_SYS(outcome) out.
|
||||
{ key: "BLOOD_PRESSURE_SYS", role: "behaviour", points: longSeries(sys) },
|
||||
{ key: "BLOOD_PRESSURE_SYS", role: "outcome", points: longSeries(sys) },
|
||||
]);
|
||||
expect(
|
||||
result.discovered.find(
|
||||
(p) =>
|
||||
p.behaviour === "BLOOD_PRESSURE_SYS" &&
|
||||
p.outcome === "BLOOD_PRESSURE_SYS",
|
||||
),
|
||||
).toBeUndefined();
|
||||
const link = result.discovered.find(
|
||||
(p) =>
|
||||
p.behaviour === MEDICATION_COMPLIANCE_CHANNEL_KEY &&
|
||||
p.outcome === "BLOOD_PRESSURE_SYS",
|
||||
);
|
||||
expect(link).toBeDefined();
|
||||
expect(link!.r).toBeLessThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("discoverEmergingCorrelations (early detection)", () => {
|
||||
it("filterSeriesToWindow keeps only on/after the cutoff", () => {
|
||||
const s: NamedSeries[] = [
|
||||
{
|
||||
key: "MOOD",
|
||||
role: "behaviour",
|
||||
points: seriesFrom("2026-03-01", [1, 2, 3, 4, 5]),
|
||||
},
|
||||
];
|
||||
const filtered = filterSeriesToWindow(s, "2026-03-03");
|
||||
expect(filtered[0].points.map((p) => p.day)).toEqual([
|
||||
"2026-03-03",
|
||||
"2026-03-04",
|
||||
"2026-03-05",
|
||||
]);
|
||||
});
|
||||
|
||||
it("surfaces a fortnight-long signal the retrospective n≥20 floor cannot yet see", () => {
|
||||
// Only ~18 days of data: a clean behaviour→next-day-outcome coupling. The
|
||||
// retrospective scan needs ≥ 20 lagged pairs, so it sees NOTHING; the early
|
||||
// window (floor 10) catches it — exactly the early-detection mandate.
|
||||
const len = 18;
|
||||
const beh = Array.from({ length: len }, (_, i) => (i % 2 ? 2 : 9));
|
||||
const out = beh.map((b, i) => 40 + b * 3 + (i % 2 ? 0.3 : -0.3));
|
||||
const series: NamedSeries[] = [
|
||||
{ key: "MOOD", role: "behaviour", points: longSeries(beh) },
|
||||
{
|
||||
key: "HEART_RATE_VARIABILITY",
|
||||
role: "outcome",
|
||||
points: longSeries(out),
|
||||
},
|
||||
];
|
||||
const retrospective = discoverCorrelations(series);
|
||||
expect(retrospective.discovered).toHaveLength(0); // n too low for the floor
|
||||
|
||||
const cutoff = longSeries(beh)[0].day; // whole (short) window
|
||||
const { emerging } = discoverEmergingCorrelations(series, retrospective, {
|
||||
recentFromDayKey: cutoff,
|
||||
minPairs: 10,
|
||||
});
|
||||
const pair = emerging.find(
|
||||
(p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY",
|
||||
);
|
||||
expect(pair).toBeDefined();
|
||||
expect(pair!.window).toBe("recent");
|
||||
expect(pair!.provisional).toBe(true);
|
||||
});
|
||||
|
||||
it("does not double-count a pair already established retrospectively", () => {
|
||||
// A strong relationship across the WHOLE 80-day window → retrospective finds
|
||||
// it; the emerging pass must NOT re-surface it.
|
||||
const n = 80;
|
||||
const beh = Array.from({ length: n }, (_, i) => (i % 2 ? 2 : 9));
|
||||
const out = beh.map((b, i) => 40 + b * 3 + (i % 2 ? 0.3 : -0.3));
|
||||
const series: NamedSeries[] = [
|
||||
{ key: "MOOD", role: "behaviour", points: longSeries(beh) },
|
||||
{
|
||||
key: "HEART_RATE_VARIABILITY",
|
||||
role: "outcome",
|
||||
points: longSeries(out),
|
||||
},
|
||||
];
|
||||
const retrospective = discoverCorrelations(series);
|
||||
expect(
|
||||
retrospective.discovered.some(
|
||||
(p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY",
|
||||
),
|
||||
).toBe(true);
|
||||
const cutoff = new Date(
|
||||
Date.UTC(2026, 0, 1) + (n - EARLY_WINDOW_DAYS) * 24 * 60 * 60 * 1000,
|
||||
)
|
||||
.toISOString()
|
||||
.slice(0, 10);
|
||||
const { emerging } = discoverEmergingCorrelations(series, retrospective, {
|
||||
recentFromDayKey: cutoff,
|
||||
minPairs: 10,
|
||||
});
|
||||
expect(
|
||||
emerging.find(
|
||||
(p) => p.behaviour === "MOOD" && p.outcome === "HEART_RATE_VARIABILITY",
|
||||
),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns nothing on pure noise (FDR + high bar hold in the short window)", () => {
|
||||
const n = 30;
|
||||
const beh = Array.from({ length: n }, (_, i) => (i * 7) % 11);
|
||||
const out = Array.from({ length: n }, (_, i) => (i * 3) % 5);
|
||||
const series: NamedSeries[] = [
|
||||
{ key: "MOOD", role: "behaviour", points: longSeries(beh) },
|
||||
{
|
||||
key: "HEART_RATE_VARIABILITY",
|
||||
role: "outcome",
|
||||
points: longSeries(out),
|
||||
},
|
||||
];
|
||||
const retrospective = discoverCorrelations(series);
|
||||
const cutoff = longSeries(beh)[Math.max(0, n - EARLY_WINDOW_DAYS)].day;
|
||||
const { emerging } = discoverEmergingCorrelations(series, retrospective, {
|
||||
recentFromDayKey: cutoff,
|
||||
minPairs: 10,
|
||||
});
|
||||
expect(emerging).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("discoverLabOutcomeCorrelations (labs ↔ outcomes)", () => {
|
||||
function weightSeries(): NamedSeries {
|
||||
// Daily weight that drifts up over ~6 months.
|
||||
const values = Array.from({ length: 180 }, (_, i) => 80 + i * 0.05);
|
||||
return {
|
||||
key: "WEIGHT",
|
||||
role: "outcome",
|
||||
points: seriesFrom("2026-01-01", values),
|
||||
};
|
||||
}
|
||||
|
||||
it("surfaces a lab that tracks the contemporaneous outcome window-mean", () => {
|
||||
const weight = weightSeries();
|
||||
// 8 monthly-ish HbA1c-like draws that rise with weight.
|
||||
const draws: LabDrawPoint[] = [];
|
||||
for (let i = 0; i < 8; i++) {
|
||||
const dayIdx = 30 + i * 18; // spread draws across the window
|
||||
const day = weight.points[dayIdx].day;
|
||||
// Value tracks the period's weight mean (with tiny jitter).
|
||||
const v = 5.0 + dayIdx * 0.01 + (i % 2 ? 0.02 : -0.02);
|
||||
draws.push({ key: "LAB:HbA1c", day, value: v });
|
||||
}
|
||||
const result = discoverLabOutcomeCorrelations(draws, [weight], {
|
||||
minDraws: 5,
|
||||
minWindowPoints: 5,
|
||||
});
|
||||
const pair = result.discovered.find(
|
||||
(d) => d.lab === "LAB:HbA1c" && d.outcome === "WEIGHT",
|
||||
);
|
||||
expect(pair).toBeDefined();
|
||||
expect(pair!.n).toBeGreaterThanOrEqual(5);
|
||||
expect(pair!.r).toBeGreaterThan(0);
|
||||
expect(pair!.qValue).toBeLessThanOrEqual(result.fdrQ);
|
||||
expect(pair!.interpretation).toMatch(/never a cause/i);
|
||||
});
|
||||
|
||||
it("degrades to absent when the marker has too few draws", () => {
|
||||
const weight = weightSeries();
|
||||
const draws: LabDrawPoint[] = [0, 1, 2].map((i) => ({
|
||||
key: "LAB:HbA1c",
|
||||
day: weight.points[30 + i * 18].day,
|
||||
value: 5 + i,
|
||||
}));
|
||||
const result = discoverLabOutcomeCorrelations(draws, [weight], {
|
||||
minDraws: 5,
|
||||
});
|
||||
expect(result.discovered).toHaveLength(0);
|
||||
expect(result.pairsTested).toBe(0);
|
||||
});
|
||||
|
||||
it("ignores outcomes outside the curated lab-target set", () => {
|
||||
const hrv: NamedSeries = {
|
||||
key: "HEART_RATE_VARIABILITY",
|
||||
role: "outcome",
|
||||
points: seriesFrom(
|
||||
"2026-01-01",
|
||||
Array.from({ length: 180 }, (_, i) => 50 + i * 0.1),
|
||||
),
|
||||
};
|
||||
const draws: LabDrawPoint[] = Array.from({ length: 8 }, (_, i) => ({
|
||||
key: "LAB:HbA1c",
|
||||
day: hrv.points[30 + i * 18].day,
|
||||
value: 5 + i,
|
||||
}));
|
||||
const result = discoverLabOutcomeCorrelations(draws, [hrv], {
|
||||
minDraws: 5,
|
||||
});
|
||||
// HRV is not a curated lab-outcome target → no pair tested.
|
||||
expect(result.pairsTested).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -24,8 +24,11 @@ import {
|
||||
import {
|
||||
MEDICATION_COMPLIANCE_CHANNEL_KEY,
|
||||
SYMPTOM_SEVERITY_CHANNEL_KEY,
|
||||
type LabDrawPoint,
|
||||
type NamedSeries,
|
||||
} from "@/lib/insights/correlation-discovery";
|
||||
import { resolveLabFields } from "@/lib/labs/serialise";
|
||||
import { wallClockInTz } from "@/lib/tz/wall-clock";
|
||||
import {
|
||||
buildComplianceDailySeries,
|
||||
buildSymptomSeverityDailySeries,
|
||||
@@ -177,3 +180,76 @@ export async function fetchSymptomSeries(
|
||||
role: "outcome",
|
||||
});
|
||||
}
|
||||
|
||||
/** Day key (YYYY-MM-DD) for an instant in the user's display timezone. */
|
||||
function tzDayKey(at: Date, tz: string): string {
|
||||
const { year, month, day } = wallClockInTz(at, tz);
|
||||
return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* v1.22 — build the user's lab draws for the labs ↔ outcome correlation pass.
|
||||
*
|
||||
* One {@link LabDrawPoint} per QUANTITATIVE reading in the window, keyed
|
||||
* `LAB:<canonical analyte>` (the resolved name, so two spellings of one marker
|
||||
* collapse). HIDDEN biomarkers are excluded (the W3 catalog `hidden` flag — a
|
||||
* marker the user retired from the active list must not silently re-enter an
|
||||
* analysis surface). Qualitative readings (no numeric `value`) and rows whose
|
||||
* resolved value is non-finite are dropped — there is nothing to correlate.
|
||||
* The encrypted note column is never selected.
|
||||
*
|
||||
* Returns an EMPTY array when the user has no usable readings, so the discovery
|
||||
* pass degrades to absent rather than fabricating a link.
|
||||
*/
|
||||
export async function fetchLabDraws(
|
||||
userId: string,
|
||||
tz: string,
|
||||
since: Date,
|
||||
): Promise<LabDrawPoint[]> {
|
||||
const now = new Date();
|
||||
const rows = await prisma.labResult.findMany({
|
||||
where: {
|
||||
userId,
|
||||
deletedAt: null,
|
||||
value: { not: null },
|
||||
takenAt: { gte: since, lte: now },
|
||||
},
|
||||
orderBy: { takenAt: "asc" },
|
||||
take: 5000,
|
||||
select: {
|
||||
analyte: true,
|
||||
unit: true,
|
||||
referenceLow: true,
|
||||
referenceHigh: true,
|
||||
panel: true,
|
||||
value: true,
|
||||
takenAt: true,
|
||||
biomarkerId: true,
|
||||
biomarker: {
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
unit: true,
|
||||
lowerBound: true,
|
||||
upperBound: true,
|
||||
panel: true,
|
||||
hidden: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const draws: LabDrawPoint[] = [];
|
||||
for (const row of rows) {
|
||||
// Exclude retired markers (W3 hidden flag); unlinked rows cannot be hidden.
|
||||
if (row.biomarker?.hidden) continue;
|
||||
if (row.value === null || !Number.isFinite(row.value)) continue;
|
||||
const resolved = resolveLabFields(row, row.biomarker);
|
||||
draws.push({
|
||||
key: `LAB:${resolved.analyte}`,
|
||||
day: tzDayKey(row.takenAt, tz),
|
||||
value: row.value,
|
||||
});
|
||||
}
|
||||
return draws;
|
||||
}
|
||||
@@ -179,6 +179,22 @@ export interface DiscoveredCorrelation {
|
||||
interpretation: string;
|
||||
/** Lag in days applied (always 1 here). */
|
||||
lagDays: number;
|
||||
/**
|
||||
* v1.22 — which window surfaced the pair. `retrospective` = the standard
|
||||
* 180-day scan (the established pattern). `recent` = the rolling
|
||||
* early-detection pass over the trailing {@link EARLY_WINDOW_DAYS} days,
|
||||
* surfaced ONLY when the pair is not already established retrospectively.
|
||||
* Absent on the legacy path so existing consumers read it as the
|
||||
* retrospective default.
|
||||
*/
|
||||
window?: "retrospective" | "recent";
|
||||
/**
|
||||
* v1.22 — true for a `recent`-window emerging pair: it cleared a tighter
|
||||
* FDR and a no-faint effect bar over a short window, but rests on fewer days
|
||||
* than the retrospective scan, so the narration must hedge it as emerging /
|
||||
* provisional rather than established.
|
||||
*/
|
||||
provisional?: boolean;
|
||||
}
|
||||
|
||||
export interface CorrelationDiscoveryResult {
|
||||
@@ -470,6 +486,13 @@ export const DISCOVERY_BEHAVIOURS = [
|
||||
"BLOOD_GLUCOSE",
|
||||
"BLOOD_PRESSURE_SYS",
|
||||
"ACTIVITY_STEPS",
|
||||
// v1.22 — sleep duration as a lag SOURCE too (it was outcome-only). This
|
||||
// opens the clinically-obvious "poor sleep last night → next-day mood / HRV /
|
||||
// a vital drifting" direction. The same-family guard collapses only the
|
||||
// sleep→sleep self-lag (the key is its own family), so sleep stays free to
|
||||
// pair cross-domain. The daily series is the same per-day collapse the outcome
|
||||
// role already uses, so neither role sees a fabricated value.
|
||||
"SLEEP_DURATION",
|
||||
// v1.21.0 (FDREXTEND) — daily adherence rate as a lag source: the high-value
|
||||
// "adherence dip today → a worse outcome tomorrow" direction.
|
||||
MEDICATION_COMPLIANCE_CHANNEL_KEY,
|
||||
@@ -483,6 +506,16 @@ export const DISCOVERY_OUTCOMES = [
|
||||
"HEART_RATE_VARIABILITY",
|
||||
"RESTING_HEART_RATE",
|
||||
"WEIGHT",
|
||||
// v1.22 — blood pressure as an OUTCOME (it was behaviour-only). This makes
|
||||
// the clinically-obvious "medication adherence → next-day BP control" link
|
||||
// discoverable (compliance reached HRV / RHR / weight but never BP). Both
|
||||
// components are added so diastolic — previously never a candidate channel at
|
||||
// all — can correlate too. The same-family guard skips every
|
||||
// BP-behaviour → BP-outcome self/cross-component lag (serial autocorrelation),
|
||||
// so only genuine cross-domain behaviours (compliance, mood, daylight, …) →
|
||||
// BP survive. BH-FDR controls the wider pair family this opens up.
|
||||
"BLOOD_PRESSURE_SYS",
|
||||
"BLOOD_PRESSURE_DIA",
|
||||
// v1.11.5 (F3) — mood is now an OUTCOME channel too, not only a behaviour.
|
||||
// The discovery loop skips the MOOD→MOOD self-pair (`b.key === o.key`), so
|
||||
// promoting it lets the FDR scan surface "behaviour today → next-day mood"
|
||||
@@ -504,3 +537,364 @@ export const DISCOVERY_OUTCOMES = [
|
||||
// documented here so the omission is intentional, not a gap (same posture
|
||||
// as the medication-compliance omission above).
|
||||
] as const;
|
||||
|
||||
// ─── v1.22 — rolling early-detection window ───────────────────────────────
|
||||
|
||||
/**
|
||||
* v1.22 — trailing window (days) for the rolling EARLY-detection pass.
|
||||
*
|
||||
* The standard discovery scan is retrospective: it needs n ≥ 20 paired days, so
|
||||
* a pattern only surfaces once ~3 weeks of overlap have accrued INSIDE the
|
||||
* 180-day window — it explains the past well but reacts slowly to an emerging
|
||||
* shift (an illness pulling HRV + sleep down together over the last fortnight).
|
||||
* This shorter window re-runs the SAME engine over only the trailing days so a
|
||||
* fresh, concentrated pattern can surface before the long window dilutes it.
|
||||
*/
|
||||
export const EARLY_WINDOW_DAYS = 21;
|
||||
|
||||
/**
|
||||
* v1.22 — paired-day floor for the early window. Lower than the retrospective
|
||||
* `MIN_PAIRED_N` (20) because 21 days cannot reach 20 lagged pairs, but still
|
||||
* high enough that Pearson is defensible. The shorter, noisier series is
|
||||
* counter-balanced by a TIGHTER FDR ({@link EARLY_FDR_Q}) and a no-faint effect
|
||||
* bar (see {@link discoverEmergingCorrelations}), so the early pass keeps a HIGH
|
||||
* firing bar despite the smaller n.
|
||||
*/
|
||||
export const EARLY_MIN_PAIRS = 12;
|
||||
|
||||
/**
|
||||
* v1.22 — tighter Benjamini-Hochberg target for the early window (vs the
|
||||
* retrospective {@link FDR_Q} = 0.10). A short window is noisier, so the early
|
||||
* pass demands a stricter false-discovery rate before it will narrate an
|
||||
* emerging pattern.
|
||||
*/
|
||||
export const EARLY_FDR_Q = 0.05;
|
||||
|
||||
/** Pure: keep only the points on/after `fromDayKey` (inclusive). */
|
||||
export function filterSeriesToWindow(
|
||||
series: NamedSeries[],
|
||||
fromDayKey: string,
|
||||
): NamedSeries[] {
|
||||
return series.map((s) => ({
|
||||
...s,
|
||||
points: s.points.filter((p) => p.day >= fromDayKey),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Stable pair identity for double-count exclusion. */
|
||||
function pairKey(behaviour: string, outcome: string): string {
|
||||
return `${behaviour} | ||||