Files
HealthLog/src/lib/insights/correlation-channel-series.ts
T
Marc-André Bombeck 6c2eb15f05 fix(insights): hoist the correlation window fetch so every reader shares one grain
The asc+take(20000) window-cap bug (fixed on the correlations route
alone) and the mixed-grain toDailyMeans bug (fixed on three of four
readers) both survived on the Coach get_correlations tool, the
per-metric assessment card, and the period narrative. Dense accounts
saw the cap drop the newest rows on the coach/card readers and drop
the CURRENT period on the narrative — backwards for a recent-window
and a current-vs-prior surface respectively; the narrative's own local
toDailyMeans twin also averaged sleep stage-segments and blended
per-sample step chunks with drained daily totals instead of summing.

Hoists the fetch + grain resolution into
fetchMeasurementWindowSeries/fetchMoodWindowSeries
(correlation-channel-series.ts) — desc+cap+resort, full column select
for buildMeasurementDailySeries's per-type grain — and routes the
route, the coach tool, the per-metric card, and the period narrative
through it, deleting every local copy.
2026-07-18 06:47:32 +02:00

575 lines
19 KiB
TypeScript

/**
* v1.21.0 (INTEGFIX) — shared data-fetching helpers for the two non-measurement
* FDR correlation channels (medication compliance + symptom severity).
*
* The pure series shapers live in `correlation-series-builders.ts`; the FDR
* engine itself is pure over `NamedSeries[]`. These two helpers own the DB reads
* that feed those shapers — the dose-history ledger (compliance) and the illness
* day-log (symptom severity) — so EVERY consumer of the discovery matrix (the
* `/api/insights/correlations` route AND the Coach `get_correlations` tool)
* builds the channels identically rather than re-implementing the queries, the
* tz keying, and the episode-span clamping per call site.
*
* Each helper degrades to an EMPTY series when the user has no data, so the
* channel drops out of discovery (it cannot clear the n ≥ 20 floor) rather than
* fabricating a constant.
*/
import { prisma } from "@/lib/db";
import {
buildComplianceMedicationContext,
buildMedicationComplianceBundle,
lastNonSkippedTakenAt,
SCHEDULE_COMPLIANCE_SELECT,
} from "@/lib/analytics/compliance";
import {
MEDICATION_COMPLIANCE_CHANNEL_KEY,
SYMPTOM_SEVERITY_CHANNEL_KEY,
type DailySeriesPoint,
type LabDrawPoint,
type NamedSeries,
} from "@/lib/insights/correlation-discovery";
import { resolveLabFields } from "@/lib/labs/serialise";
import { wallClockInTz } from "@/lib/tz/wall-clock";
import { ENVIRONMENT_FIELDS } from "@/lib/environment/fields";
import {
buildComplianceDailySeries,
buildSymptomSeverityDailySeries,
type SymptomDayLogRow,
type SymptomEpisodeSpan,
} from "@/lib/insights/correlation-series-builders";
import type { DoseHistoryRow } from "@/lib/medications/scheduling/dose-history";
import { CUMULATIVE_HK_TYPES } from "@/lib/measurements/apple-health-mapping";
import { metricKeyForType } from "@/lib/measurements/cumulative-day-sum";
import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority";
import {
reconstructSleepSessions,
pickMainNightAndNaps,
type SleepStageRow,
} from "@/lib/analytics/sleep-night";
import type {
MeasurementSource,
MeasurementType,
} from "@/generated/prisma/client";
/**
* v1.21.0 (FDREXTEND) — build the user's MEDICATION_COMPLIANCE daily series.
*
* Pools every active, non-PRN medication's unified dose-history ledger over the
* window, then collapses to one per-day adherence rate (user-tz day keys). A
* user with no active medications (or no resolved slots) yields an empty
* series, so the channel degrades to absent.
*/
export async function fetchComplianceSeries(
userId: string,
tz: string,
since: Date,
): Promise<NamedSeries> {
const medications = await prisma.medication.findMany({
// PRN (as-needed) medications have no expected doses → no defensible rate.
where: { userId, active: true, asNeeded: false },
include: {
schedules: { select: SCHEDULE_COMPLIANCE_SELECT },
scheduleRevisions: { orderBy: { validFrom: "asc" } },
// v1.25 H-MED1 — pause eras so paused days drop out of the denominator.
pauseEras: { select: { pausedAt: true, resumedAt: true } },
},
orderBy: { name: "asc" },
});
if (medications.length === 0) {
return {
key: MEDICATION_COMPLIANCE_CHANNEL_KEY,
role: "behaviour",
points: [],
};
}
const events = await prisma.medicationIntakeEvent.findMany({
where: {
userId,
deletedAt: null,
medicationId: { in: medications.map((med) => med.id) },
scheduledFor: { gte: since },
},
orderBy: { scheduledFor: "asc" },
select: {
medicationId: true,
scheduledFor: true,
takenAt: true,
skipped: true,
},
});
const now = new Date();
const ledgerRows: DoseHistoryRow[] = [];
for (const medication of medications) {
const medEvents = events.filter((e) => e.medicationId === medication.id);
const ctx = buildComplianceMedicationContext(
medication,
lastNonSkippedTakenAt(medEvents),
tz,
);
const bundle = buildMedicationComplianceBundle(
medEvents,
medication.schedules,
ctx,
now,
);
ledgerRows.push(...bundle.ledgerRows);
}
return buildComplianceDailySeries(ledgerRows, tz);
}
/**
* v1.21.0 (FDREXTEND) — build the user's SYMPTOM_SEVERITY daily series in the
* `outcome` role (callers that need the behaviour role re-tag the returned
* series — the points are role-invariant). Reads every in-window illness episode
* + its day-logs; the builder zero-fills healthy days ONLY across real episode
* spans, so a user with no episodes yields an empty series that degrades to
* absent.
*/
export async function fetchSymptomSeries(
userId: string,
tz: string,
since: Date,
): Promise<NamedSeries> {
const now = new Date();
const episodes = await prisma.illnessEpisode.findMany({
// An episode overlaps the window when it onset before `now` and either is
// ongoing or resolved at/after the window start.
where: {
userId,
deletedAt: null,
onsetAt: { lte: now },
OR: [{ resolvedAt: null }, { resolvedAt: { gte: since } }],
},
select: { id: true, onsetAt: true, resolvedAt: true },
});
if (episodes.length === 0) {
return { key: SYMPTOM_SEVERITY_CHANNEL_KEY, role: "outcome", points: [] };
}
const dayLogRows = await prisma.illnessDayLog.findMany({
where: {
userId,
deletedAt: null,
episodeId: { in: episodes.map((e) => e.id) },
},
select: {
date: true,
functionalImpact: true,
symptomLinks: { select: { severity: true } },
},
});
// Collapse each day-log to one burden value (functionalImpact, else the day's
// max linked symptom severity) — the same rule the recovery-gap track uses.
const dayLogs: SymptomDayLogRow[] = [];
for (const row of dayLogRows) {
if (row.functionalImpact != null) {
dayLogs.push({ day: row.date, impact: row.functionalImpact });
continue;
}
let maxSeverity: number | null = null;
for (const link of row.symptomLinks) {
if (link.severity == null) continue;
maxSeverity =
maxSeverity === null
? link.severity
: Math.max(maxSeverity, link.severity);
}
if (maxSeverity != null)
dayLogs.push({ day: row.date, impact: maxSeverity });
}
const spans: SymptomEpisodeSpan[] = episodes.map((e) => ({
onsetAt: e.onsetAt,
resolvedAt: e.resolvedAt,
}));
return buildSymptomSeverityDailySeries({
dayLogs,
episodes: spans,
tz,
windowStart: since,
windowEnd: now,
role: "outcome",
});
}
/**
* v1.25 (W-ENV) — build the user's environmental-exposure BEHAVIOUR channels.
*
* One {@link NamedSeries} per registered env field (temperature, daylight,
* sunshine, precipitation, pressure mean + intraday swing), read from the daily
* `EnvironmentContext` rows the nightly job stores. Each row's `date` is already
* a YYYY-MM-DD key, so points need no re-keying. Sunshine / daylight are stored
* in seconds and surfaced as hours (correlation r is scale-invariant; hours just
* keep the series readable). A field with no non-null values yields an empty
* series that degrades to absent — never a fabricated constant. The whole set is
* empty when the user has no environment rows (module off / no home set).
*/
export async function fetchEnvironmentSeries(
userId: string,
since: Date,
): Promise<NamedSeries[]> {
const sinceKey = since.toISOString().slice(0, 10);
const rows = await prisma.environmentContext.findMany({
where: { userId, date: { gte: sinceKey } },
orderBy: { date: "asc" },
take: 1000,
select: {
date: true,
tempMean: true,
tempMin: true,
tempMax: true,
apparentMean: true,
sunshineSec: true,
daylightSec: true,
precipSum: true,
pressureMean: true,
pressureDelta: true,
humidityMean: true,
cloudMean: true,
},
});
return ENVIRONMENT_FIELDS.map((field) => {
const points: DailySeriesPoint[] = [];
for (const row of rows) {
const raw = row[field.column];
if (raw == null || !Number.isFinite(raw)) continue;
// Seconds → hours for the duration fields; pass through otherwise.
const value =
field.column === "sunshineSec" || field.column === "daylightSec"
? raw / 3600
: raw;
points.push({ day: row.date, value });
}
return { key: field.key, role: "behaviour" as const, points };
});
}
/** Day key (YYYY-MM-DD) for an instant in the user's display timezone. */
function tzDayKey(at: Date, tz: string): string {
const { year, month, day } = wallClockInTz(at, tz);
return `${year}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`;
}
const MEASUREMENT_READ_CAP = 20000;
const MOOD_READ_CAP = 5000;
/** One measurement-window fetch's result: per-type raw rows + the cap flag. */
export interface MeasurementWindowFetch {
/** Raw rows shaped for `buildMeasurementDailySeries`, grouped by type. */
byType: Map<string, MeasurementSeriesRow[]>;
/** True when the read hit the cap — the window may be missing OLDER rows. */
measurementsCapped: boolean;
}
/**
* v1.30.3 (QA F1/F2/F3) — shared measurement-window fetch for every
* correlation-adjacent surface: the `/api/insights/correlations` route, the
* Coach `get_correlations` tool, the per-metric assessment card, and the
* period narrative. Hoisted here so a fourth independently-maintained copy
* can't drift the way the period-narrative one did.
*
* Orders DESC + caps at {@link MEASUREMENT_READ_CAP}, then re-sorts ASC in
* JS so a dense account's cap falls on the OLDEST rows, never the newest —
* the inverse of a naive `orderBy asc, take N`, which silently drops the
* NEWEST reads once an account's in-window row count crosses the cap
* (exactly backwards for a recent-window read, e.g. the Coach's emerging-
* correlations pass or a current-vs-prior period narrative). Selects
* `source` / `deviceType` / `sleepStage` unconditionally so every caller can
* feed `buildMeasurementDailySeries`'s per-type grain resolution (source
* collapse for cumulative types, per-night reconstruction for sleep)
* without a second query.
*/
export async function fetchMeasurementWindowSeries(
userId: string,
since: Date,
types: MeasurementType[],
): Promise<MeasurementWindowFetch> {
const rowsDesc = await prisma.measurement.findMany({
where: {
userId,
deletedAt: null,
measuredAt: { gte: since },
type: { in: types },
},
orderBy: { measuredAt: "desc" },
take: MEASUREMENT_READ_CAP,
select: {
type: true,
value: true,
measuredAt: true,
source: true,
deviceType: true,
sleepStage: true,
},
});
const measurementsCapped = rowsDesc.length >= MEASUREMENT_READ_CAP;
const byType = new Map<string, MeasurementSeriesRow[]>();
for (const m of [...rowsDesc].sort(
(a, b) => a.measuredAt.getTime() - b.measuredAt.getTime(),
)) {
const list = byType.get(m.type) ?? [];
list.push({
value: m.value,
at: m.measuredAt,
source: m.source,
deviceType: m.deviceType,
sleepStage: m.sleepStage,
});
byType.set(m.type, list);
}
return { byType, measurementsCapped };
}
/** One mood-window fetch's result: the daily-mean series + the cap flag. */
export interface MoodWindowFetch {
moodDaily: DailySeriesPoint[];
/** True when the read hit the cap — the window may be missing OLDER rows. */
moodCapped: boolean;
}
/**
* v1.30.3 (QA F1/F2/F3) — shared mood-window fetch, same desc+cap+resort
* discipline as {@link fetchMeasurementWindowSeries}. Callers that also need
* per-entry factor tag-links (the period narrative's RATED-factor channels)
* read `MoodEntry` themselves for the extra `tagLinks` select, but MUST
* apply the same desc+resort discipline — this helper only covers the
* plain-score case the route / coach tool / per-metric card share.
*/
export async function fetchMoodWindowSeries(
userId: string,
tz: string,
since: Date,
): Promise<MoodWindowFetch> {
const rowsDesc = await prisma.moodEntry.findMany({
where: { userId, deletedAt: null, moodLoggedAt: { gte: since } },
orderBy: { moodLoggedAt: "desc" },
take: MOOD_READ_CAP,
select: { score: true, moodLoggedAt: true },
});
const moodCapped = rowsDesc.length >= MOOD_READ_CAP;
const rows = [...rowsDesc].sort(
(a, b) => a.moodLoggedAt.getTime() - b.moodLoggedAt.getTime(),
);
const moodDaily = toDailyMeans(
rows.map((e) => ({ value: e.score, at: e.moodLoggedAt })),
tz,
);
return { moodDaily, moodCapped };
}
/**
* v1.29.6 — collapse rows to per-day MEANS keyed in the user's tz. This is
* the correct grain for spot metrics (BP, glucose, HRV, resting HR, weight,
* mood) where a day's reduction is the average of its readings. Cumulative
* metrics and sleep must NOT use this — see `buildMeasurementDailySeries`.
*
* Hoisted here from the two former call sites (`/api/insights/correlations`
* and `metric-correlation-context.ts`) so both surfaces stay byte-identical
* instead of drifting as two independently-maintained copies — the same
* class of drift that let the cumulative/sleep grain bug below slip into
* one file and not the other.
*/
export function toDailyMeans(
rows: Array<{ value: number; at: Date }>,
tz: string,
): DailySeriesPoint[] {
const byDay = new Map<string, { sum: number; count: number }>();
for (const r of rows) {
if (!Number.isFinite(r.value)) continue;
const day = tzDayKey(r.at, tz);
const acc = byDay.get(day) ?? { sum: 0, count: 0 };
acc.sum += r.value;
acc.count += 1;
byDay.set(day, acc);
}
return [...byDay.entries()]
.map(([day, acc]) => ({ day, value: acc.sum / acc.count }))
.sort((a, b) => (a.day < b.day ? -1 : 1));
}
/** Minimum row shape `buildMeasurementDailySeries` needs per raw reading. */
export interface MeasurementSeriesRow {
value: number;
at: Date;
source: MeasurementSource;
deviceType: string | null;
/** Only populated (and only consulted) for SLEEP_DURATION rows. */
sleepStage: SleepStageRow["sleepStage"] | null;
}
/**
* v1.29.6 — collapse one MeasurementType's raw rows to a single-grain daily
* series, keyed in the user's tz. Fixes a correlation-discovery distortion:
* `ACTIVITY_STEPS` and other cumulative HK types were being reduced with
* `toDailyMeans`, which blends per-sample chunk averages (~350 steps, from
* the not-yet-nightly-drained window) with drained `stats:` daily totals
* (~8400 steps) into one meaningless per-day figure. `SLEEP_DURATION` was
* averaging per-STAGE segment durations (~45 min) instead of summing a
* night's total time asleep, and without collapsing overlapping sources
* first (a WHOOP + Apple Health night double-counted).
*
* - `SLEEP_DURATION` → per-night TOTAL time asleep for the MAIN session
* (naps excluded, matching the dashboard/list convention), via
* `reconstructSleepSessions` — the same writer-dedup + per-night
* collapse the sleep list route and dashboard tile use.
* - `CUMULATIVE_HK_TYPES` (steps, active energy, distance, flights,
* daylight, falls) → source-collapsed per-day SUM, via
* `pickCanonicalSourceRows` (a type with no ladder passes every row
* through unchanged, matching the picker's documented fallback).
* - everything else → per-day MEAN via `toDailyMeans` (unchanged).
*/
export function buildMeasurementDailySeries(
type: MeasurementType,
rows: MeasurementSeriesRow[],
tz: string,
priorityJson: unknown,
): DailySeriesPoint[] {
if (type === "SLEEP_DURATION") {
return buildSleepDailySeries(rows, tz, priorityJson);
}
if (CUMULATIVE_HK_TYPES.has(type)) {
return buildCumulativeDailySeries(type, rows, tz, priorityJson);
}
return toDailyMeans(
rows.map((r) => ({ value: r.value, at: r.at })),
tz,
);
}
function buildSleepDailySeries(
rows: MeasurementSeriesRow[],
tz: string,
priorityJson: unknown,
): DailySeriesPoint[] {
const stageRows: SleepStageRow[] = rows.map((r) => ({
value: r.value,
measuredAt: r.at,
sleepStage: r.sleepStage,
source: r.source,
deviceType: r.deviceType,
}));
const sessions = reconstructSleepSessions(stageRows, tz, priorityJson);
const byNight = new Map<string, typeof sessions>();
for (const s of sessions) {
const list = byNight.get(s.night) ?? [];
list.push(s);
byNight.set(s.night, list);
}
const points: DailySeriesPoint[] = [];
for (const [night, nightSessions] of byNight) {
const { main } = pickMainNightAndNaps(nightSessions);
if (!main) continue;
points.push({ day: night, value: main.asleepMinutes });
}
return points.sort((a, b) => (a.day < b.day ? -1 : 1));
}
function buildCumulativeDailySeries(
type: MeasurementType,
rows: MeasurementSeriesRow[],
tz: string,
priorityJson: unknown,
): DailySeriesPoint[] {
const metricKey = metricKeyForType(type);
const canonicalRows = metricKey
? pickCanonicalSourceRows(
rows.map((r) => ({
measuredAt: r.at,
source: r.source,
deviceType: r.deviceType,
type,
value: r.value,
})),
metricKey,
priorityJson,
(d) => tzDayKey(d, tz),
).canonicalRows
: rows.map((r) => ({ measuredAt: r.at, value: r.value }));
const byDay = new Map<string, number>();
for (const row of canonicalRows) {
if (!Number.isFinite(row.value)) continue;
const key = tzDayKey(row.measuredAt, tz);
byDay.set(key, (byDay.get(key) ?? 0) + row.value);
}
return [...byDay.entries()]
.map(([day, value]) => ({ day, value }))
.sort((a, b) => (a.day < b.day ? -1 : 1));
}
/**
* v1.22 — build the user's lab draws for the labs ↔ outcome correlation pass.
*
* One {@link LabDrawPoint} per QUANTITATIVE reading in the window, keyed
* `LAB:<canonical analyte>` (the resolved name, so two spellings of one marker
* collapse). HIDDEN biomarkers are excluded (the W3 catalog `hidden` flag — a
* marker the user retired from the active list must not silently re-enter an
* analysis surface). Qualitative readings (no numeric `value`) and rows whose
* resolved value is non-finite are dropped — there is nothing to correlate.
* The encrypted note column is never selected.
*
* Returns an EMPTY array when the user has no usable readings, so the discovery
* pass degrades to absent rather than fabricating a link.
*/
export async function fetchLabDraws(
userId: string,
tz: string,
since: Date,
): Promise<LabDrawPoint[]> {
const now = new Date();
const rows = await prisma.labResult.findMany({
where: {
userId,
deletedAt: null,
value: { not: null },
takenAt: { gte: since, lte: now },
},
orderBy: { takenAt: "asc" },
take: 5000,
select: {
analyte: true,
unit: true,
referenceLow: true,
referenceHigh: true,
panel: true,
value: true,
takenAt: true,
biomarkerId: true,
biomarker: {
select: {
id: true,
name: true,
unit: true,
lowerBound: true,
upperBound: true,
panel: true,
hidden: true,
},
},
},
});
const draws: LabDrawPoint[] = [];
for (const row of rows) {
// Exclude retired markers (W3 hidden flag); unlinked rows cannot be hidden.
if (row.biomarker?.hidden) continue;
if (row.value === null || !Number.isFinite(row.value)) continue;
const resolved = resolveLabFields(row, row.biomarker);
draws.push({
key: `LAB:${resolved.analyte}`,
day: tzDayKey(row.takenAt, tz),
value: row.value,
});
}
return draws;
}