fix(insights): report every correlation through the significance gate

The /insights envelope (weight x BP, mood x BP, mood x weight, mood x
pulse, BP medication continuity) and the AI feature payload computed
their coefficients with an ungated Pearson that banded |r| above five
pairs, so five days of noise sloping the right way read as a strong
correlation there while every other surface refused the same points.
Both now use `significantPearsonCorrelation` (n >= 20 and p < 0.05,
through the engine's exact p-value), and the ungated function is gone.
The allowlist guard that tracked its remaining callers becomes a guard
that there is one Pearson implementation.
This commit is contained in:
Marc-André Bombeck committed 2026-10-20 03:41:54 +02:00
1 parent 04855b1a67
commit a22bf91311
9 files changed
+150 -269

No files matched your search

@@ -569,16 +569,10 @@ describe("GET /api/insights/comprehensive — envelope shape", () => {
expect(body.data.moodSummary?.count).toBe(2);
});
it("derives weight × BP scatter from daily-bucketed series", async () => {
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
// Five day-aligned pairs (n >= 5 unlocks pearsonCorrelation).
const days = [
"2026-05-01",
"2026-05-02",
"2026-05-03",
"2026-05-04",
"2026-05-05",
];
function mockWeightBpDays(count: number) {
const days = Array.from({ length: count }, (_, i) =>
new Date(Date.UTC(2026, 4, 1 + i)).toISOString().slice(0, 10),
);
(buildComprehensiveAggregate as ReturnType<typeof vi.fn>).mockResolvedValue(
{
summaries: {},
@@ -592,22 +586,41 @@ describe("GET /api/insights/comprehensive — envelope shape", () => {
})),
},
firstMeasurementAt: new Date(),
totalMeasurements: 10,
totalMeasurements: count * 2,
},
);
}
type WeightBpBody = {
data: {
scatterData: Array<{ weight: number; sysBP: number }>;
weightBpCorrelation: { r: number; n: number } | null;
};
};
it("derives weight × BP scatter from daily-bucketed series", async () => {
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
mockWeightBpDays(25);
const res = await callGet(makeReq());
const body = (await res.json()) as {
data: {
scatterData: Array<{ weight: number; sysBP: number }>;
weightBpCorrelation: { r: number; n: number } | null;
};
};
expect(body.data.scatterData).toHaveLength(5);
const body = (await res.json()) as WeightBpBody;
expect(body.data.scatterData).toHaveLength(25);
expect(body.data.scatterData[0]).toEqual({ weight: 80, sysBP: 120 });
// Perfect linear relationship → r === 1.
// Perfect linear relationship over 25 days → r === 1.
expect(body.data.weightBpCorrelation?.r).toBe(1);
expect(body.data.weightBpCorrelation?.n).toBe(5);
expect(body.data.weightBpCorrelation?.n).toBe(25);
});
it("reports no correlation below the significance bar every surface uses", async () => {
// Five perfectly aligned days: the scatter shows them, but five points
// cannot carry a correlation (n >= 20 and p < 0.05).
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
mockWeightBpDays(5);
const res = await callGet(makeReq());
const body = (await res.json()) as WeightBpBody;
expect(body.data.scatterData).toHaveLength(5);
expect(body.data.weightBpCorrelation).toBeNull();
});
});
@@ -849,13 +862,10 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing (
// tier in R12. Both read the UTC `dailyByType` / UTC `scheduledFor`
// slice, so their output must be invariant to the caller's timezone.
// Run the SAME fixture under UTC then America/New_York and deep-equal.
const days = [
"2026-05-01",
"2026-05-02",
"2026-05-03",
"2026-05-04",
"2026-05-05",
];
// 25 days: a correlation has to clear n >= 20 and p < 0.05 to exist.
const days = Array.from({ length: 25 }, (_, i) =>
new Date(Date.UTC(2026, 4, 1 + i)).toISOString().slice(0, 10),
);
const aggregate = {
summaries: {},
bpRawRows: { sys: [], dia: [] },
@@ -896,20 +906,20 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing (
scheduleRevisions: [],
pauseEras: [],
};
// Intake taken on the first three days, logged as skipped on the last
// two → a non-degenerate continuity series. The skips used to be absent
// Intake taken on the first fifteen days, logged as skipped on the
// last ten → a non-degenerate continuity series. The skips used to be absent
// rows: a day with no record read as a measured 0 % continuity, which is
// the fabrication the route no longer makes, and a fixture that relies on
// it would be pinning the defect rather than the timezone question this
// test asks. Recording the two days as skips keeps five paired days AND
// test asks. Recording the days as skips keeps every day paired AND
// keeps every one of them a thing the person actually logged.
const intakeEvents = days.map((d, i) => ({
medicationId: "med-bp",
userId: "user-comp-1",
deletedAt: null,
scheduledFor: new Date(`${d}T07:00:00Z`),
takenAt: i < 3 ? new Date(`${d}T07:05:00Z`) : null,
skipped: i >= 3,
takenAt: i < 15 ? new Date(`${d}T07:05:00Z`) : null,
skipped: i >= 15,
}));
function primeMocks() {
@@ -948,7 +958,8 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing (
);
// And the guard is meaningful only if these joins actually produced
// output to compare.
expect(utcBody.data.scatterData).toHaveLength(5);
expect(utcBody.data.scatterData).toHaveLength(25);
expect(utcBody.data.weightBpCorrelation).not.toBeNull();
expect(utcBody.data.bpMedicationCorrelation).not.toBeNull();
});
});
+7 -7
View File
@@ -12,7 +12,7 @@ import {
generateAlerts,
} from "@/lib/analytics/classifications";
import {
pearsonCorrelation,
significantPearsonCorrelation,
type PairedPoint,
} from "@/lib/analytics/correlations";
import {
@@ -285,7 +285,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) {
const weightDaily = aggregate.dailyByType.WEIGHT ?? [];
const sysDaily = aggregate.dailyByType.BLOOD_PRESSURE_SYS ?? [];
const weightBpPairs: PairedPoint[] = joinDailyByDay(weightDaily, sysDaily);
const weightBpCorrelation = pearsonCorrelation(weightBpPairs);
const weightBpCorrelation = significantPearsonCorrelation(weightBpPairs);
const scatterData = weightBpPairs.map((p) => ({
weight: p.a,
sysBP: p.b,
@@ -308,11 +308,11 @@ export async function buildComprehensiveResponse(user: AuthedUser) {
);
const moodBpPairs = pairMoodWithDaily(dailyMoodEntries, sysDailyLocal);
const moodBpCorrelation = pearsonCorrelation(moodBpPairs);
const moodBpCorrelation = significantPearsonCorrelation(moodBpPairs);
const moodBpScatterData = moodBpPairs.map((p) => ({ mood: p.a, sysBP: p.b }));
const moodWeightPairs = pairMoodWithDaily(dailyMoodEntries, weightDailyLocal);
const moodWeightCorrelation = pearsonCorrelation(moodWeightPairs);
const moodWeightCorrelation = significantPearsonCorrelation(moodWeightPairs);
const moodWeightScatterData = moodWeightPairs.map((p) => ({
mood: p.a,
weight: p.b,
@@ -374,7 +374,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) {
.sort(([a], [b]) => a.localeCompare(b))
.map(([day, stats]) => ({ day, value: stats.sum / stats.count }));
const moodPulsePairs = pairMoodWithDaily(dailyMoodEntries, restingPulseDaily);
const moodPulseCorrelation = pearsonCorrelation(moodPulsePairs);
const moodPulseCorrelation = significantPearsonCorrelation(moodPulsePairs);
const moodPulseScatterData = moodPulsePairs.map((p) => ({
mood: p.a,
pulse: p.b,
@@ -558,7 +558,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) {
});
}
const corr = pearsonCorrelation(pairs);
const corr = significantPearsonCorrelation(pairs);
if (corr) {
bpMedicationCorrelation = {
...corr,
@@ -667,7 +667,7 @@ function deriveDailyLocalMeans(
/**
* Daily-key inner join between two daily-aggregated series. Returns
* `PairedPoint` shapes compatible with `pearsonCorrelation`.
* `PairedPoint` shapes compatible with `significantPearsonCorrelation`.
*
* Inputs are sorted ASC by `day` (the aggregator's ORDER BY
* guarantees that). The walk is O(n + m) merge-join style instead of