fix(insights): report every correlation through the significance gate

The /insights envelope (weight x BP, mood x BP, mood x weight, mood x
pulse, BP medication continuity) and the AI feature payload computed
their coefficients with an ungated Pearson that banded |r| above five
pairs, so five days of noise sloping the right way read as a strong
correlation there while every other surface refused the same points.
Both now use `significantPearsonCorrelation` (n >= 20 and p < 0.05,
through the engine's exact p-value), and the ungated function is gone.
The allowlist guard that tracked its remaining callers becomes a guard
that there is one Pearson implementation.
This commit is contained in:
Marc-André Bombeck committed 2026-10-20 03:41:54 +02:00
1 parent 04855b1a67
commit a22bf91311
9 files changed
+150 -269

No files matched your search

@@ -2,7 +2,6 @@ import { describe, it, expect } from "vitest";
import type { DataPoint } from "../trends";
import {
pairByTimestamp,
pearsonCorrelation,
significantPearsonCorrelation,
} from "../correlations";
import type { PairedPoint } from "../correlations";
@@ -52,62 +51,6 @@ describe("pairByTimestamp", () => {
});
});
describe("pearsonCorrelation", () => {
it("returns null for fewer than minPairs", () => {
const pairs = [
{ a: 1, b: 2, date: new Date() },
{ a: 2, b: 4, date: new Date() },
];
expect(pearsonCorrelation(pairs)).toBeNull();
});
it("detects perfect positive correlation", () => {
const pairs = Array.from({ length: 10 }, (_, i) => ({
a: i,
b: i * 2,
date: new Date(),
}));
const result = pearsonCorrelation(pairs)!;
expect(result.r).toBe(1);
expect(result.strength).toBe("stark");
expect(result.n).toBe(10);
});
it("detects perfect negative correlation", () => {
const pairs = Array.from({ length: 10 }, (_, i) => ({
a: i,
b: 100 - i * 2,
date: new Date(),
}));
const result = pearsonCorrelation(pairs)!;
expect(result.r).toBe(-1);
expect(result.strength).toBe("stark");
});
it("detects no correlation for random-like data", () => {
const pairs = [
{ a: 1, b: 5, date: new Date() },
{ a: 2, b: 3, date: new Date() },
{ a: 3, b: 7, date: new Date() },
{ a: 4, b: 2, date: new Date() },
{ a: 5, b: 6, date: new Date() },
];
const result = pearsonCorrelation(pairs)!;
expect(Math.abs(result.r)).toBeLessThan(0.4);
});
it("returns keine for constant values", () => {
const pairs = Array.from({ length: 5 }, () => ({
a: 5,
b: 5,
date: new Date(),
}));
const result = pearsonCorrelation(pairs)!;
expect(result.r).toBe(0);
expect(result.strength).toBe("keine");
});
});
describe("significantPearsonCorrelation (M-CS3 — n>=20 AND p<0.05)", () => {
const mk = (a: number, b: number): PairedPoint => ({
a,
@@ -116,14 +59,9 @@ describe("significantPearsonCorrelation (M-CS3 — n>=20 AND p<0.05)", () => {
});
it("suppresses a 5-pair r≈0.7 dataset (below the n>=20 floor)", () => {
// Five points with a strong-looking slope — exactly the small-n fluke the
// plain `pearsonCorrelation` would surface as a "stark" correlation.
// Five points with a strong-looking slope: the small-n fluke an
// ungated band would call a "stark" correlation.
const pairs = [mk(1, 1), mk(2, 2), mk(3, 2), mk(4, 4), mk(5, 3)];
// Sanity-check the legacy path WOULD have surfaced it (non-null, n=5).
const legacy = pearsonCorrelation(pairs);
expect(legacy).not.toBeNull();
expect(legacy!.n).toBe(5);
// The gated path refuses it — too few pairs.
expect(significantPearsonCorrelation(pairs)).toBeNull();
});
+6 -42
View File
@@ -72,54 +72,18 @@ export function pairByTimestamp(
}
/**
* Pearson correlation coefficient.
* Returns null if fewer than minPairs data points.
*/
export function pearsonCorrelation(
pairs: PairedPoint[],
minPairs: number = 5,
): CorrelationResult | null {
if (pairs.length < minPairs) return null;
const n = pairs.length;
const sumA = pairs.reduce((s, p) => s + p.a, 0);
const sumB = pairs.reduce((s, p) => s + p.b, 0);
const sumAB = pairs.reduce((s, p) => s + p.a * p.b, 0);
const sumAA = pairs.reduce((s, p) => s + p.a * p.a, 0);
const sumBB = pairs.reduce((s, p) => s + p.b * p.b, 0);
const denominator = Math.sqrt(
(n * sumAA - sumA * sumA) * (n * sumBB - sumB * sumB),
);
if (denominator === 0) return { r: 0, strength: "keine", n };
const r = Math.round(((n * sumAB - sumA * sumB) / denominator) * 1000) / 1000;
const absR = Math.abs(r);
let strength: CorrelationResult["strength"];
if (absR >= 0.7) strength = "stark";
else if (absR >= 0.4) strength = "moderat";
else if (absR >= 0.2) strength = "schwach";
else strength = "keine";
return { r, strength, n };
}
/**
* v1.2.5 (M-CS3) — significance-gated Pearson for USER-FACING surfaces.
* v1.2.5 (M-CS3) — significance-gated Pearson, the one correlation every
* surface reports.
*
* The plain `pearsonCorrelation` above assigns a strength band from |r|
* alone with no significance test and a default floor of only 5 pairs —
* so a 5-point r ≈ 0.7 fluke surfaces as a "stark" correlation. Every
* surfaced insight-card / status-snapshot correlation must instead clear
* the same rigorous bar the `/insights` correlation engine enforces:
* A strength band read from |r| alone, above a floor of five pairs, let a
* 5-point r ≈ 0.7 fluke surface as a "stark" correlation. Every reported
* correlation clears the bar the `/insights` correlation engine enforces:
*
* - n >= 20 paired points (the engine's `MIN_PAIRED_N`), AND
* - a two-sided Student-t p-value < 0.05 (`MAX_P_VALUE`).
*
* Below either bar this returns `null` so the caller suppresses the
* correlation exactly as it already does for the < minPairs case — no
* correlation — no
* call-site shape change. The returned `{ r, strength, n }` keeps the
* legacy shape (German strength band) so the snapshot byte-shape is
* unchanged on the happy path.
+2 -2
View File
@@ -32,7 +32,7 @@ import { getBpTargets } from "@/lib/analytics/bp-targets";
import { isBpReadingInTarget } from "@/lib/analytics/bp-in-target";
import {
pairByTimestamp,
pearsonCorrelation,
significantPearsonCorrelation,
type CorrelationResult,
} from "@/lib/analytics/correlations";
import { getMedicationCategories } from "@/lib/medication-category";
@@ -1656,7 +1656,7 @@ export async function extractFeatures(
}));
const computeCorr = (a: DataPoint[], b: DataPoint[]) =>
pearsonCorrelation(pairByTimestamp(a, b));
significantPearsonCorrelation(pairByTimestamp(a, b));
const sleepPoints = toDataPoints(sleepData);
+3 -3
View File
@@ -428,9 +428,9 @@ export interface MoodMetricCorrelation {
*
* This routes through `significantPearsonCorrelation` — n >= 20 AND
* p < 0.05 — for the same reason the weight, blood-pressure and
* mood-status surfaces do: the plain `pearsonCorrelation` bands from |r|
* alone above a floor of five pairs, so five days of noise sloping the
* right way arrive at the reader labelled "strong". A mood page is where
* mood-status surfaces do: a band read from |r| alone above a floor of
* five pairs lets five days of noise sloping the right way arrive at the
* reader labelled "strong". A mood page is where
* that costs the most, because the reader is looking for something to
* change about their week.
*/