mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 23:52:24 +00:00
fix(insights): report every correlation through the significance gate
The /insights envelope (weight x BP, mood x BP, mood x weight, mood x pulse, BP medication continuity) and the AI feature payload computed their coefficients with an ungated Pearson that banded |r| above five pairs, so five days of noise sloping the right way read as a strong correlation there while every other surface refused the same points. Both now use `significantPearsonCorrelation` (n >= 20 and p < 0.05, through the engine's exact p-value), and the ungated function is gone. The allowlist guard that tracked its remaining callers becomes a guard that there is one Pearson implementation.
This commit is contained in:
1 parent
04855b1a67
commit
a22bf91311
9 files changed
+150
-269
No files matched your search
@@ -2,7 +2,6 @@ import { describe, it, expect } from "vitest";
|
||||
import type { DataPoint } from "../trends";
|
||||
import {
|
||||
pairByTimestamp,
|
||||
pearsonCorrelation,
|
||||
significantPearsonCorrelation,
|
||||
} from "../correlations";
|
||||
import type { PairedPoint } from "../correlations";
|
||||
@@ -52,62 +51,6 @@ describe("pairByTimestamp", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("pearsonCorrelation", () => {
|
||||
it("returns null for fewer than minPairs", () => {
|
||||
const pairs = [
|
||||
{ a: 1, b: 2, date: new Date() },
|
||||
{ a: 2, b: 4, date: new Date() },
|
||||
];
|
||||
expect(pearsonCorrelation(pairs)).toBeNull();
|
||||
});
|
||||
|
||||
it("detects perfect positive correlation", () => {
|
||||
const pairs = Array.from({ length: 10 }, (_, i) => ({
|
||||
a: i,
|
||||
b: i * 2,
|
||||
date: new Date(),
|
||||
}));
|
||||
const result = pearsonCorrelation(pairs)!;
|
||||
expect(result.r).toBe(1);
|
||||
expect(result.strength).toBe("stark");
|
||||
expect(result.n).toBe(10);
|
||||
});
|
||||
|
||||
it("detects perfect negative correlation", () => {
|
||||
const pairs = Array.from({ length: 10 }, (_, i) => ({
|
||||
a: i,
|
||||
b: 100 - i * 2,
|
||||
date: new Date(),
|
||||
}));
|
||||
const result = pearsonCorrelation(pairs)!;
|
||||
expect(result.r).toBe(-1);
|
||||
expect(result.strength).toBe("stark");
|
||||
});
|
||||
|
||||
it("detects no correlation for random-like data", () => {
|
||||
const pairs = [
|
||||
{ a: 1, b: 5, date: new Date() },
|
||||
{ a: 2, b: 3, date: new Date() },
|
||||
{ a: 3, b: 7, date: new Date() },
|
||||
{ a: 4, b: 2, date: new Date() },
|
||||
{ a: 5, b: 6, date: new Date() },
|
||||
];
|
||||
const result = pearsonCorrelation(pairs)!;
|
||||
expect(Math.abs(result.r)).toBeLessThan(0.4);
|
||||
});
|
||||
|
||||
it("returns keine for constant values", () => {
|
||||
const pairs = Array.from({ length: 5 }, () => ({
|
||||
a: 5,
|
||||
b: 5,
|
||||
date: new Date(),
|
||||
}));
|
||||
const result = pearsonCorrelation(pairs)!;
|
||||
expect(result.r).toBe(0);
|
||||
expect(result.strength).toBe("keine");
|
||||
});
|
||||
});
|
||||
|
||||
describe("significantPearsonCorrelation (M-CS3 — n>=20 AND p<0.05)", () => {
|
||||
const mk = (a: number, b: number): PairedPoint => ({
|
||||
a,
|
||||
@@ -116,14 +59,9 @@ describe("significantPearsonCorrelation (M-CS3 — n>=20 AND p<0.05)", () => {
|
||||
});
|
||||
|
||||
it("suppresses a 5-pair r≈0.7 dataset (below the n>=20 floor)", () => {
|
||||
// Five points with a strong-looking slope — exactly the small-n fluke the
|
||||
// plain `pearsonCorrelation` would surface as a "stark" correlation.
|
||||
// Five points with a strong-looking slope: the small-n fluke an
|
||||
// ungated band would call a "stark" correlation.
|
||||
const pairs = [mk(1, 1), mk(2, 2), mk(3, 2), mk(4, 4), mk(5, 3)];
|
||||
// Sanity-check the legacy path WOULD have surfaced it (non-null, n=5).
|
||||
const legacy = pearsonCorrelation(pairs);
|
||||
expect(legacy).not.toBeNull();
|
||||
expect(legacy!.n).toBe(5);
|
||||
// The gated path refuses it — too few pairs.
|
||||
expect(significantPearsonCorrelation(pairs)).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
@@ -72,54 +72,18 @@ export function pairByTimestamp(
|
||||
}
|
||||
|
||||
/**
|
||||
* Pearson correlation coefficient.
|
||||
* Returns null if fewer than minPairs data points.
|
||||
*/
|
||||
export function pearsonCorrelation(
|
||||
pairs: PairedPoint[],
|
||||
minPairs: number = 5,
|
||||
): CorrelationResult | null {
|
||||
if (pairs.length < minPairs) return null;
|
||||
|
||||
const n = pairs.length;
|
||||
const sumA = pairs.reduce((s, p) => s + p.a, 0);
|
||||
const sumB = pairs.reduce((s, p) => s + p.b, 0);
|
||||
const sumAB = pairs.reduce((s, p) => s + p.a * p.b, 0);
|
||||
const sumAA = pairs.reduce((s, p) => s + p.a * p.a, 0);
|
||||
const sumBB = pairs.reduce((s, p) => s + p.b * p.b, 0);
|
||||
|
||||
const denominator = Math.sqrt(
|
||||
(n * sumAA - sumA * sumA) * (n * sumBB - sumB * sumB),
|
||||
);
|
||||
|
||||
if (denominator === 0) return { r: 0, strength: "keine", n };
|
||||
|
||||
const r = Math.round(((n * sumAB - sumA * sumB) / denominator) * 1000) / 1000;
|
||||
const absR = Math.abs(r);
|
||||
|
||||
let strength: CorrelationResult["strength"];
|
||||
if (absR >= 0.7) strength = "stark";
|
||||
else if (absR >= 0.4) strength = "moderat";
|
||||
else if (absR >= 0.2) strength = "schwach";
|
||||
else strength = "keine";
|
||||
|
||||
return { r, strength, n };
|
||||
}
|
||||
|
||||
/**
|
||||
* v1.2.5 (M-CS3) — significance-gated Pearson for USER-FACING surfaces.
|
||||
* v1.2.5 (M-CS3) — significance-gated Pearson, the one correlation every
|
||||
* surface reports.
|
||||
*
|
||||
* The plain `pearsonCorrelation` above assigns a strength band from |r|
|
||||
* alone with no significance test and a default floor of only 5 pairs —
|
||||
* so a 5-point r ≈ 0.7 fluke surfaces as a "stark" correlation. Every
|
||||
* surfaced insight-card / status-snapshot correlation must instead clear
|
||||
* the same rigorous bar the `/insights` correlation engine enforces:
|
||||
* A strength band read from |r| alone, above a floor of five pairs, let a
|
||||
* 5-point r ≈ 0.7 fluke surface as a "stark" correlation. Every reported
|
||||
* correlation clears the bar the `/insights` correlation engine enforces:
|
||||
*
|
||||
* - n >= 20 paired points (the engine's `MIN_PAIRED_N`), AND
|
||||
* - a two-sided Student-t p-value < 0.05 (`MAX_P_VALUE`).
|
||||
*
|
||||
* Below either bar this returns `null` so the caller suppresses the
|
||||
* correlation exactly as it already does for the < minPairs case — no
|
||||
* correlation — no
|
||||
* call-site shape change. The returned `{ r, strength, n }` keeps the
|
||||
* legacy shape (German strength band) so the snapshot byte-shape is
|
||||
* unchanged on the happy path.
|
||||
|
||||
@@ -32,7 +32,7 @@ import { getBpTargets } from "@/lib/analytics/bp-targets";
|
||||
import { isBpReadingInTarget } from "@/lib/analytics/bp-in-target";
|
||||
import {
|
||||
pairByTimestamp,
|
||||
pearsonCorrelation,
|
||||
significantPearsonCorrelation,
|
||||
type CorrelationResult,
|
||||
} from "@/lib/analytics/correlations";
|
||||
import { getMedicationCategories } from "@/lib/medication-category";
|
||||
@@ -1656,7 +1656,7 @@ export async function extractFeatures(
|
||||
}));
|
||||
|
||||
const computeCorr = (a: DataPoint[], b: DataPoint[]) =>
|
||||
pearsonCorrelation(pairByTimestamp(a, b));
|
||||
significantPearsonCorrelation(pairByTimestamp(a, b));
|
||||
|
||||
const sleepPoints = toDataPoints(sleepData);
|
||||
|
||||
|
||||
@@ -428,9 +428,9 @@ export interface MoodMetricCorrelation {
|
||||
*
|
||||
* This routes through `significantPearsonCorrelation` — n >= 20 AND
|
||||
* p < 0.05 — for the same reason the weight, blood-pressure and
|
||||
* mood-status surfaces do: the plain `pearsonCorrelation` bands from |r|
|
||||
* alone above a floor of five pairs, so five days of noise sloping the
|
||||
* right way arrive at the reader labelled "strong". A mood page is where
|
||||
* mood-status surfaces do: a band read from |r| alone above a floor of
|
||||
* five pairs lets five days of noise sloping the right way arrive at the
|
||||
* reader labelled "strong". A mood page is where
|
||||
* that costs the most, because the reader is looking for something to
|
||||
* change about their week.
|
||||
*/
|
||||
|
||||
Reference in new issue
Block a user