mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 20:42:43 +00:00
fix(insights): report every correlation through the significance gate
The /insights envelope (weight x BP, mood x BP, mood x weight, mood x pulse, BP medication continuity) and the AI feature payload computed their coefficients with an ungated Pearson that banded |r| above five pairs, so five days of noise sloping the right way read as a strong correlation there while every other surface refused the same points. Both now use `significantPearsonCorrelation` (n >= 20 and p < 0.05, through the engine's exact p-value), and the ungated function is gone. The allowlist guard that tracked its remaining callers becomes a guard that there is one Pearson implementation.
This commit is contained in:
1 parent
04855b1a67
commit
a22bf91311
9 files changed
+150
-269
No files matched your search
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Structural guard: one Pearson correlation, one bar.
|
||||
*
|
||||
* Three implementations used to answer "how correlated are these two
|
||||
* series": the engine's `pearson` (exact p-value, `lib/insights/
|
||||
* correlations.ts`), its gated wrapper `significantPearsonCorrelation`
|
||||
* (n >= 20 and p < 0.05), and an ungated `pearsonCorrelation` that banded a
|
||||
* coefficient from |r| alone above five pairs. The ungated one fed the
|
||||
* /insights envelope and the AI feature payload, so five days of noise
|
||||
* sloping the right way reached a reader labelled "strong" there while every
|
||||
* other surface refused the same five points. It is gone; every surface goes
|
||||
* through the wrapper, and the wrapper through the engine.
|
||||
*
|
||||
* The matcher flags a function whose name says Pearson or correlation
|
||||
* coefficient, declared anywhere but the engine and the wrapper's file, and
|
||||
* any mention of the deleted name. It is a tripwire: a coefficient computed
|
||||
* inline under another name slips it. Comments are stripped first; the
|
||||
* planted cases prove it fires.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { walkSourceFiles } from "./helpers/source-files";
|
||||
|
||||
const SRC = join(process.cwd(), "src");
|
||||
|
||||
const HOMES = new Set([
|
||||
"lib/insights/correlations.ts",
|
||||
"lib/analytics/correlations.ts",
|
||||
]);
|
||||
|
||||
const PEARSON_DEFINITION =
|
||||
/(?:function\s+|const\s+)\w*(?:pearson|correlationCoefficient)\w*\s*(?:=|\()/i;
|
||||
const DELETED = /(?<![A-Za-z])pearsonCorrelation\b/;
|
||||
|
||||
function stripComments(text: string): string {
|
||||
return text.replace(/\/\*[\s\S]*?\*\//g, "").replace(/^\s*\/\/.*$/gm, "");
|
||||
}
|
||||
|
||||
function sourceFiles(): string[] {
|
||||
return walkSourceFiles(SRC, { floor: 3000 }).filter(
|
||||
(p) =>
|
||||
!p.startsWith("generated/") &&
|
||||
!p.includes("__tests__/") &&
|
||||
!/\.test\.tsx?$/.test(p),
|
||||
);
|
||||
}
|
||||
|
||||
describe("one Pearson correlation", () => {
|
||||
it("the matchers fire on a second implementation", () => {
|
||||
expect(PEARSON_DEFINITION.test("export function pearsonR(xs, ys) {")).toBe(
|
||||
true,
|
||||
);
|
||||
expect(PEARSON_DEFINITION.test("const quickPearson = (a, b) =>")).toBe(
|
||||
true,
|
||||
);
|
||||
expect(DELETED.test("pearsonCorrelation(pairs)")).toBe(true);
|
||||
expect(DELETED.test("significantPearsonCorrelation(pairs)")).toBe(false);
|
||||
});
|
||||
|
||||
it("both homes still hold their function", () => {
|
||||
const engine = readFileSync(
|
||||
join(SRC, "lib/insights/correlations.ts"),
|
||||
"utf8",
|
||||
);
|
||||
const wrapper = readFileSync(
|
||||
join(SRC, "lib/analytics/correlations.ts"),
|
||||
"utf8",
|
||||
);
|
||||
expect(engine).toMatch(/export function pearson\s*\(/);
|
||||
expect(wrapper).toMatch(
|
||||
/export function significantPearsonCorrelation\s*\(/,
|
||||
);
|
||||
});
|
||||
|
||||
it("no other file defines a Pearson coefficient or calls the deleted one", () => {
|
||||
const offenders = sourceFiles().filter((file) => {
|
||||
const code = stripComments(readFileSync(join(SRC, file), "utf8"));
|
||||
if (DELETED.test(code)) return true;
|
||||
return !HOMES.has(file) && PEARSON_DEFINITION.test(code);
|
||||
});
|
||||
expect(offenders).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,117 +0,0 @@
|
||||
/**
|
||||
* Structural guard on the unguarded Pearson helper.
|
||||
*
|
||||
* `pearsonCorrelation` bands a coefficient from |r| alone above a floor of
|
||||
* five pairs, with no significance test. Its guarded twin
|
||||
* `significantPearsonCorrelation` sits directly beneath it in the same file
|
||||
* and exists because of exactly one failure: five days of noise sloping the
|
||||
* right way reaching a reader labelled "strong". The comment above the twin
|
||||
* says so in as many words.
|
||||
*
|
||||
* The twin was written once and adopted one surface at a time, which is how
|
||||
* the mood page kept the naive call for four minor lines after the surfaces
|
||||
* either side of it had moved. Nothing failed when it was skipped, because
|
||||
* nothing was watching. This is what watches: a new caller of the unguarded
|
||||
* helper has to be added to the list below, in a diff a reviewer reads, next
|
||||
* to a written reason.
|
||||
*
|
||||
* It is a tripwire, not a proof. It cannot say the remaining entries are
|
||||
* right — only that the set has not grown without someone editing this file.
|
||||
* The two entries it carries today are named as open work, not as blessings.
|
||||
*/
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { walkSourceFiles } from "./helpers/source-files";
|
||||
|
||||
const SRC = join(process.cwd(), "src");
|
||||
|
||||
/**
|
||||
* Files that may still call `pearsonCorrelation` directly, and why.
|
||||
*
|
||||
* Neither entry is settled. Both are the same migration the user-facing
|
||||
* surfaces already made, still owed by two payload builders whose consumers
|
||||
* pin the n >= 5 shape; moving them changes what those payloads contain and
|
||||
* belongs in its own change with its own tests, not smuggled into this one.
|
||||
*/
|
||||
const ALLOWED: Record<string, string> = {
|
||||
"app/api/insights/comprehensive/route.ts":
|
||||
"Five correlations on the /insights envelope (weight x BP, mood x BP, " +
|
||||
"mood x weight, mood x pulse, BP-medication continuity). Open: they " +
|
||||
"carry the same small-n exposure the mood page had, and the page's " +
|
||||
"cards read `n` straight off the envelope.",
|
||||
"lib/insights/features.ts":
|
||||
"Eight cross-metric coefficients on the AI feature payload. Open: the " +
|
||||
"prompt reads them as findings, so the same significance bar applies; " +
|
||||
"the snapshot shape and its budget tests move with it.",
|
||||
};
|
||||
|
||||
/** Every non-test, non-generated source file under `src/`. */
|
||||
function sourceFiles(): string[] {
|
||||
return walkSourceFiles(SRC, { floor: 3000 })
|
||||
.filter((p) => !p.startsWith("generated/"))
|
||||
.filter((p) => !p.includes("__tests__"))
|
||||
.filter((p) => !p.endsWith(".test.ts") && !p.endsWith(".test.tsx"))
|
||||
.sort();
|
||||
}
|
||||
|
||||
function read(rel: string): string {
|
||||
return readFileSync(join(SRC, rel), "utf8");
|
||||
}
|
||||
|
||||
/**
|
||||
* A call to the unguarded helper. Whitespace-tolerant and anchored on a word
|
||||
* boundary so `significantPearsonCorrelation(` cannot match: a matcher that
|
||||
* quietly matches nothing is the failure mode this repository has shipped
|
||||
* before, so the count is asserted below as well.
|
||||
*/
|
||||
const CALL = /(?<![A-Za-z])pearsonCorrelation\s*\(/;
|
||||
|
||||
describe("the unguarded Pearson helper has a closed caller set", () => {
|
||||
it("finds callers at all, so an empty result cannot read as a pass", () => {
|
||||
const definition = read("lib/analytics/correlations.ts");
|
||||
expect(definition).toMatch(/export function pearsonCorrelation\s*\(/);
|
||||
expect(
|
||||
definition,
|
||||
"the guarded twin must exist for this guard to mean anything",
|
||||
).toMatch(/export function significantPearsonCorrelation\s*\(/);
|
||||
|
||||
const callers = sourceFiles().filter(
|
||||
(rel) => rel !== "lib/analytics/correlations.ts" && CALL.test(read(rel)),
|
||||
);
|
||||
expect(callers.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("names every caller outside the allowlist", () => {
|
||||
const callers = sourceFiles().filter(
|
||||
(rel) => rel !== "lib/analytics/correlations.ts" && CALL.test(read(rel)),
|
||||
);
|
||||
|
||||
expect(
|
||||
callers.filter((rel) => !(rel in ALLOWED)),
|
||||
"A new caller of the unguarded `pearsonCorrelation`. A surface a " +
|
||||
"person reads must use `significantPearsonCorrelation` (n >= 20 " +
|
||||
"AND p < 0.05). If this one genuinely cannot, add it to ALLOWED " +
|
||||
"in this file with the reason, so the exception is reviewed rather " +
|
||||
"than assumed.",
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps the allowlist honest by refusing stale entries", () => {
|
||||
const callers = new Set(
|
||||
sourceFiles().filter(
|
||||
(rel) =>
|
||||
rel !== "lib/analytics/correlations.ts" && CALL.test(read(rel)),
|
||||
),
|
||||
);
|
||||
|
||||
// An entry whose file no longer calls the helper is a licence nobody
|
||||
// needs any more; leaving it standing is how the next one gets waved in.
|
||||
expect(
|
||||
Object.keys(ALLOWED).filter((rel) => !callers.has(rel)),
|
||||
"this file is on the allowlist but no longer calls the unguarded " +
|
||||
"helper — drop the entry",
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -569,16 +569,10 @@ describe("GET /api/insights/comprehensive — envelope shape", () => {
|
||||
expect(body.data.moodSummary?.count).toBe(2);
|
||||
});
|
||||
|
||||
it("derives weight × BP scatter from daily-bucketed series", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
// Five day-aligned pairs (n >= 5 unlocks pearsonCorrelation).
|
||||
const days = [
|
||||
"2026-05-01",
|
||||
"2026-05-02",
|
||||
"2026-05-03",
|
||||
"2026-05-04",
|
||||
"2026-05-05",
|
||||
];
|
||||
function mockWeightBpDays(count: number) {
|
||||
const days = Array.from({ length: count }, (_, i) =>
|
||||
new Date(Date.UTC(2026, 4, 1 + i)).toISOString().slice(0, 10),
|
||||
);
|
||||
(buildComprehensiveAggregate as ReturnType<typeof vi.fn>).mockResolvedValue(
|
||||
{
|
||||
summaries: {},
|
||||
@@ -592,22 +586,41 @@ describe("GET /api/insights/comprehensive — envelope shape", () => {
|
||||
})),
|
||||
},
|
||||
firstMeasurementAt: new Date(),
|
||||
totalMeasurements: 10,
|
||||
totalMeasurements: count * 2,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
type WeightBpBody = {
|
||||
data: {
|
||||
scatterData: Array<{ weight: number; sysBP: number }>;
|
||||
weightBpCorrelation: { r: number; n: number } | null;
|
||||
};
|
||||
};
|
||||
|
||||
it("derives weight × BP scatter from daily-bucketed series", async () => {
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
mockWeightBpDays(25);
|
||||
|
||||
const res = await callGet(makeReq());
|
||||
const body = (await res.json()) as {
|
||||
data: {
|
||||
scatterData: Array<{ weight: number; sysBP: number }>;
|
||||
weightBpCorrelation: { r: number; n: number } | null;
|
||||
};
|
||||
};
|
||||
expect(body.data.scatterData).toHaveLength(5);
|
||||
const body = (await res.json()) as WeightBpBody;
|
||||
expect(body.data.scatterData).toHaveLength(25);
|
||||
expect(body.data.scatterData[0]).toEqual({ weight: 80, sysBP: 120 });
|
||||
// Perfect linear relationship → r === 1.
|
||||
// Perfect linear relationship over 25 days → r === 1.
|
||||
expect(body.data.weightBpCorrelation?.r).toBe(1);
|
||||
expect(body.data.weightBpCorrelation?.n).toBe(5);
|
||||
expect(body.data.weightBpCorrelation?.n).toBe(25);
|
||||
});
|
||||
|
||||
it("reports no correlation below the significance bar every surface uses", async () => {
|
||||
// Five perfectly aligned days: the scatter shows them, but five points
|
||||
// cannot carry a correlation (n >= 20 and p < 0.05).
|
||||
vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
|
||||
mockWeightBpDays(5);
|
||||
|
||||
const res = await callGet(makeReq());
|
||||
const body = (await res.json()) as WeightBpBody;
|
||||
expect(body.data.scatterData).toHaveLength(5);
|
||||
expect(body.data.weightBpCorrelation).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -849,13 +862,10 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing (
|
||||
// tier in R12. Both read the UTC `dailyByType` / UTC `scheduledFor`
|
||||
// slice, so their output must be invariant to the caller's timezone.
|
||||
// Run the SAME fixture under UTC then America/New_York and deep-equal.
|
||||
const days = [
|
||||
"2026-05-01",
|
||||
"2026-05-02",
|
||||
"2026-05-03",
|
||||
"2026-05-04",
|
||||
"2026-05-05",
|
||||
];
|
||||
// 25 days: a correlation has to clear n >= 20 and p < 0.05 to exist.
|
||||
const days = Array.from({ length: 25 }, (_, i) =>
|
||||
new Date(Date.UTC(2026, 4, 1 + i)).toISOString().slice(0, 10),
|
||||
);
|
||||
const aggregate = {
|
||||
summaries: {},
|
||||
bpRawRows: { sys: [], dia: [] },
|
||||
@@ -896,20 +906,20 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing (
|
||||
scheduleRevisions: [],
|
||||
pauseEras: [],
|
||||
};
|
||||
// Intake taken on the first three days, logged as skipped on the last
|
||||
// two → a non-degenerate continuity series. The skips used to be absent
|
||||
// Intake taken on the first fifteen days, logged as skipped on the
|
||||
// last ten → a non-degenerate continuity series. The skips used to be absent
|
||||
// rows: a day with no record read as a measured 0 % continuity, which is
|
||||
// the fabrication the route no longer makes, and a fixture that relies on
|
||||
// it would be pinning the defect rather than the timezone question this
|
||||
// test asks. Recording the two days as skips keeps five paired days AND
|
||||
// test asks. Recording the days as skips keeps every day paired AND
|
||||
// keeps every one of them a thing the person actually logged.
|
||||
const intakeEvents = days.map((d, i) => ({
|
||||
medicationId: "med-bp",
|
||||
userId: "user-comp-1",
|
||||
deletedAt: null,
|
||||
scheduledFor: new Date(`${d}T07:00:00Z`),
|
||||
takenAt: i < 3 ? new Date(`${d}T07:05:00Z`) : null,
|
||||
skipped: i >= 3,
|
||||
takenAt: i < 15 ? new Date(`${d}T07:05:00Z`) : null,
|
||||
skipped: i >= 15,
|
||||
}));
|
||||
|
||||
function primeMocks() {
|
||||
@@ -948,7 +958,8 @@ describe("GET /api/insights/comprehensive — mood × metric local-day pairing (
|
||||
);
|
||||
// And the guard is meaningful only if these joins actually produced
|
||||
// output to compare.
|
||||
expect(utcBody.data.scatterData).toHaveLength(5);
|
||||
expect(utcBody.data.scatterData).toHaveLength(25);
|
||||
expect(utcBody.data.weightBpCorrelation).not.toBeNull();
|
||||
expect(utcBody.data.bpMedicationCorrelation).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
generateAlerts,
|
||||
} from "@/lib/analytics/classifications";
|
||||
import {
|
||||
pearsonCorrelation,
|
||||
significantPearsonCorrelation,
|
||||
type PairedPoint,
|
||||
} from "@/lib/analytics/correlations";
|
||||
import {
|
||||
@@ -285,7 +285,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) {
|
||||
const weightDaily = aggregate.dailyByType.WEIGHT ?? [];
|
||||
const sysDaily = aggregate.dailyByType.BLOOD_PRESSURE_SYS ?? [];
|
||||
const weightBpPairs: PairedPoint[] = joinDailyByDay(weightDaily, sysDaily);
|
||||
const weightBpCorrelation = pearsonCorrelation(weightBpPairs);
|
||||
const weightBpCorrelation = significantPearsonCorrelation(weightBpPairs);
|
||||
const scatterData = weightBpPairs.map((p) => ({
|
||||
weight: p.a,
|
||||
sysBP: p.b,
|
||||
@@ -308,11 +308,11 @@ export async function buildComprehensiveResponse(user: AuthedUser) {
|
||||
);
|
||||
|
||||
const moodBpPairs = pairMoodWithDaily(dailyMoodEntries, sysDailyLocal);
|
||||
const moodBpCorrelation = pearsonCorrelation(moodBpPairs);
|
||||
const moodBpCorrelation = significantPearsonCorrelation(moodBpPairs);
|
||||
const moodBpScatterData = moodBpPairs.map((p) => ({ mood: p.a, sysBP: p.b }));
|
||||
|
||||
const moodWeightPairs = pairMoodWithDaily(dailyMoodEntries, weightDailyLocal);
|
||||
const moodWeightCorrelation = pearsonCorrelation(moodWeightPairs);
|
||||
const moodWeightCorrelation = significantPearsonCorrelation(moodWeightPairs);
|
||||
const moodWeightScatterData = moodWeightPairs.map((p) => ({
|
||||
mood: p.a,
|
||||
weight: p.b,
|
||||
@@ -374,7 +374,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) {
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([day, stats]) => ({ day, value: stats.sum / stats.count }));
|
||||
const moodPulsePairs = pairMoodWithDaily(dailyMoodEntries, restingPulseDaily);
|
||||
const moodPulseCorrelation = pearsonCorrelation(moodPulsePairs);
|
||||
const moodPulseCorrelation = significantPearsonCorrelation(moodPulsePairs);
|
||||
const moodPulseScatterData = moodPulsePairs.map((p) => ({
|
||||
mood: p.a,
|
||||
pulse: p.b,
|
||||
@@ -558,7 +558,7 @@ export async function buildComprehensiveResponse(user: AuthedUser) {
|
||||
});
|
||||
}
|
||||
|
||||
const corr = pearsonCorrelation(pairs);
|
||||
const corr = significantPearsonCorrelation(pairs);
|
||||
if (corr) {
|
||||
bpMedicationCorrelation = {
|
||||
...corr,
|
||||
@@ -667,7 +667,7 @@ function deriveDailyLocalMeans(
|
||||
|
||||
/**
|
||||
* Daily-key inner join between two daily-aggregated series. Returns
|
||||
* `PairedPoint` shapes compatible with `pearsonCorrelation`.
|
||||
* `PairedPoint` shapes compatible with `significantPearsonCorrelation`.
|
||||
*
|
||||
* Inputs are sorted ASC by `day` (the aggregator's ORDER BY
|
||||
* guarantees that). The walk is O(n + m) merge-join style instead of
|
||||
|
||||
@@ -2,7 +2,6 @@ import { describe, it, expect } from "vitest";
|
||||
import type { DataPoint } from "../trends";
|
||||
import {
|
||||
pairByTimestamp,
|
||||
pearsonCorrelation,
|
||||
significantPearsonCorrelation,
|
||||
} from "../correlations";
|
||||
import type { PairedPoint } from "../correlations";
|
||||
@@ -52,62 +51,6 @@ describe("pairByTimestamp", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("pearsonCorrelation", () => {
|
||||
it("returns null for fewer than minPairs", () => {
|
||||
const pairs = [
|
||||
{ a: 1, b: 2, date: new Date() },
|
||||
{ a: 2, b: 4, date: new Date() },
|
||||
];
|
||||
expect(pearsonCorrelation(pairs)).toBeNull();
|
||||
});
|
||||
|
||||
it("detects perfect positive correlation", () => {
|
||||
const pairs = Array.from({ length: 10 }, (_, i) => ({
|
||||
a: i,
|
||||
b: i * 2,
|
||||
date: new Date(),
|
||||
}));
|
||||
const result = pearsonCorrelation(pairs)!;
|
||||
expect(result.r).toBe(1);
|
||||
expect(result.strength).toBe("stark");
|
||||
expect(result.n).toBe(10);
|
||||
});
|
||||
|
||||
it("detects perfect negative correlation", () => {
|
||||
const pairs = Array.from({ length: 10 }, (_, i) => ({
|
||||
a: i,
|
||||
b: 100 - i * 2,
|
||||
date: new Date(),
|
||||
}));
|
||||
const result = pearsonCorrelation(pairs)!;
|
||||
expect(result.r).toBe(-1);
|
||||
expect(result.strength).toBe("stark");
|
||||
});
|
||||
|
||||
it("detects no correlation for random-like data", () => {
|
||||
const pairs = [
|
||||
{ a: 1, b: 5, date: new Date() },
|
||||
{ a: 2, b: 3, date: new Date() },
|
||||
{ a: 3, b: 7, date: new Date() },
|
||||
{ a: 4, b: 2, date: new Date() },
|
||||
{ a: 5, b: 6, date: new Date() },
|
||||
];
|
||||
const result = pearsonCorrelation(pairs)!;
|
||||
expect(Math.abs(result.r)).toBeLessThan(0.4);
|
||||
});
|
||||
|
||||
it("returns keine for constant values", () => {
|
||||
const pairs = Array.from({ length: 5 }, () => ({
|
||||
a: 5,
|
||||
b: 5,
|
||||
date: new Date(),
|
||||
}));
|
||||
const result = pearsonCorrelation(pairs)!;
|
||||
expect(result.r).toBe(0);
|
||||
expect(result.strength).toBe("keine");
|
||||
});
|
||||
});
|
||||
|
||||
describe("significantPearsonCorrelation (M-CS3 — n>=20 AND p<0.05)", () => {
|
||||
const mk = (a: number, b: number): PairedPoint => ({
|
||||
a,
|
||||
@@ -116,14 +59,9 @@ describe("significantPearsonCorrelation (M-CS3 — n>=20 AND p<0.05)", () => {
|
||||
});
|
||||
|
||||
it("suppresses a 5-pair r≈0.7 dataset (below the n>=20 floor)", () => {
|
||||
// Five points with a strong-looking slope — exactly the small-n fluke the
|
||||
// plain `pearsonCorrelation` would surface as a "stark" correlation.
|
||||
// Five points with a strong-looking slope: the small-n fluke an
|
||||
// ungated band would call a "stark" correlation.
|
||||
const pairs = [mk(1, 1), mk(2, 2), mk(3, 2), mk(4, 4), mk(5, 3)];
|
||||
// Sanity-check the legacy path WOULD have surfaced it (non-null, n=5).
|
||||
const legacy = pearsonCorrelation(pairs);
|
||||
expect(legacy).not.toBeNull();
|
||||
expect(legacy!.n).toBe(5);
|
||||
// The gated path refuses it — too few pairs.
|
||||
expect(significantPearsonCorrelation(pairs)).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
@@ -72,54 +72,18 @@ export function pairByTimestamp(
|
||||
}
|
||||
|
||||
/**
|
||||
* Pearson correlation coefficient.
|
||||
* Returns null if fewer than minPairs data points.
|
||||
*/
|
||||
export function pearsonCorrelation(
|
||||
pairs: PairedPoint[],
|
||||
minPairs: number = 5,
|
||||
): CorrelationResult | null {
|
||||
if (pairs.length < minPairs) return null;
|
||||
|
||||
const n = pairs.length;
|
||||
const sumA = pairs.reduce((s, p) => s + p.a, 0);
|
||||
const sumB = pairs.reduce((s, p) => s + p.b, 0);
|
||||
const sumAB = pairs.reduce((s, p) => s + p.a * p.b, 0);
|
||||
const sumAA = pairs.reduce((s, p) => s + p.a * p.a, 0);
|
||||
const sumBB = pairs.reduce((s, p) => s + p.b * p.b, 0);
|
||||
|
||||
const denominator = Math.sqrt(
|
||||
(n * sumAA - sumA * sumA) * (n * sumBB - sumB * sumB),
|
||||
);
|
||||
|
||||
if (denominator === 0) return { r: 0, strength: "keine", n };
|
||||
|
||||
const r = Math.round(((n * sumAB - sumA * sumB) / denominator) * 1000) / 1000;
|
||||
const absR = Math.abs(r);
|
||||
|
||||
let strength: CorrelationResult["strength"];
|
||||
if (absR >= 0.7) strength = "stark";
|
||||
else if (absR >= 0.4) strength = "moderat";
|
||||
else if (absR >= 0.2) strength = "schwach";
|
||||
else strength = "keine";
|
||||
|
||||
return { r, strength, n };
|
||||
}
|
||||
|
||||
/**
|
||||
* v1.2.5 (M-CS3) — significance-gated Pearson for USER-FACING surfaces.
|
||||
* v1.2.5 (M-CS3) — significance-gated Pearson, the one correlation every
|
||||
* surface reports.
|
||||
*
|
||||
* The plain `pearsonCorrelation` above assigns a strength band from |r|
|
||||
* alone with no significance test and a default floor of only 5 pairs —
|
||||
* so a 5-point r ≈ 0.7 fluke surfaces as a "stark" correlation. Every
|
||||
* surfaced insight-card / status-snapshot correlation must instead clear
|
||||
* the same rigorous bar the `/insights` correlation engine enforces:
|
||||
* A strength band read from |r| alone, above a floor of five pairs, let a
|
||||
* 5-point r ≈ 0.7 fluke surface as a "stark" correlation. Every reported
|
||||
* correlation clears the bar the `/insights` correlation engine enforces:
|
||||
*
|
||||
* - n >= 20 paired points (the engine's `MIN_PAIRED_N`), AND
|
||||
* - a two-sided Student-t p-value < 0.05 (`MAX_P_VALUE`).
|
||||
*
|
||||
* Below either bar this returns `null` so the caller suppresses the
|
||||
* correlation exactly as it already does for the < minPairs case — no
|
||||
* correlation — no
|
||||
* call-site shape change. The returned `{ r, strength, n }` keeps the
|
||||
* legacy shape (German strength band) so the snapshot byte-shape is
|
||||
* unchanged on the happy path.
|
||||
|
||||
@@ -32,7 +32,7 @@ import { getBpTargets } from "@/lib/analytics/bp-targets";
|
||||
import { isBpReadingInTarget } from "@/lib/analytics/bp-in-target";
|
||||
import {
|
||||
pairByTimestamp,
|
||||
pearsonCorrelation,
|
||||
significantPearsonCorrelation,
|
||||
type CorrelationResult,
|
||||
} from "@/lib/analytics/correlations";
|
||||
import { getMedicationCategories } from "@/lib/medication-category";
|
||||
@@ -1656,7 +1656,7 @@ export async function extractFeatures(
|
||||
}));
|
||||
|
||||
const computeCorr = (a: DataPoint[], b: DataPoint[]) =>
|
||||
pearsonCorrelation(pairByTimestamp(a, b));
|
||||
significantPearsonCorrelation(pairByTimestamp(a, b));
|
||||
|
||||
const sleepPoints = toDataPoints(sleepData);
|
||||
|
||||
|
||||
@@ -428,9 +428,9 @@ export interface MoodMetricCorrelation {
|
||||
*
|
||||
* This routes through `significantPearsonCorrelation` — n >= 20 AND
|
||||
* p < 0.05 — for the same reason the weight, blood-pressure and
|
||||
* mood-status surfaces do: the plain `pearsonCorrelation` bands from |r|
|
||||
* alone above a floor of five pairs, so five days of noise sloping the
|
||||
* right way arrive at the reader labelled "strong". A mood page is where
|
||||
* mood-status surfaces do: a band read from |r| alone above a floor of
|
||||
* five pairs lets five days of noise sloping the right way arrive at the
|
||||
* reader labelled "strong". A mood page is where
|
||||
* that costs the most, because the reader is looking for something to
|
||||
* change about their week.
|
||||
*/
|
||||
|
||||
@@ -159,7 +159,7 @@ describe("GET /api/insights/comprehensive — server cache", () => {
|
||||
|
||||
// A tight fixture: 5 paired BP readings + 5 weight rows + 5 mood
|
||||
// entries spread across 5 distinct days. Enough to populate every
|
||||
// correlation block (n >= 5 is the pearsonCorrelation gate).
|
||||
// correlation key (each is null below the significance bar).
|
||||
const now = Date.now();
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const dayDates = Array.from(
|
||||
|
||||
Reference in new issue
Block a user