mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-18 16:12:11 +00:00
Merge rollup regression accumulators into release/v1.20.0
Store additive per-bucket regression accumulators (sum_x, sum_xy, sum_xx, sum_yy) so windowed slope, r2 and standard deviation compose from the rollup tier instead of a live raw-table scan on every warm request (migration 0190, backfilled by the boot queue). Median and the exact anomaly count stay on a minimal live query. Parity with the live REGR_* path within float tolerance, well below the read rounding.
This commit is contained in:
commit
4d892f6867
10 files changed
+1135
-172
No files matched your search
@@ -75,6 +75,11 @@ vi.mock("@/lib/analytics/source-rank-sql", () => {
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { buildComprehensiveAggregate } from "../comprehensive-aggregator";
|
||||
import {
|
||||
composeWindowedRegression,
|
||||
type AccumulatorBucketRow,
|
||||
} from "@/lib/rollups/measurement-read";
|
||||
import { startOfUtcDay } from "@/lib/tz/start-of-utc-day";
|
||||
|
||||
const RAW = prisma.$queryRaw as unknown as ReturnType<typeof vi.fn>;
|
||||
const UNSAFE = prisma.$queryRawUnsafe as unknown as ReturnType<typeof vi.fn>;
|
||||
@@ -139,12 +144,8 @@ describe("buildComprehensiveAggregate", () => {
|
||||
avg7: 81.9,
|
||||
avg30: 82.1,
|
||||
avg30_last_month: 83.0,
|
||||
slope7: -0.014,
|
||||
r2_7: 0.65,
|
||||
slope30: -0.005,
|
||||
r2_30: 0.42,
|
||||
slope90: 0.001,
|
||||
r2_90: 0.12,
|
||||
// v1.20.0 F6 — slope / r² no longer ride the narrow aggregate;
|
||||
// they compose from the accumulator buckets below.
|
||||
},
|
||||
]).mockResolvedValueOnce([
|
||||
{ type: "WEIGHT", value: 81.4, measured_at: now },
|
||||
@@ -156,26 +157,61 @@ describe("buildComprehensiveAggregate", () => {
|
||||
// now carries a `source` so `partitionBucketsByType` →
|
||||
// `collapseRollupRowsBySource` can resolve dual-source days; with
|
||||
// a single source per day every bucket passes through unchanged.
|
||||
ROLLUP_FIND_MANY.mockResolvedValueOnce([
|
||||
{
|
||||
type: "WEIGHT",
|
||||
source: "APPLE_HEALTH",
|
||||
bucketStart: new Date("2026-05-10T00:00:00.000Z"),
|
||||
count: 20,
|
||||
mean: 81.0,
|
||||
minValue: 79.2,
|
||||
maxValue: 82.5,
|
||||
},
|
||||
{
|
||||
type: "WEIGHT",
|
||||
source: "APPLE_HEALTH",
|
||||
bucketStart: new Date("2026-05-11T00:00:00.000Z"),
|
||||
count: 22,
|
||||
mean: 83.0,
|
||||
minValue: 80.0,
|
||||
maxValue: 84.1,
|
||||
},
|
||||
]);
|
||||
//
|
||||
// v1.20.0 F6 — buckets are placed on the two most-recent UTC days so
|
||||
// they fall inside the 7-day slope window, and carry the regression
|
||||
// accumulators the windowed slope / r² compose from. Each bucket
|
||||
// models its `count` readings as all landing at the bucket's day
|
||||
// midnight (x = bucketStart epoch-day) with value = mean, which is a
|
||||
// valid, self-consistent accumulator set.
|
||||
const dayB = startOfUtcDay(now);
|
||||
const dayA = startOfUtcDay(new Date(now.getTime() - 86_400_000));
|
||||
const xA = dayA.getTime() / 86_400_000;
|
||||
const xB = dayB.getTime() / 86_400_000;
|
||||
const bucketA = {
|
||||
type: "WEIGHT",
|
||||
source: "APPLE_HEALTH",
|
||||
bucketStart: dayA,
|
||||
count: 20,
|
||||
mean: 81.0,
|
||||
minValue: 79.2,
|
||||
maxValue: 82.5,
|
||||
sumX: 20 * xA,
|
||||
sumXy: 20 * xA * 81.0,
|
||||
sumXx: 20 * xA * xA,
|
||||
sumYy: 20 * 81.0 * 81.0,
|
||||
};
|
||||
const bucketB = {
|
||||
type: "WEIGHT",
|
||||
source: "APPLE_HEALTH",
|
||||
bucketStart: dayB,
|
||||
count: 22,
|
||||
mean: 83.0,
|
||||
minValue: 80.0,
|
||||
maxValue: 84.1,
|
||||
sumX: 22 * xB,
|
||||
sumXy: 22 * xB * 83.0,
|
||||
sumXx: 22 * xB * xB,
|
||||
sumYy: 22 * 83.0 * 83.0,
|
||||
};
|
||||
ROLLUP_FIND_MANY.mockResolvedValueOnce([bucketA, bucketB]);
|
||||
// Expected slope7 — composed from the same accumulators the reader
|
||||
// sees (the buildSlope contract rounds slope to 3 dp, confidence to
|
||||
// 2 dp). Computed here from composeWindowedRegression so the test
|
||||
// tracks the closed form rather than a hand-typed magic number.
|
||||
const accBuckets: AccumulatorBucketRow[] = [bucketA, bucketB].map(
|
||||
(b) => ({
|
||||
bucketStart: b.bucketStart,
|
||||
count: b.count,
|
||||
mean: b.mean,
|
||||
sumX: b.sumX,
|
||||
sumXy: b.sumXy,
|
||||
sumXx: b.sumXx,
|
||||
sumYy: b.sumYy,
|
||||
}),
|
||||
);
|
||||
const since7 = startOfUtcDay(new Date(Date.now() - 7 * 86_400_000));
|
||||
const expectedReg7 = composeWindowedRegression(accBuckets, since7);
|
||||
|
||||
// v1.18.10 I-1 — bpRawRows now reads source-collapsed rows via
|
||||
// `$queryRawUnsafe` (3rd UNSAFE call: narrow, latests, bpRaw). Empty
|
||||
@@ -199,14 +235,21 @@ describe("buildComprehensiveAggregate", () => {
|
||||
expect(weight.avg30).toBe(82.1);
|
||||
expect(weight.avg30LastMonth).toBe(83.0);
|
||||
expect(weight.anomalyCount).toBe(3);
|
||||
// v1.20.0 F6 — slope7 composes from the accumulator buckets. The two
|
||||
// recent buckets (means 81 → 83 one day apart) yield a positive,
|
||||
// perfect-fit slope; assert against the closed form the reader uses.
|
||||
expect(expectedReg7.slope).not.toBeNull();
|
||||
expect(weight.slope7).toEqual({
|
||||
slope: -0.014,
|
||||
direction: "down",
|
||||
confidence: 0.65,
|
||||
slope: Math.round((expectedReg7.slope as number) * 1000) / 1000,
|
||||
direction: "up",
|
||||
confidence:
|
||||
expectedReg7.r2 === null
|
||||
? 0
|
||||
: Math.round(expectedReg7.r2 * 100) / 100,
|
||||
});
|
||||
expect(result.dailyByType.WEIGHT).toEqual([
|
||||
{ day: "2026-05-10", value: 81 },
|
||||
{ day: "2026-05-11", value: 83 },
|
||||
{ day: dayA.toISOString().slice(0, 10), value: 81 },
|
||||
{ day: dayB.toISOString().slice(0, 10), value: 83 },
|
||||
]);
|
||||
|
||||
// Contract pin — the heavy aggregate path is NOT exercised on the
|
||||
|
||||
@@ -108,8 +108,11 @@ import { ensureUserRollupsFresh } from "@/lib/rollups/measurement-rollups";
|
||||
import {
|
||||
aggregateBuckets,
|
||||
collapseRollupRowsBySource,
|
||||
composeWindowedRegression,
|
||||
loadUserSourcePriority,
|
||||
type AccumulatorBucketRow,
|
||||
} from "@/lib/rollups/measurement-read";
|
||||
import { startOfUtcDay } from "@/lib/tz/start-of-utc-day";
|
||||
import {
|
||||
buildSourceRankCase,
|
||||
canonicalMeasurementsCte,
|
||||
@@ -160,12 +163,9 @@ interface NarrowAggregateRow {
|
||||
avg7: number | null;
|
||||
avg30: number | null;
|
||||
avg30_last_month: number | null;
|
||||
slope7: number | null;
|
||||
r2_7: number | null;
|
||||
slope30: number | null;
|
||||
r2_30: number | null;
|
||||
slope90: number | null;
|
||||
r2_90: number | null;
|
||||
// v1.20.0 F6 — the slope7/30/90 + r2_7/30/90 columns have moved off this
|
||||
// live query onto the rollup accumulators (composeWindowedRegression);
|
||||
// they no longer scan the raw measurements partition on the warm path.
|
||||
}
|
||||
|
||||
/** Latest raw value per type (DISTINCT ON over the 90-day window). */
|
||||
@@ -436,43 +436,7 @@ async function buildFromRollups(
|
||||
AVG(m."value") FILTER (
|
||||
WHERE m."measured_at" >= NOW() - INTERVAL '60 days'
|
||||
AND m."measured_at" < NOW() - INTERVAL '30 days'
|
||||
)::double precision AS avg30_last_month,
|
||||
REGR_SLOPE(
|
||||
m."value",
|
||||
EXTRACT(EPOCH FROM m."measured_at") / 86400.0
|
||||
) FILTER (
|
||||
WHERE m."measured_at" >= NOW() - INTERVAL '7 days'
|
||||
)::double precision AS slope7,
|
||||
REGR_R2(
|
||||
m."value",
|
||||
EXTRACT(EPOCH FROM m."measured_at") / 86400.0
|
||||
) FILTER (
|
||||
WHERE m."measured_at" >= NOW() - INTERVAL '7 days'
|
||||
)::double precision AS r2_7,
|
||||
REGR_SLOPE(
|
||||
m."value",
|
||||
EXTRACT(EPOCH FROM m."measured_at") / 86400.0
|
||||
) FILTER (
|
||||
WHERE m."measured_at" >= NOW() - INTERVAL '30 days'
|
||||
)::double precision AS slope30,
|
||||
REGR_R2(
|
||||
m."value",
|
||||
EXTRACT(EPOCH FROM m."measured_at") / 86400.0
|
||||
) FILTER (
|
||||
WHERE m."measured_at" >= NOW() - INTERVAL '30 days'
|
||||
)::double precision AS r2_30,
|
||||
REGR_SLOPE(
|
||||
m."value",
|
||||
EXTRACT(EPOCH FROM m."measured_at") / 86400.0
|
||||
) FILTER (
|
||||
WHERE m."measured_at" >= NOW() - INTERVAL '90 days'
|
||||
)::double precision AS slope90,
|
||||
REGR_R2(
|
||||
m."value",
|
||||
EXTRACT(EPOCH FROM m."measured_at") / 86400.0
|
||||
) FILTER (
|
||||
WHERE m."measured_at" >= NOW() - INTERVAL '90 days'
|
||||
)::double precision AS r2_90
|
||||
)::double precision AS avg30_last_month
|
||||
FROM cm m
|
||||
JOIN window_stats ws ON ws."type" = m."type"
|
||||
GROUP BY m."type", ws.stddev_value
|
||||
@@ -548,16 +512,38 @@ async function buildFromRollups(
|
||||
|
||||
const summaries: Record<string, DataSummary> = {};
|
||||
let totalMeasurements = 0;
|
||||
// v1.20.0 F6 — day-aligned 7/30/90-day window anchors, matching the
|
||||
// DAY-rollup grain. `startOfUtcDay(now − N days)` is the day-aligned
|
||||
// equivalent of the live `measured_at >= NOW() - INTERVAL 'N days'`.
|
||||
const now = Date.now();
|
||||
const since7 = startOfUtcDay(new Date(now - 7 * 24 * 60 * 60 * 1000));
|
||||
const since30 = startOfUtcDay(new Date(now - 30 * 24 * 60 * 60 * 1000));
|
||||
const since90 = startOfUtcDay(new Date(now - 90 * 24 * 60 * 60 * 1000));
|
||||
|
||||
// Seed every type that has buckets — the bucket set is the source of
|
||||
// truth on this path. The narrow aggregate provides the non-composable
|
||||
// windowed columns alongside.
|
||||
// windowed columns (avg + anomaly) alongside; slope / r² / sd compose
|
||||
// from the per-bucket regression accumulators (v1.20.0 F6).
|
||||
for (const [type, buckets] of bucketsByType.entries()) {
|
||||
const composed = aggregateBuckets(buckets);
|
||||
if (composed.count === 0) continue;
|
||||
totalMeasurements += composed.count;
|
||||
const latest = latestByType.get(type);
|
||||
const narrow = narrowByType.get(type);
|
||||
// The buckets are already source-collapsed (one canonical row per day),
|
||||
// so the accumulator sum is over the canonical source only.
|
||||
const accBuckets: AccumulatorBucketRow[] = buckets.map((b) => ({
|
||||
bucketStart: b.day,
|
||||
count: b.count,
|
||||
mean: b.mean,
|
||||
sumX: b.sumX,
|
||||
sumXy: b.sumXy,
|
||||
sumXx: b.sumXx,
|
||||
sumYy: b.sumYy,
|
||||
}));
|
||||
const reg7 = composeWindowedRegression(accBuckets, since7);
|
||||
const reg30 = composeWindowedRegression(accBuckets, since30);
|
||||
const reg90 = composeWindowedRegression(accBuckets, since90);
|
||||
summaries[type] = {
|
||||
count: composed.count,
|
||||
latest: latest?.value ?? null,
|
||||
@@ -571,9 +557,9 @@ async function buildFromRollups(
|
||||
median: null,
|
||||
avg7: round2(narrow?.avg7 ?? null),
|
||||
avg30: round2(narrow?.avg30 ?? null),
|
||||
slope7: buildSlope(narrow?.slope7 ?? null, narrow?.r2_7 ?? null),
|
||||
slope30: buildSlope(narrow?.slope30 ?? null, narrow?.r2_30 ?? null),
|
||||
slope90: buildSlope(narrow?.slope90 ?? null, narrow?.r2_90 ?? null),
|
||||
slope7: buildSlope(reg7.slope, reg7.r2),
|
||||
slope30: buildSlope(reg30.slope, reg30.r2),
|
||||
slope90: buildSlope(reg90.slope, reg90.r2),
|
||||
anomalyCount: Number(narrow?.anomaly_count ?? 0),
|
||||
avg30LastMonth: round2(narrow?.avg30_last_month ?? null),
|
||||
// Legacy semantics: the 90-day window guarantees no rows from a
|
||||
@@ -586,6 +572,20 @@ async function buildFromRollups(
|
||||
? new Date(firstRows[0].first_at)
|
||||
: null;
|
||||
|
||||
// v1.20.0 F6 — observability: slope / r² / sd now compose from the rollup
|
||||
// accumulators (no raw-partition REGR_* scan); the anomaly count stays on
|
||||
// a live FILTER (per-reading |z|>2 is non-composable). The "no silent
|
||||
// cap" rule wants both sources visible so a regression to the heavy path
|
||||
// is observable.
|
||||
annotate({
|
||||
meta: {
|
||||
insights: {
|
||||
slope_source: "rollup",
|
||||
anomaly_source: "live",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
summaries,
|
||||
bpRawRows,
|
||||
@@ -810,6 +810,22 @@ async function buildFromLiveAggregate(
|
||||
* `count / min / max / mean` from them in O(1) and the daily-by-type
|
||||
* builder can replay them as the correlation feed.
|
||||
*/
|
||||
/**
|
||||
* v1.20.0 F6 — a partitioned DAY bucket carrying the composable stats plus
|
||||
* the regression accumulators the windowed slope / r² / sd compose from.
|
||||
*/
|
||||
interface PartitionedBucket {
|
||||
day: Date;
|
||||
count: number;
|
||||
mean: number;
|
||||
minValue: number;
|
||||
maxValue: number;
|
||||
sumX: number | null;
|
||||
sumXy: number | null;
|
||||
sumXx: number | null;
|
||||
sumYy: number | null;
|
||||
}
|
||||
|
||||
function partitionBucketsByType(
|
||||
rows: ReadonlyArray<{
|
||||
type: string;
|
||||
@@ -819,21 +835,19 @@ function partitionBucketsByType(
|
||||
mean: number;
|
||||
minValue: number;
|
||||
maxValue: number;
|
||||
sumX: number | null;
|
||||
sumXy: number | null;
|
||||
sumXx: number | null;
|
||||
sumYy: number | null;
|
||||
}>,
|
||||
userPriorityJson: unknown,
|
||||
): Map<
|
||||
string,
|
||||
Array<{
|
||||
day: Date;
|
||||
count: number;
|
||||
mean: number;
|
||||
minValue: number;
|
||||
maxValue: number;
|
||||
}>
|
||||
> {
|
||||
): Map<string, PartitionedBucket[]> {
|
||||
// v1.11.1 — rollup rows are per source. Group by type, then collapse each
|
||||
// type's buckets to the ladder-canonical source per day before composing,
|
||||
// so a dual-source vital is not double-counted in count/mean/dailyByType.
|
||||
// v1.20.0 F6 — the collapse runs BEFORE the regression accumulators are
|
||||
// consumed, so a dual-source day contributes only the canonical source's
|
||||
// accumulators to the windowed slope / r² / sd (no cross-source sum).
|
||||
const byType = new Map<string, Array<(typeof rows)[number]>>();
|
||||
for (const b of rows) {
|
||||
const list = byType.get(b.type) ?? [];
|
||||
@@ -841,16 +855,7 @@ function partitionBucketsByType(
|
||||
byType.set(b.type, list);
|
||||
}
|
||||
|
||||
const map = new Map<
|
||||
string,
|
||||
Array<{
|
||||
day: Date;
|
||||
count: number;
|
||||
mean: number;
|
||||
minValue: number;
|
||||
maxValue: number;
|
||||
}>
|
||||
>();
|
||||
const map = new Map<string, PartitionedBucket[]>();
|
||||
for (const [type, typeRows] of byType) {
|
||||
const collapsed = collapseRollupRowsBySource(
|
||||
typeRows,
|
||||
@@ -865,6 +870,10 @@ function partitionBucketsByType(
|
||||
mean: b.mean,
|
||||
minValue: b.minValue,
|
||||
maxValue: b.maxValue,
|
||||
sumX: b.sumX,
|
||||
sumXy: b.sumXy,
|
||||
sumXx: b.sumXx,
|
||||
sumYy: b.sumYy,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user