mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 04:50:59 +00:00
observability(rollups): surface the pending-backfill regression miss
The windowed slope / r2 / sd readers compose from the per-bucket regression accumulators (migration 0190). Between a deploy and the boot-backfill completing, a tenant's pre-0190 DAY buckets still carry NULL accumulators, so composeWindowedRegression returns a full miss and the slim / comprehensive readers emit a null slope. The null is the honest answer until the backfill refills the accumulators, but the miss was passing as a plain rollup read with no signal, against the no-silent- cap rule. Add hasPendingAccumulatorBackfill to the read tier and key a new regression_source annotation off it (unavailable_pending_backfill vs rollup) on both warm paths. No live-SQL fallback is added; the design intentionally avoids the heavy scan.
This commit is contained in:
1 parent
745991a922
commit
21f6dcfe13
4 files changed
+133
No files matched your search
@@ -65,6 +65,7 @@ import { ensureUserRollupsFresh } from "@/lib/rollups/measurement-rollups";
|
||||
import {
|
||||
collapseRollupRowsBySource,
|
||||
composeWindowedRegression,
|
||||
hasPendingAccumulatorBackfill,
|
||||
loadUserSourcePriority,
|
||||
type AccumulatorBucketRow,
|
||||
} from "@/lib/rollups/measurement-read";
|
||||
@@ -603,6 +604,12 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
|
||||
let totalRows = 0;
|
||||
let typeCount = 0;
|
||||
const typesWithData: string[] = [];
|
||||
// v1.20.0 P3 M-1 — set once any in-window bucket carries a NULL
|
||||
// accumulator (pre-0190 row, or boot re-fold pending). The slope stays
|
||||
// null then — honest, it converges after backfill — but the miss must be
|
||||
// observable per the "no silent cap" rule, so it surfaces in the
|
||||
// annotation below rather than passing silently as a plain rollup read.
|
||||
let regressionPendingBackfill = false;
|
||||
for (const row of dayBuckets) {
|
||||
if (!row.count || row.count <= 0) continue;
|
||||
typeCount += 1;
|
||||
@@ -616,6 +623,12 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
|
||||
// readings) yields a null slope, the same shape the empty REGR_* FILTER
|
||||
// returned.
|
||||
const accBuckets = accBucketsByType.get(row.type) ?? [];
|
||||
if (
|
||||
!regressionPendingBackfill &&
|
||||
hasPendingAccumulatorBackfill(accBuckets, since90)
|
||||
) {
|
||||
regressionPendingBackfill = true;
|
||||
}
|
||||
const reg7 = composeWindowedRegression(accBuckets, since7);
|
||||
const reg30 = composeWindowedRegression(accBuckets, since30);
|
||||
const reg90 = composeWindowedRegression(accBuckets, since90);
|
||||
@@ -667,6 +680,14 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
|
||||
// per the "no silent cap" rule so a regression is observable.
|
||||
slope_source: "rollup",
|
||||
median_source: "live",
|
||||
// v1.20.0 P3 M-1 — when a NULL accumulator (pre-0190 row, or boot
|
||||
// re-fold pending) forced a slope miss, the read intentionally
|
||||
// returns null without a live-SQL fallback (the design avoids the
|
||||
// heavy scan). Surface the gap so the silent null is observable;
|
||||
// it clears once the boot backfill refills the accumulators.
|
||||
regression_source: regressionPendingBackfill
|
||||
? "unavailable_pending_backfill"
|
||||
: "rollup",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -109,6 +109,7 @@ import {
|
||||
aggregateBuckets,
|
||||
collapseRollupRowsBySource,
|
||||
composeWindowedRegression,
|
||||
hasPendingAccumulatorBackfill,
|
||||
loadUserSourcePriority,
|
||||
type AccumulatorBucketRow,
|
||||
} from "@/lib/rollups/measurement-read";
|
||||
@@ -519,6 +520,12 @@ async function buildFromRollups(
|
||||
const since7 = startOfUtcDay(new Date(now - 7 * 24 * 60 * 60 * 1000));
|
||||
const since30 = startOfUtcDay(new Date(now - 30 * 24 * 60 * 60 * 1000));
|
||||
const since90 = startOfUtcDay(new Date(now - 90 * 24 * 60 * 60 * 1000));
|
||||
// v1.20.0 P3 M-1 — set once any in-window bucket carries a NULL
|
||||
// accumulator (pre-0190 row, or boot re-fold pending). The slope stays
|
||||
// null then — it converges after backfill — but the miss must be
|
||||
// observable per the "no silent cap" rule, so it surfaces in the
|
||||
// annotation below rather than passing silently as a plain rollup read.
|
||||
let regressionPendingBackfill = false;
|
||||
|
||||
// Seed every type that has buckets — the bucket set is the source of
|
||||
// truth on this path. The narrow aggregate provides the non-composable
|
||||
@@ -541,6 +548,12 @@ async function buildFromRollups(
|
||||
sumXx: b.sumXx,
|
||||
sumYy: b.sumYy,
|
||||
}));
|
||||
if (
|
||||
!regressionPendingBackfill &&
|
||||
hasPendingAccumulatorBackfill(accBuckets, since90)
|
||||
) {
|
||||
regressionPendingBackfill = true;
|
||||
}
|
||||
const reg7 = composeWindowedRegression(accBuckets, since7);
|
||||
const reg30 = composeWindowedRegression(accBuckets, since30);
|
||||
const reg90 = composeWindowedRegression(accBuckets, since90);
|
||||
@@ -582,6 +595,14 @@ async function buildFromRollups(
|
||||
insights: {
|
||||
slope_source: "rollup",
|
||||
anomaly_source: "live",
|
||||
// v1.20.0 P3 M-1 — when a NULL accumulator (pre-0190 row, or boot
|
||||
// re-fold pending) forced a slope miss, the read intentionally
|
||||
// returns null without a live-SQL fallback (the design avoids the
|
||||
// heavy scan). Surface the gap so the silent null is observable; it
|
||||
// clears once the boot backfill refills the accumulators.
|
||||
regression_source: regressionPendingBackfill
|
||||
? "unavailable_pending_backfill"
|
||||
: "rollup",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -28,6 +28,8 @@ import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
composeRegression,
|
||||
hasPendingAccumulatorBackfill,
|
||||
type AccumulatorBucketRow,
|
||||
type RegressionAccumulators,
|
||||
} from "../measurement-read";
|
||||
|
||||
@@ -202,3 +204,55 @@ describe("composeRegression — closed-form parity", () => {
|
||||
expect(composed.sdPop!).toBeCloseTo(1, 12); // values 90, 92 → pop sd = 1
|
||||
});
|
||||
});
|
||||
|
||||
describe("hasPendingAccumulatorBackfill — observable miss signal", () => {
|
||||
const day = (iso: string): Date => new Date(iso);
|
||||
const full = (bucketStart: Date): AccumulatorBucketRow => ({
|
||||
bucketStart,
|
||||
count: 2,
|
||||
mean: 90,
|
||||
sumX: 1,
|
||||
sumXy: 2,
|
||||
sumXx: 3,
|
||||
sumYy: 4,
|
||||
});
|
||||
const nullAcc = (bucketStart: Date): AccumulatorBucketRow => ({
|
||||
bucketStart,
|
||||
count: 2,
|
||||
mean: 88,
|
||||
sumX: null,
|
||||
sumXy: null,
|
||||
sumXx: null,
|
||||
sumYy: null,
|
||||
});
|
||||
const since = day("2026-03-01T00:00:00.000Z");
|
||||
|
||||
it("is true when an in-window bucket carries a NULL accumulator", () => {
|
||||
const rows = [
|
||||
full(day("2026-03-02T00:00:00.000Z")),
|
||||
nullAcc(day("2026-03-10T00:00:00.000Z")),
|
||||
];
|
||||
expect(hasPendingAccumulatorBackfill(rows, since)).toBe(true);
|
||||
});
|
||||
|
||||
it("is false when every in-window bucket has full accumulators", () => {
|
||||
const rows = [
|
||||
full(day("2026-03-02T00:00:00.000Z")),
|
||||
full(day("2026-03-10T00:00:00.000Z")),
|
||||
];
|
||||
expect(hasPendingAccumulatorBackfill(rows, since)).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores a NULL accumulator that falls before the window cutoff", () => {
|
||||
const rows = [
|
||||
// Pre-window NULL row — outside the regression window, must not flag.
|
||||
nullAcc(day("2026-02-20T00:00:00.000Z")),
|
||||
full(day("2026-03-10T00:00:00.000Z")),
|
||||
];
|
||||
expect(hasPendingAccumulatorBackfill(rows, since)).toBe(false);
|
||||
});
|
||||
|
||||
it("is false for an empty bucket set", () => {
|
||||
expect(hasPendingAccumulatorBackfill([], since)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -276,6 +276,43 @@ export function composeWindowedRegression(
|
||||
return composeRegression(inWindow);
|
||||
}
|
||||
|
||||
/**
|
||||
* v1.20.0 P3 M-1 — true iff any in-window DAY bucket carries a NULL
|
||||
* regression accumulator (a row that predates migration 0190, or whose
|
||||
* boot re-fold has not yet refilled it). `composeWindowedRegression`
|
||||
* collapses both that case AND the legitimate "< 2 readings" / degenerate
|
||||
* cases into the same `{ null, null, null }` miss, so the reader cannot
|
||||
* tell from the composed result alone whether a null slope is the honest
|
||||
* answer or a coverage gap pending backfill.
|
||||
*
|
||||
* The slim / comprehensive readers call this alongside the compose so they
|
||||
* can annotate the miss (`regression_source:"unavailable_pending_backfill"`)
|
||||
* per the project's "no silent cap / log any truncation" rule. The null
|
||||
* value itself stays — it converges once the boot backfill refills the
|
||||
* accumulators — but the MISS becomes observable rather than silent.
|
||||
*
|
||||
* Window contract matches `composeWindowedRegression`: a bucket is
|
||||
* in-window iff its `bucketStart` is `>= since`.
|
||||
*/
|
||||
export function hasPendingAccumulatorBackfill(
|
||||
rows: ReadonlyArray<AccumulatorBucketRow>,
|
||||
since: Date,
|
||||
): boolean {
|
||||
const cutoff = since.getTime();
|
||||
for (const r of rows) {
|
||||
if (r.bucketStart.getTime() < cutoff) continue;
|
||||
if (
|
||||
r.sumX === null ||
|
||||
r.sumXy === null ||
|
||||
r.sumXx === null ||
|
||||
r.sumYy === null
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Combine DAY buckets into the linearly-composable window stats —
|
||||
* `count`, `min`, `max`, `mean`. SD / slope / R² are intentionally
|
||||
|
||||
Reference in new issue
Block a user