observability(rollups): surface the pending-backfill regression miss

The windowed slope / r2 / sd readers compose from the per-bucket
regression accumulators (migration 0190). Between a deploy and the
boot-backfill completing, a tenant's pre-0190 DAY buckets still carry
NULL accumulators, so composeWindowedRegression returns a full miss and
the slim / comprehensive readers emit a null slope. The null is the
honest answer until the backfill refills the accumulators, but the miss
was passing as a plain rollup read with no signal, against the no-silent-
cap rule.

Add hasPendingAccumulatorBackfill to the read tier and key a new
regression_source annotation off it (unavailable_pending_backfill vs
rollup) on both warm paths. No live-SQL fallback is added; the design
intentionally avoids the heavy scan.
This commit is contained in:
Marc-André Bombeck committed 2026-06-22 08:07:58 +02:00
1 parent 745991a922
commit 21f6dcfe13
4 files changed
+133

No files matched your search

+21
View File
@@ -65,6 +65,7 @@ import { ensureUserRollupsFresh } from "@/lib/rollups/measurement-rollups";
import {
collapseRollupRowsBySource,
composeWindowedRegression,
hasPendingAccumulatorBackfill,
loadUserSourcePriority,
type AccumulatorBucketRow,
} from "@/lib/rollups/measurement-read";
@@ -603,6 +604,12 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
let totalRows = 0;
let typeCount = 0;
const typesWithData: string[] = [];
// v1.20.0 P3 M-1 — set once any in-window bucket carries a NULL
// accumulator (pre-0190 row, or boot re-fold pending). The slope stays
// null then — honest, it converges after backfill — but the miss must be
// observable per the "no silent cap" rule, so it surfaces in the
// annotation below rather than passing silently as a plain rollup read.
let regressionPendingBackfill = false;
for (const row of dayBuckets) {
if (!row.count || row.count <= 0) continue;
typeCount += 1;
@@ -616,6 +623,12 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
// readings) yields a null slope, the same shape the empty REGR_* FILTER
// returned.
const accBuckets = accBucketsByType.get(row.type) ?? [];
if (
!regressionPendingBackfill &&
hasPendingAccumulatorBackfill(accBuckets, since90)
) {
regressionPendingBackfill = true;
}
const reg7 = composeWindowedRegression(accBuckets, since7);
const reg30 = composeWindowedRegression(accBuckets, since30);
const reg90 = composeWindowedRegression(accBuckets, since90);
@@ -667,6 +680,14 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
// per the "no silent cap" rule so a regression is observable.
slope_source: "rollup",
median_source: "live",
// v1.20.0 P3 M-1 — when a NULL accumulator (pre-0190 row, or boot
// re-fold pending) forced a slope miss, the read intentionally
// returns null without a live-SQL fallback (the design avoids the
// heavy scan). Surface the gap so the silent null is observable;
// it clears once the boot backfill refills the accumulators.
regression_source: regressionPendingBackfill
? "unavailable_pending_backfill"
: "rollup",
},
},
},
@@ -109,6 +109,7 @@ import {
aggregateBuckets,
collapseRollupRowsBySource,
composeWindowedRegression,
hasPendingAccumulatorBackfill,
loadUserSourcePriority,
type AccumulatorBucketRow,
} from "@/lib/rollups/measurement-read";
@@ -519,6 +520,12 @@ async function buildFromRollups(
const since7 = startOfUtcDay(new Date(now - 7 * 24 * 60 * 60 * 1000));
const since30 = startOfUtcDay(new Date(now - 30 * 24 * 60 * 60 * 1000));
const since90 = startOfUtcDay(new Date(now - 90 * 24 * 60 * 60 * 1000));
// v1.20.0 P3 M-1 — set once any in-window bucket carries a NULL
// accumulator (pre-0190 row, or boot re-fold pending). The slope stays
// null then — it converges after backfill — but the miss must be
// observable per the "no silent cap" rule, so it surfaces in the
// annotation below rather than passing silently as a plain rollup read.
let regressionPendingBackfill = false;
// Seed every type that has buckets — the bucket set is the source of
// truth on this path. The narrow aggregate provides the non-composable
@@ -541,6 +548,12 @@ async function buildFromRollups(
sumXx: b.sumXx,
sumYy: b.sumYy,
}));
if (
!regressionPendingBackfill &&
hasPendingAccumulatorBackfill(accBuckets, since90)
) {
regressionPendingBackfill = true;
}
const reg7 = composeWindowedRegression(accBuckets, since7);
const reg30 = composeWindowedRegression(accBuckets, since30);
const reg90 = composeWindowedRegression(accBuckets, since90);
@@ -582,6 +595,14 @@ async function buildFromRollups(
insights: {
slope_source: "rollup",
anomaly_source: "live",
// v1.20.0 P3 M-1 — when a NULL accumulator (pre-0190 row, or boot
// re-fold pending) forced a slope miss, the read intentionally
// returns null without a live-SQL fallback (the design avoids the
// heavy scan). Surface the gap so the silent null is observable; it
// clears once the boot backfill refills the accumulators.
regression_source: regressionPendingBackfill
? "unavailable_pending_backfill"
: "rollup",
},
},
});
@@ -28,6 +28,8 @@ import { describe, expect, it } from "vitest";
import {
composeRegression,
hasPendingAccumulatorBackfill,
type AccumulatorBucketRow,
type RegressionAccumulators,
} from "../measurement-read";
@@ -202,3 +204,55 @@ describe("composeRegression — closed-form parity", () => {
expect(composed.sdPop!).toBeCloseTo(1, 12); // values 90, 92 → pop sd = 1
});
});
describe("hasPendingAccumulatorBackfill — observable miss signal", () => {
const day = (iso: string): Date => new Date(iso);
const full = (bucketStart: Date): AccumulatorBucketRow => ({
bucketStart,
count: 2,
mean: 90,
sumX: 1,
sumXy: 2,
sumXx: 3,
sumYy: 4,
});
const nullAcc = (bucketStart: Date): AccumulatorBucketRow => ({
bucketStart,
count: 2,
mean: 88,
sumX: null,
sumXy: null,
sumXx: null,
sumYy: null,
});
const since = day("2026-03-01T00:00:00.000Z");
it("is true when an in-window bucket carries a NULL accumulator", () => {
const rows = [
full(day("2026-03-02T00:00:00.000Z")),
nullAcc(day("2026-03-10T00:00:00.000Z")),
];
expect(hasPendingAccumulatorBackfill(rows, since)).toBe(true);
});
it("is false when every in-window bucket has full accumulators", () => {
const rows = [
full(day("2026-03-02T00:00:00.000Z")),
full(day("2026-03-10T00:00:00.000Z")),
];
expect(hasPendingAccumulatorBackfill(rows, since)).toBe(false);
});
it("ignores a NULL accumulator that falls before the window cutoff", () => {
const rows = [
// Pre-window NULL row — outside the regression window, must not flag.
nullAcc(day("2026-02-20T00:00:00.000Z")),
full(day("2026-03-10T00:00:00.000Z")),
];
expect(hasPendingAccumulatorBackfill(rows, since)).toBe(false);
});
it("is false for an empty bucket set", () => {
expect(hasPendingAccumulatorBackfill([], since)).toBe(false);
});
});
+37
View File
@@ -276,6 +276,43 @@ export function composeWindowedRegression(
return composeRegression(inWindow);
}
/**
* v1.20.0 P3 M-1 — true iff any in-window DAY bucket carries a NULL
* regression accumulator (a row that predates migration 0190, or whose
* boot re-fold has not yet refilled it). `composeWindowedRegression`
* collapses both that case AND the legitimate "< 2 readings" / degenerate
* cases into the same `{ null, null, null }` miss, so the reader cannot
* tell from the composed result alone whether a null slope is the honest
* answer or a coverage gap pending backfill.
*
* The slim / comprehensive readers call this alongside the compose so they
* can annotate the miss (`regression_source:"unavailable_pending_backfill"`)
* per the project's "no silent cap / log any truncation" rule. The null
* value itself stays — it converges once the boot backfill refills the
* accumulators — but the MISS becomes observable rather than silent.
*
* Window contract matches `composeWindowedRegression`: a bucket is
* in-window iff its `bucketStart` is `>= since`.
*/
export function hasPendingAccumulatorBackfill(
rows: ReadonlyArray<AccumulatorBucketRow>,
since: Date,
): boolean {
const cutoff = since.getTime();
for (const r of rows) {
if (r.bucketStart.getTime() < cutoff) continue;
if (
r.sumX === null ||
r.sumXy === null ||
r.sumXx === null ||
r.sumYy === null
) {
return true;
}
}
return false;
}
/**
* Combine DAY buckets into the linearly-composable window stats —
* `count`, `min`, `max`, `mean`. SD / slope / R² are intentionally