mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-11 13:52:20 +00:00
Stop narrating rollup buckets the record cannot account for
A rollup band outlives the rows it was folded from. The read-swap falls back to live SQL only when a band is EMPTY, so a bucket that is merely wrong is served as current; `ensureUserRollupsFresh` repairs the DAY tier over the trailing 90 days only, and keys off `Measurement.updatedAt`, which a hard delete does not bump. The WHOOP body sync hard-deletes a WEIGHT row with no invalidation, so a MONTH bucket keeps reporting a value the record no longer holds. The block was handing the model two irreconcilable accounts of the same record and marking neither: the coarse tail said one thing, the all-time extremes read live beside it said another, and `asOf` stamped the whole thing current because it is derived from raw reading age and knows nothing about rollup age. Reconcile them where they are assembled. A bucket mean is an average of real rows, so it cannot fall outside those rows' own all-time extremes — when it does, the band is describing readings the live read cannot see. That is an invariant rather than a heuristic, so the disputed tail is dropped and `asOf.coarseHistoryWithheld` records that it went. Narrating a ghost is worse than narrating less. `currentForTodayClaims` stays keyed to raw recency, which is what it means. The surviving readings really are from today, and suppressing a true present-tense statement to punish a stale history band would trade one dishonesty for another. The bucket on disk is still wrong, and a wide event now says so. The deeper fix is to make the tier notice — compare a bucket's `computedAt` against the type's in-window MAX(`updatedAt`), or invalidate on the hard-delete path — which is a larger change than this repair.
This commit is contained in:
1 parent
251a42bb0a
commit
c1811bc85b
4 files changed
+224
-43
No files matched your search
@@ -16,6 +16,14 @@
|
||||
* directly, so the answer arrives with the data instead of depending on the
|
||||
* reader working it out.
|
||||
*
|
||||
* The same pass reconciles each block's coarse (MONTH / YEAR) timeline, served
|
||||
* from the rollup tier, against the all-time extremes read live beside it. The
|
||||
* read-swap falls back to live SQL only when a band is EMPTY, so a bucket whose
|
||||
* rows were hard-deleted with no invalidation keeps being served as current —
|
||||
* and the block then states two different things about the same record with
|
||||
* nothing saying which is right. A bucket mean cannot lie outside its own rows'
|
||||
* extremes, so when it does the band is dropped rather than narrated.
|
||||
*
|
||||
* Pure and in-place — the snapshot is a plain record by the time it gets here.
|
||||
*/
|
||||
import {
|
||||
@@ -35,6 +43,12 @@ export interface SnapshotAsOf {
|
||||
* the day they were taken on, and must be stated with that day.
|
||||
*/
|
||||
currentForTodayClaims: boolean;
|
||||
/**
|
||||
* Set when the block's coarse (MONTH / YEAR) timeline was dropped because it
|
||||
* could not be reconciled with the live record beside it. Present only in
|
||||
* that case, so its absence is the normal state and costs no prompt budget.
|
||||
*/
|
||||
coarseHistoryWithheld?: true;
|
||||
}
|
||||
|
||||
function readNewestDaysAgo(block: unknown): number | null {
|
||||
@@ -61,27 +75,117 @@ export function asOfFromDaysAgo(daysAgo: number): SnapshotAsOf {
|
||||
};
|
||||
}
|
||||
|
||||
/** Coarse-band bucket rows are `[bucketStart, mean, min, max]`. */
|
||||
type CoarseBucket = [string, number, number, number];
|
||||
|
||||
/** Rounding headroom, so a float artefact is never read as a contradiction. */
|
||||
const RECONCILE_EPSILON = 1e-6;
|
||||
|
||||
function readAllTimeExtremes(
|
||||
block: unknown,
|
||||
): { min: number; max: number } | null {
|
||||
if (typeof block !== "object" || block === null) return null;
|
||||
const aggregate = (block as { aggregate?: unknown }).aggregate;
|
||||
if (typeof aggregate !== "object" || aggregate === null) return null;
|
||||
const min = (aggregate as { allTimeMin?: unknown }).allTimeMin;
|
||||
const max = (aggregate as { allTimeMax?: unknown }).allTimeMax;
|
||||
if (typeof min !== "number" || !Number.isFinite(min)) return null;
|
||||
if (typeof max !== "number" || !Number.isFinite(max)) return null;
|
||||
return { min, max };
|
||||
}
|
||||
|
||||
function readCoarse(block: unknown): Record<string, unknown> | null {
|
||||
if (typeof block !== "object" || block === null) return null;
|
||||
const timeline = (block as { timeline?: unknown }).timeline;
|
||||
if (typeof timeline !== "object" || timeline === null) return null;
|
||||
const coarse = (timeline as { coarse?: unknown }).coarse;
|
||||
if (typeof coarse !== "object" || coarse === null) return null;
|
||||
return coarse as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function bucketsOf(coarse: Record<string, unknown>, band: string): number[] {
|
||||
const rows = coarse[band];
|
||||
if (!Array.isArray(rows)) return [];
|
||||
return (rows as CoarseBucket[])
|
||||
.map((row) => (Array.isArray(row) ? row[1] : Number.NaN))
|
||||
.filter((v): v is number => typeof v === "number" && Number.isFinite(v));
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a block's coarse timeline can be reconciled with the live
|
||||
* record it sits beside.
|
||||
*
|
||||
* The coarse MONTH / YEAR bands are served from the rollup tier; the aggregate's
|
||||
* all-time extremes are read live. A bucket mean is an average of real rows, so
|
||||
* it MUST lie within the all-time min/max of those same rows. When it does not,
|
||||
* the rollup is describing readings the live read cannot see — the bucket
|
||||
* outlived the rows it was folded from.
|
||||
*
|
||||
* This is an invariant, not a heuristic: no arrangement of existing rows can
|
||||
* average to a value outside their own extremes.
|
||||
*/
|
||||
function coarseContradictsRecord(block: unknown): boolean {
|
||||
const extremes = readAllTimeExtremes(block);
|
||||
if (extremes === null) return false;
|
||||
const coarse = readCoarse(block);
|
||||
if (coarse === null) return false;
|
||||
const means = [
|
||||
...bucketsOf(coarse, "monthly"),
|
||||
...bucketsOf(coarse, "yearly"),
|
||||
];
|
||||
if (means.length === 0) return false;
|
||||
return means.some(
|
||||
(mean) =>
|
||||
mean < extremes.min - RECONCILE_EPSILON ||
|
||||
mean > extremes.max + RECONCILE_EPSILON,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach `asOf` to every snapshot block whose aggregate reports a freshest
|
||||
* reading. Blocks with no coverage (narrative memory, plans, the reference
|
||||
* grounding table) are untouched — they carry no measurement to date.
|
||||
*
|
||||
* Returns the metric keys that were stamped stale, so the caller can annotate
|
||||
* them: a briefing narrated off a week-old series is worth seeing in the wide
|
||||
* event, not just in the reader's confusion.
|
||||
* Also drops a coarse timeline that contradicts the live record beside it. The
|
||||
* rollup tier's read-swap falls back to live SQL only when a band is EMPTY, so
|
||||
* a bucket that is merely WRONG — the rows behind it deleted, and no
|
||||
* invalidation fired — is served as though it were current. The block then
|
||||
* carried two irreconcilable accounts of the same record (a coarse tail saying
|
||||
* one thing, all-time extremes saying another) with nothing marking which to
|
||||
* believe, under an `asOf` stamped from raw reading age that knows nothing
|
||||
* about rollup age. Narrating a ghost is worse than narrating less, so the
|
||||
* disputed tail goes and `coarseHistoryWithheld` says it went.
|
||||
*
|
||||
* `currentForTodayClaims` stays keyed to raw recency, which is what it means:
|
||||
* the freshest READING is still as fresh as it was, and suppressing it would
|
||||
* silence a true present-tense statement to punish a stale history band.
|
||||
*
|
||||
* Returns the metric keys stamped stale and the keys whose coarse tail was
|
||||
* withheld, so the caller can annotate both: a briefing narrated off a week-old
|
||||
* series is worth seeing in the wide event, and so is a rollup that has drifted
|
||||
* away from the rows underneath it.
|
||||
*/
|
||||
export function annotateSnapshotFreshness(
|
||||
snapshot: Record<string, unknown>,
|
||||
): string[] {
|
||||
export function annotateSnapshotFreshness(snapshot: Record<string, unknown>): {
|
||||
stale: string[];
|
||||
coarseWithheld: string[];
|
||||
} {
|
||||
const stale: string[] = [];
|
||||
const coarseWithheld: string[] = [];
|
||||
for (const [key, block] of Object.entries(snapshot)) {
|
||||
const daysAgo = readNewestDaysAgo(block);
|
||||
if (daysAgo === null) continue;
|
||||
const asOf = asOfFromDaysAgo(daysAgo);
|
||||
if (coarseContradictsRecord(block)) {
|
||||
const timeline = (block as { timeline?: Record<string, unknown> })
|
||||
.timeline;
|
||||
if (timeline) delete timeline.coarse;
|
||||
asOf.coarseHistoryWithheld = true;
|
||||
coarseWithheld.push(key);
|
||||
}
|
||||
(block as Record<string, unknown>).asOf = asOf;
|
||||
if (!asOf.currentForTodayClaims) stale.push(key);
|
||||
}
|
||||
return stale;
|
||||
return { stale, coarseWithheld };
|
||||
}
|
||||
|
||||
export { TODAY_CLAIM_MAX_AGE_DAYS };
|
||||
@@ -1238,11 +1238,20 @@ async function buildCoachSnapshotImpl(
|
||||
// hero line said "today" about a metric last measured five days earlier. The
|
||||
// stamp sits on the block itself so the age travels with the numbers into
|
||||
// every surface the snapshot feeds — hero, briefing, Coach reply, tools.
|
||||
const staleBlocks = annotateSnapshotFreshness(compactSnapshot);
|
||||
if (staleBlocks.length > 0) {
|
||||
const freshness = annotateSnapshotFreshness(compactSnapshot);
|
||||
if (freshness.stale.length > 0) {
|
||||
annotate({
|
||||
action: { name: "coach.snapshot.stale_blocks" },
|
||||
meta: { blocks: staleBlocks.sort() },
|
||||
meta: { blocks: freshness.stale.sort() },
|
||||
});
|
||||
}
|
||||
// A rollup band that no longer reconciles with the rows underneath it is an
|
||||
// operator-visible defect, not just a narration problem — the bucket is wrong
|
||||
// on disk and stays wrong until something recomputes it.
|
||||
if (freshness.coarseWithheld.length > 0) {
|
||||
annotate({
|
||||
action: { name: "coach.snapshot.coarse_withheld" },
|
||||
meta: { blocks: freshness.coarseWithheld.sort() },
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -119,9 +119,11 @@ describe("the Coach snapshot stamp", () => {
|
||||
scope: { sources: ["pulse", "weight"] },
|
||||
};
|
||||
|
||||
const stale = annotateSnapshotFreshness(snapshot);
|
||||
const { stale, coarseWithheld } = annotateSnapshotFreshness(snapshot);
|
||||
|
||||
expect(stale).toEqual(["pulse"]);
|
||||
// Neither block carries a coarse band, so nothing is disputed.
|
||||
expect(coarseWithheld).toEqual([]);
|
||||
expect((snapshot.pulse as { asOf: unknown }).asOf).toEqual({
|
||||
daysAgo: 5,
|
||||
isToday: false,
|
||||
@@ -135,4 +137,63 @@ describe("the Coach snapshot stamp", () => {
|
||||
expect(snapshot.memory).toEqual({ headline: "steady month" });
|
||||
expect(snapshot.scope).toEqual({ sources: ["pulse", "weight"] });
|
||||
});
|
||||
|
||||
/**
|
||||
* A coarse bucket mean is an average of real rows, so it cannot sit outside
|
||||
* those rows' own all-time extremes. When it does, the rollup band is
|
||||
* describing readings the live read cannot see and must not be narrated.
|
||||
*/
|
||||
it("drops a coarse band whose bucket mean the live extremes cannot account for", () => {
|
||||
const snapshot: Record<string, unknown> = {
|
||||
weight: {
|
||||
aggregate: {
|
||||
allTimeMin: 80,
|
||||
allTimeMax: 80,
|
||||
coverage: { count: 6, newestDaysAgo: 0 },
|
||||
},
|
||||
timeline: {
|
||||
recent: [],
|
||||
coarse: { monthly: [["2026-01-01", 95, 95, 95]], yearly: [] },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const { coarseWithheld } = annotateSnapshotFreshness(snapshot);
|
||||
|
||||
expect(coarseWithheld).toEqual(["weight"]);
|
||||
const weight = snapshot.weight as {
|
||||
timeline: { coarse?: unknown };
|
||||
asOf: { coarseHistoryWithheld?: true; currentForTodayClaims: boolean };
|
||||
};
|
||||
expect(weight.timeline.coarse).toBeUndefined();
|
||||
expect(weight.asOf.coarseHistoryWithheld).toBe(true);
|
||||
// Raw recency is untouched — the surviving readings are still from today.
|
||||
expect(weight.asOf.currentForTodayClaims).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps a coarse band that reconciles with the live extremes", () => {
|
||||
const snapshot: Record<string, unknown> = {
|
||||
weight: {
|
||||
aggregate: {
|
||||
allTimeMin: 78,
|
||||
allTimeMax: 96,
|
||||
coverage: { count: 40, newestDaysAgo: 0 },
|
||||
},
|
||||
timeline: {
|
||||
recent: [],
|
||||
coarse: { monthly: [["2026-01-01", 95, 94, 96]], yearly: [] },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const { coarseWithheld } = annotateSnapshotFreshness(snapshot);
|
||||
|
||||
expect(coarseWithheld).toEqual([]);
|
||||
const weight = snapshot.weight as {
|
||||
timeline: { coarse?: unknown };
|
||||
asOf: { coarseHistoryWithheld?: true };
|
||||
};
|
||||
expect(weight.timeline.coarse).toBeDefined();
|
||||
expect(weight.asOf.coarseHistoryWithheld).toBeUndefined();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user