Merge pull request #825 from MBombeck/verify/coach-grounding

Grade the Coach turn that went looking and found nothing
This commit is contained in:
Marc-André Bombeck authored and GitHub committed 2026-08-22 16:38:12 +02:00
commit fefda545ae
18 files changed
+1030 -52

No files matched your search

@@ -0,0 +1,125 @@
/**
* CHARACTERISATION — which tool results reach `toolResults`, and why it matters.
*
* `toolResults` is not just a trace. The chat route derives
* `activatingPayloads` from it, and the numeric grounding guard runs ONLY when
* that array is non-empty (`insights/chat/route.ts`, the
* `activatingPayloads.length > 0` gate). So whatever the loop drops here, the
* guard never sees.
*
* Two shapes of empty read, from `availability.ts` `resolveEmptyRead`:
* - nothing recorded at all -> `{ present: false, reason: "none" }`, no `available`
* - rows exist out of window -> `{ present: false, reason: "unavailable_in_scope", available }`
*
* Only the second survives `loop.ts`'s push condition. The first is dropped,
* which disarms the grounding guard for that turn. This test pins the drop so
* the consequence is visible at the seam where it happens.
*/
import { describe, expect, it, vi, beforeEach } from "vitest";
import type { CoachToolResult } from "@/lib/ai/coach/tools/executor";
const executeCoachTool = vi.fn<(args?: unknown) => Promise<CoachToolResult>>();
vi.mock("@/lib/ai/coach/tools/executor", () => ({
executeCoachTool: (args: unknown) => executeCoachTool(args),
}));
const runRawCompletionWithFallback = vi.fn();
vi.mock("@/lib/ai/provider-runner", () => ({
runRawCompletionWithFallback: (args: unknown) =>
runRawCompletionWithFallback(args),
}));
import { runCoachToolLoop } from "@/lib/ai/coach/tools/loop";
import { COACH_TOOL_DEFS } from "@/lib/ai/coach/tools/definitions";
const baseArgs = {
userId: "u1",
providers: [],
system: "sys",
messages: [{ role: "user" as const, content: "how did I sleep?" }],
tools: COACH_TOOL_DEFS,
};
describe("coach tool loop — which results reach the grounding guard", () => {
beforeEach(() => {
executeCoachTool.mockReset();
runRawCompletionWithFallback.mockReset();
});
it("a pure miss (present:false, no available) is dropped from toolResults", async () => {
executeCoachTool.mockResolvedValue({ present: false, reason: "none" });
runRawCompletionWithFallback
.mockResolvedValueOnce({
result: {
content: "",
tokensUsed: 30,
model: "mock",
providerType: "anthropic" as const,
toolCalls: [
{ id: "t1", name: "get_sleep", arguments: JSON.stringify({}) },
],
finishReason: "tool_calls",
},
workingProvider: { providerType: "anthropic" },
fallbackHops: [],
})
.mockResolvedValueOnce({
result: {
content: "You averaged 7 h 12 min of sleep.",
tokensUsed: 20,
model: "mock",
providerType: "anthropic" as const,
finishReason: "stop",
},
workingProvider: { providerType: "anthropic" },
fallbackHops: [],
});
const loop = await runCoachToolLoop(baseArgs);
// The trace records that the tool ran and found nothing…
expect(loop.toolTrace).toEqual([{ name: "get_sleep", present: false }]);
// …but the payload set the route grades against is empty, so the route's
// `activatingPayloads.length > 0` gate is false and the numeric grounding
// guard never runs for this turn.
expect(loop.toolResults).toEqual([]);
});
it("an out-of-window miss WITH `available` is kept, so the guard stays armed", async () => {
executeCoachTool.mockResolvedValue({
present: false,
reason: "unavailable_in_scope",
available: { count: 12, reachableWithWindow: "lastYear" },
} as unknown as CoachToolResult);
runRawCompletionWithFallback
.mockResolvedValueOnce({
result: {
content: "",
tokensUsed: 30,
model: "mock",
providerType: "anthropic" as const,
toolCalls: [
{ id: "t1", name: "get_sleep", arguments: JSON.stringify({}) },
],
finishReason: "tool_calls",
},
workingProvider: { providerType: "anthropic" },
fallbackHops: [],
})
.mockResolvedValueOnce({
result: {
content: "Nothing in the last 30 days.",
tokensUsed: 20,
model: "mock",
providerType: "anthropic" as const,
finishReason: "stop",
},
workingProvider: { providerType: "anthropic" },
fallbackHops: [],
});
const loop = await runCoachToolLoop(baseArgs);
expect(loop.toolResults).toHaveLength(1);
});
});
+15 -3
View File
@@ -739,11 +739,23 @@ export function findUnverifiedCoachNumbersInLedger(
prose: string,
ledger: ReadonlyArray<LedgerEntry>,
locale: Locale = "en",
opts: { gradeAgainstEmptyLedger?: boolean } = {},
): UnverifiedCoachNumber[] {
if (!prose) return [];
// No authoritative figures (a qualitative turn / no-tools path) — nothing to
// grade against. The prompt-level grounding rule remains the backstop.
if (ledger.length === 0) return [];
// An empty ledger is ambiguous on its own, so the caller resolves it.
//
// Default (`false`): no authoritative figures were delivered — a qualitative
// turn, or the no-tools path on a cheap follow-up. Nothing to grade against,
// and the prompt-level grounding rule remains the backstop.
//
// `gradeAgainstEmptyLedger: true`: the caller KNOWS this turn went looking
// and came back with nothing — every retrieval tool it called reported
// `{ present: false }`. Then an empty ledger is not "no information", it is
// the positive statement that the record holds nothing to cite, so every
// magnitude in the reply is unreconciled by construction and must be graded
// as such. Only the chat route passes this; the fenced-chat and eval callers
// keep the default.
if (ledger.length === 0 && !opts.gradeAgainstEmptyLedger) return [];
const spans = sentenceSpans(prose);
const sentenceAt = (index: number): string => {
+111 -7
View File
@@ -16,6 +16,14 @@
* directly, so the answer arrives with the data instead of depending on the
* reader working it out.
*
* The same pass reconciles each block's coarse (MONTH / YEAR) timeline, served
* from the rollup tier, against the all-time extremes read live beside it. The
* read-swap falls back to live SQL only when a band is EMPTY, so a bucket whose
* rows were hard-deleted with no invalidation keeps being served as current —
* and the block then states two different things about the same record with
* nothing saying which is right. A bucket mean cannot lie outside its own rows'
* extremes, so when it does the band is dropped rather than narrated.
*
* Pure and in-place — the snapshot is a plain record by the time it gets here.
*/
import {
@@ -35,6 +43,12 @@ export interface SnapshotAsOf {
* the day they were taken on, and must be stated with that day.
*/
currentForTodayClaims: boolean;
/**
* Set when the block's coarse (MONTH / YEAR) timeline was dropped because it
* could not be reconciled with the live record beside it. Present only in
* that case, so its absence is the normal state and costs no prompt budget.
*/
coarseHistoryWithheld?: true;
}
function readNewestDaysAgo(block: unknown): number | null {
@@ -61,27 +75,117 @@ export function asOfFromDaysAgo(daysAgo: number): SnapshotAsOf {
};
}
/** Coarse-band bucket rows are `[bucketStart, mean, min, max]`. */
type CoarseBucket = [string, number, number, number];
/** Rounding headroom, so a float artefact is never read as a contradiction. */
const RECONCILE_EPSILON = 1e-6;
function readAllTimeExtremes(
block: unknown,
): { min: number; max: number } | null {
if (typeof block !== "object" || block === null) return null;
const aggregate = (block as { aggregate?: unknown }).aggregate;
if (typeof aggregate !== "object" || aggregate === null) return null;
const min = (aggregate as { allTimeMin?: unknown }).allTimeMin;
const max = (aggregate as { allTimeMax?: unknown }).allTimeMax;
if (typeof min !== "number" || !Number.isFinite(min)) return null;
if (typeof max !== "number" || !Number.isFinite(max)) return null;
return { min, max };
}
function readCoarse(block: unknown): Record<string, unknown> | null {
if (typeof block !== "object" || block === null) return null;
const timeline = (block as { timeline?: unknown }).timeline;
if (typeof timeline !== "object" || timeline === null) return null;
const coarse = (timeline as { coarse?: unknown }).coarse;
if (typeof coarse !== "object" || coarse === null) return null;
return coarse as Record<string, unknown>;
}
function bucketsOf(coarse: Record<string, unknown>, band: string): number[] {
const rows = coarse[band];
if (!Array.isArray(rows)) return [];
return (rows as CoarseBucket[])
.map((row) => (Array.isArray(row) ? row[1] : Number.NaN))
.filter((v): v is number => typeof v === "number" && Number.isFinite(v));
}
/**
* Decide whether a block's coarse timeline can be reconciled with the live
* record it sits beside.
*
* The coarse MONTH / YEAR bands are served from the rollup tier; the aggregate's
* all-time extremes are read live. A bucket mean is an average of real rows, so
* it MUST lie within the all-time min/max of those same rows. When it does not,
* the rollup is describing readings the live read cannot see — the bucket
* outlived the rows it was folded from.
*
* This is an invariant, not a heuristic: no arrangement of existing rows can
* average to a value outside their own extremes.
*/
function coarseContradictsRecord(block: unknown): boolean {
const extremes = readAllTimeExtremes(block);
if (extremes === null) return false;
const coarse = readCoarse(block);
if (coarse === null) return false;
const means = [
...bucketsOf(coarse, "monthly"),
...bucketsOf(coarse, "yearly"),
];
if (means.length === 0) return false;
return means.some(
(mean) =>
mean < extremes.min - RECONCILE_EPSILON ||
mean > extremes.max + RECONCILE_EPSILON,
);
}
/**
* Attach `asOf` to every snapshot block whose aggregate reports a freshest
* reading. Blocks with no coverage (narrative memory, plans, the reference
* grounding table) are untouched — they carry no measurement to date.
*
* Returns the metric keys that were stamped stale, so the caller can annotate
* them: a briefing narrated off a week-old series is worth seeing in the wide
* event, not just in the reader's confusion.
* Also drops a coarse timeline that contradicts the live record beside it. The
* rollup tier's read-swap falls back to live SQL only when a band is EMPTY, so
* a bucket that is merely WRONG — the rows behind it deleted, and no
* invalidation fired — is served as though it were current. The block then
* carried two irreconcilable accounts of the same record (a coarse tail saying
* one thing, all-time extremes saying another) with nothing marking which to
* believe, under an `asOf` stamped from raw reading age that knows nothing
* about rollup age. Narrating a ghost is worse than narrating less, so the
* disputed tail goes and `coarseHistoryWithheld` says it went.
*
* `currentForTodayClaims` stays keyed to raw recency, which is what it means:
* the freshest READING is still as fresh as it was, and suppressing it would
* silence a true present-tense statement to punish a stale history band.
*
* Returns the metric keys stamped stale and the keys whose coarse tail was
* withheld, so the caller can annotate both: a briefing narrated off a week-old
* series is worth seeing in the wide event, and so is a rollup that has drifted
* away from the rows underneath it.
*/
export function annotateSnapshotFreshness(
snapshot: Record<string, unknown>,
): string[] {
export function annotateSnapshotFreshness(snapshot: Record<string, unknown>): {
stale: string[];
coarseWithheld: string[];
} {
const stale: string[] = [];
const coarseWithheld: string[] = [];
for (const [key, block] of Object.entries(snapshot)) {
const daysAgo = readNewestDaysAgo(block);
if (daysAgo === null) continue;
const asOf = asOfFromDaysAgo(daysAgo);
if (coarseContradictsRecord(block)) {
const timeline = (block as { timeline?: Record<string, unknown> })
.timeline;
if (timeline) delete timeline.coarse;
asOf.coarseHistoryWithheld = true;
coarseWithheld.push(key);
}
(block as Record<string, unknown>).asOf = asOf;
if (!asOf.currentForTodayClaims) stale.push(key);
}
return stale;
return { stale, coarseWithheld };
}
export { TODAY_CLAIM_MAX_AGE_DAYS };
+12 -3
View File
@@ -1238,11 +1238,20 @@ async function buildCoachSnapshotImpl(
// hero line said "today" about a metric last measured five days earlier. The
// stamp sits on the block itself so the age travels with the numbers into
// every surface the snapshot feeds — hero, briefing, Coach reply, tools.
const staleBlocks = annotateSnapshotFreshness(compactSnapshot);
if (staleBlocks.length > 0) {
const freshness = annotateSnapshotFreshness(compactSnapshot);
if (freshness.stale.length > 0) {
annotate({
action: { name: "coach.snapshot.stale_blocks" },
meta: { blocks: staleBlocks.sort() },
meta: { blocks: freshness.stale.sort() },
});
}
// A rollup band that no longer reconciles with the rows underneath it is an
// operator-visible defect, not just a narration problem — the bucket is wrong
// on disk and stays wrong until something recomputes it.
if (freshness.coarseWithheld.length > 0) {
annotate({
action: { name: "coach.snapshot.coarse_withheld" },
meta: { blocks: freshness.coarseWithheld.sort() },
});
}
+7 -2
View File
@@ -68,14 +68,19 @@ export function getPersonalizedPulseTarget(
age: number | null,
gender: PulseGender,
): PersonalizedPulseTarget {
// AHA fallback for adults when profile context is missing.
// No age to personalise against (missing profile, or under 20 where the
// percentile table does not apply), so this arm returns a fixed band that is
// the same for everyone. The source label says so in as many words: the
// surrounding UI describes the pulse band as age- and sex-based, and a reader
// who falls into this arm needs to know theirs is not.
if (age == null || age < 20) {
return {
greenMin: 60,
greenMax: 100,
orangeMin: 55,
orangeMax: 105,
source: "AHA (adults, resting pulse 60-100 bpm)",
source:
"AHA general adult range (resting pulse 60-100 bpm) — not personalised",
};
}
@@ -119,9 +119,11 @@ describe("the Coach snapshot stamp", () => {
scope: { sources: ["pulse", "weight"] },
};
const stale = annotateSnapshotFreshness(snapshot);
const { stale, coarseWithheld } = annotateSnapshotFreshness(snapshot);
expect(stale).toEqual(["pulse"]);
// Neither block carries a coarse band, so nothing is disputed.
expect(coarseWithheld).toEqual([]);
expect((snapshot.pulse as { asOf: unknown }).asOf).toEqual({
daysAgo: 5,
isToday: false,
@@ -135,4 +137,63 @@ describe("the Coach snapshot stamp", () => {
expect(snapshot.memory).toEqual({ headline: "steady month" });
expect(snapshot.scope).toEqual({ sources: ["pulse", "weight"] });
});
/**
* A coarse bucket mean is an average of real rows, so it cannot sit outside
* those rows' own all-time extremes. When it does, the rollup band is
* describing readings the live read cannot see and must not be narrated.
*/
it("drops a coarse band whose bucket mean the live extremes cannot account for", () => {
const snapshot: Record<string, unknown> = {
weight: {
aggregate: {
allTimeMin: 80,
allTimeMax: 80,
coverage: { count: 6, newestDaysAgo: 0 },
},
timeline: {
recent: [],
coarse: { monthly: [["2026-01-01", 95, 95, 95]], yearly: [] },
},
},
};
const { coarseWithheld } = annotateSnapshotFreshness(snapshot);
expect(coarseWithheld).toEqual(["weight"]);
const weight = snapshot.weight as {
timeline: { coarse?: unknown };
asOf: { coarseHistoryWithheld?: true; currentForTodayClaims: boolean };
};
expect(weight.timeline.coarse).toBeUndefined();
expect(weight.asOf.coarseHistoryWithheld).toBe(true);
// Raw recency is untouched — the surviving readings are still from today.
expect(weight.asOf.currentForTodayClaims).toBe(true);
});
it("keeps a coarse band that reconciles with the live extremes", () => {
const snapshot: Record<string, unknown> = {
weight: {
aggregate: {
allTimeMin: 78,
allTimeMax: 96,
coverage: { count: 40, newestDaysAgo: 0 },
},
timeline: {
recent: [],
coarse: { monthly: [["2026-01-01", 95, 94, 96]], yearly: [] },
},
},
};
const { coarseWithheld } = annotateSnapshotFreshness(snapshot);
expect(coarseWithheld).toEqual([]);
const weight = snapshot.weight as {
timeline: { coarse?: unknown };
asOf: { coarseHistoryWithheld?: true };
};
expect(weight.timeline.coarse).toBeDefined();
expect(weight.asOf.coarseHistoryWithheld).toBeUndefined();
});
});