feat(coach): typed grounding tokenizer + normal-form reconciler + narrowed risk/dose banks

Replace the untyped magnitude bag in the Coach numeric verifier with a typed
tokenizer and a normal-form reconciler, and narrow the outbound risk and dose
banks so correct figures and correct refusals stop being blocked while every
fabrication shape still blocks.

Grounding verifier (coach-prose-grounding.ts):
- Classify each prose number before grading — ISO/written dates, clock times,
  bare years, ordinals, list markers, hyphenated time spans, ranges, percents,
  thousands-separated integers, unit-suffixed measurements. Benign types are
  never graded or stripped; ranges decompose into per-endpoint magnitudes.
- Grade against typed authoritative entries (value + kind + aggregation).
  Sign-insensitive, canonical rounding, ratio to percent in percent space, and
  minutes to hours widen the match only for an aggregate, kinded entry; kind
  scoping stops a weight value grounding a blood-pressure claim.
- Thousands/decimal separators follow the reply language.
- Boundary-safe strip so a flagged token never clips a larger grounded number.

Outbound screen (outbound-screen.ts):
- Risk bank requires an asserted figure or a categorical engine result; a bare
  engine or horizon mention (education/refusal) passes. Spelled-out percents
  count as figures. No refusal-context exemption. EN + DE.
- Dose bank keeps a maintenance-continuation restatement, voided by any change
  stem. Taper suffix fixed.

Route + fenced-chat thread the reply locale into the verifier.
This commit is contained in:
Marc-André Bombeck committed 2026-07-23 20:07:03 +02:00
1 parent eed66ae350
commit a69d0493f8
7 files changed
+1314 -166

No files matched your search

@@ -0,0 +1,333 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
/**
* v1.32.7 (Coach Guard I / G1) — route-level SSE assertions on the ASSEMBLED
* reply, the class the pure-module tests cannot see (the #591 lesson: the unit
* suite was green while the route regressed, because the guard runs inside
* `route.ts`). Unlike `route-tool-mode.test.ts`, this file uses the REAL
* outbound guard and the REAL numeric verifier — only the provider + IO are
* stubbed — so it exercises the guard exactly as production does, on both a
* tool turn and a no-tool turn.
*
* It asserts on the persisted assistant `content`, which is the fully-guarded
* text streamed to the client (every guard runs before the first token frame).
*/
const SNAPSHOT_JSON = '{"bp":{"aggregate":{"mean":128}}}';
vi.mock("@/lib/api-handler", () => ({
apiHandler: <T extends (...args: unknown[]) => unknown>(fn: T) => fn,
requireAuth: vi.fn(async () => ({ user: { id: "u1", locale: "en" } })),
HttpError: class HttpError extends Error {
status: number;
constructor(status: number, message: string) {
super(message);
this.status = status;
}
},
}));
vi.mock("@/lib/modules/gate", () => ({
requireModuleEnabled: vi.fn(async () => ({ enabled: true })),
isModuleEnabled: vi.fn(async () => true),
}));
vi.mock("@/lib/feature-flags", () => ({
requireAssistantSurface: vi.fn(async () => undefined),
}));
vi.mock("@/lib/api-response", () => ({
apiError: (error: string, status: number) => ({ data: null, error, status }),
apiSuccess: (data: unknown) => ({ data, error: null, status: 200 }),
}));
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
vi.mock("@/lib/db", () => ({
prisma: {
user: { findUnique: vi.fn(async () => ({ coachPrefsJson: null })) },
coachConversation: { findFirst: vi.fn(async () => ({ id: "c1" })) },
workout: { findFirst: vi.fn(async () => null) },
},
}));
const { checkRateLimit } = vi.hoisted(() => ({
checkRateLimit: vi.fn(async () => ({ allowed: true })),
}));
vi.mock("@/lib/rate-limit", () => ({ checkRateLimit }));
vi.mock("@/lib/i18n/server-locale", () => ({
resolveServerLocale: vi.fn(async () => "en"),
}));
const { runStreamingRawCompletionWithFallback } = vi.hoisted(() => ({
runStreamingRawCompletionWithFallback: vi.fn(),
}));
vi.mock("@/lib/ai/provider-runner", () => ({
AllProvidersFailedError: class extends Error {},
runStreamingRawCompletionWithFallback,
}));
const { resolveProviderChain } = vi.hoisted(() => ({
resolveProviderChain: vi.fn(),
}));
vi.mock("@/lib/ai/provider", () => ({
resolveProviderChain,
resolveProvider: vi.fn(),
}));
const { assertConsentForChain } = vi.hoisted(() => ({
assertConsentForChain: vi.fn(async () => undefined),
}));
vi.mock("@/lib/ai/consent-guard", () => ({ assertConsentForChain }));
vi.mock("@/lib/ai/prompts/insight-generator", () => ({ PROMPT_VERSION: "x" }));
vi.mock("@/lib/ai/ai-budgets", () => ({
AI_BUDGETS: { coach: { maxTokens: 1500, temperature: 0.4 } },
}));
vi.mock("@/lib/ai/coach/types", async () => {
const actual = await vi.importActual<typeof import("@/lib/ai/coach/types")>(
"@/lib/ai/coach/types",
);
return actual;
});
vi.mock("@/lib/ai/coach/persistence", () => ({
appendMessage: vi.fn(async () => ({ id: "m1" })),
createConversation: vi.fn(async () => ({ id: "c1" })),
fetchConversationWithMessages: vi.fn(),
listConversations: vi.fn(),
}));
vi.mock("@/lib/ai/coach/coach-memory-shared", () => ({
enqueueCoachMemoryRefresh: vi.fn(),
}));
vi.mock("@/lib/ai/coach/facts", () => ({
storeDeterministicFacts: vi.fn(async () => undefined),
}));
const { reserveBudget, reconcileSpend } = vi.hoisted(() => ({
reserveBudget: vi.fn(async () => ({ allowed: true, reserved: 3000 })),
reconcileSpend: vi.fn(async () => undefined),
}));
vi.mock("@/lib/ai/coach/budget", () => ({
buildDateKey: vi.fn(() => "2026-06-21"),
reserveBudget,
reconcileSpend,
resolveDailyCap: vi.fn(() => 2_000_000),
}));
const { detectRefusal } = vi.hoisted(() => ({
detectRefusal: vi.fn(() => ({ refuse: false })),
}));
vi.mock("@/lib/ai/coach/refusal", () => ({ detectRefusal }));
// NB: the outbound guard + numeric verifier are REAL — that is the point.
vi.mock("@/lib/ai/coach/system-prompt", () => ({
getCoachSystemPrompt: vi.fn(() => "SYSTEM"),
}));
vi.mock("@/lib/ai/coach/about-me", () => ({
getSelfContextTextForUser: vi.fn(async () => null),
}));
vi.mock("@/lib/ai/coach/snapshot", () => ({
buildCoachSnapshot: vi.fn(async () => ({
snapshotJson: SNAPSHOT_JSON,
sections: { bloodPressure: { aggregate: { mean: 128 } } },
provenance: { windows: ["last30days"], metrics: ["bp"] },
referenceGrounding: "REFERENCE RANGES",
})),
}));
vi.mock("@/lib/workouts/hr-series", () => ({
buildWorkoutHrSeries: vi.fn(async () => null),
}));
vi.mock("@/lib/workouts/zones", () => ({
computeZones: vi.fn(() => null),
hrMaxFromAge: vi.fn(() => 185),
parseWhoopZoneDurations: vi.fn(() => null),
}));
vi.mock("@/lib/workouts/sport-context", () => ({
buildSportContext: vi.fn(async () => null),
}));
const { runCoachToolLoop } = vi.hoisted(() => ({
runCoachToolLoop: vi.fn(),
}));
vi.mock("@/lib/ai/coach/tools", () => ({
COACH_TOOL_DEFS: [{ name: "get_metric_series" }],
MAX_ROUNDS: 3,
buildCoachDataInventory: vi.fn(async () => ({
entries: [],
restMode: false,
cycleEnabled: false,
window: "last30days",
probeScope: { sources: ["bp"], window: "last30days" },
})),
renderDataInventory: vi.fn(() => "DATA INVENTORY"),
renderFocusHint: vi.fn(() => ""),
buildToolModeAddendum: vi.fn(() => "TOOL ADDENDUM"),
runCoachToolLoop,
}));
const { parseKeyValuesSentinel } = vi.hoisted(() => ({
parseKeyValuesSentinel: vi.fn(),
}));
vi.mock("@/lib/ai/coach/keyvalues", () => ({ parseKeyValuesSentinel }));
const { parseSuggestReminder } = vi.hoisted(() => ({
parseSuggestReminder: vi.fn(),
}));
vi.mock("@/lib/ai/coach/suggest-reminder", () => ({ parseSuggestReminder }));
vi.mock("@/lib/ai/coach/suggest-gate", () => ({ gateSuggestion: vi.fn() }));
vi.mock("@/lib/validations/coach-prefs", () => ({
parseCoachPrefs: vi.fn(() => ({ defaultWindow: undefined })),
DEFAULT_REMINDER_SUGGESTION_PREFS: {},
}));
const { appendMessage } = await import("@/lib/ai/coach/persistence");
const sse = vi.hoisted(() => ({ done: Promise.resolve() as Promise<unknown> }));
vi.mock("@/lib/sse/create-stream", () => ({
createSseStream: (
producer: (c: {
signal: { aborted: boolean };
enqueue: () => void;
}) => void | Promise<void>,
) => {
sse.done = Promise.resolve(
producer({ signal: { aborted: false }, enqueue: () => {} }),
);
return new ReadableStream();
},
}));
import { POST } from "../route";
import { COACH_OUTBOUND_RISK_BLOCK_EN } from "@/lib/ai/coach/outbound-guard";
const post = POST as unknown as (req: Request) => Promise<Response>;
function chatReq(body: Record<string, unknown>): Request {
return new Request("http://localhost/api/insights/chat", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
}
/** The reply the model "returned" flows through the sentinel parsers verbatim. */
function echo(content: string): void {
parseKeyValuesSentinel.mockReturnValue({
prose: content,
keyValues: [],
malformed: false,
malformedEntries: [],
});
parseSuggestReminder.mockReturnValue({ prose: content });
}
/** Drive a TOOL turn returning `content` with the given tool payloads. */
async function toolTurn(
content: string,
toolResults: Array<{ present: boolean; data: unknown }> = [],
): Promise<string> {
echo(content);
resolveProviderChain.mockResolvedValue([
{ providerType: "anthropic", instance: {} },
]);
runCoachToolLoop.mockResolvedValue({
result: { content, tokensUsed: 80, model: "m" },
workingProviderType: "anthropic",
totalTokens: 80,
rounds: 1,
toolTrace: [{ name: "get_metric_series", present: true }],
toolResults,
});
const res = await post(chatReq({ message: "How is my BP?" }));
await sse.done;
expect(res.status).toBe(200);
const calls = (appendMessage as ReturnType<typeof vi.fn>).mock.calls;
const assistant = calls.find(
(c) => (c[0] as { role: string }).role === "assistant",
);
return (assistant?.[0] as { content: string }).content;
}
/** Drive a NO-TOOL turn (a provider lacking tools) returning `content`. */
async function noToolTurn(content: string): Promise<string> {
echo(content);
resolveProviderChain.mockResolvedValue([
{ providerType: "anthropic", instance: {} },
{ providerType: "local", instance: { supportsTools: false } },
]);
runStreamingRawCompletionWithFallback.mockResolvedValue({
result: { content, tokensUsed: 42, model: "m" },
workingProvider: { providerType: "anthropic" },
});
const res = await post(chatReq({ message: "How is my BP?" }));
await sse.done;
expect(res.status).toBe(200);
const calls = (appendMessage as ReturnType<typeof vi.fn>).mock.calls;
const assistant = calls.find(
(c) => (c[0] as { role: string }).role === "assistant",
);
return (assistant?.[0] as { content: string }).content;
}
describe("coach chat — Guard I at the route level (G1)", () => {
beforeEach(() => {
vi.clearAllMocks();
reserveBudget.mockResolvedValue({ allowed: true, reserved: 3000 });
detectRefusal.mockReturnValue({ refuse: false });
checkRateLimit.mockResolvedValue({ allowed: true });
assertConsentForChain.mockResolvedValue(undefined);
});
it("replaces a fabricated risk assertion end-to-end (tool turn)", async () => {
const content =
"Based on SCORE2, your risk sits at about 14% given your numbers.";
const out = await toolTurn(content, [
{ present: true, data: { metric: "bp", aggregate: { avgSys30: 128 } } },
]);
expect(out).toBe(COACH_OUTBOUND_RISK_BLOCK_EN);
expect(out).not.toContain("14%");
});
it("replaces a fabricated risk assertion end-to-end (no-tool turn)", async () => {
const out = await noToolTurn(
"Your ten-year cardiovascular risk is roughly twelve percent.",
);
expect(out).toBe(COACH_OUTBOUND_RISK_BLOCK_EN);
expect(out).not.toContain("twelve percent");
});
it("streams a model-perfect refusal intact — it is not blocked or mangled (tool turn)", async () => {
const content =
"I can't calculate a 10-year cardiovascular risk for you — an ASCVD score is something your clinician computes with lab values I don't have.";
const out = await toolTurn(content, [
{ present: true, data: { metric: "bp", aggregate: { avgSys30: 128 } } },
]);
expect(out).toBe(content);
expect(out).not.toContain("[unverified]");
});
it("streams dates, ranges and thousands unmangled when they are grounded (tool turn)", async () => {
const content =
"On 2026-07-23 your systolic sat between 120 and 135 mmHg, with about 10,000 steps logged.";
const out = await toolTurn(content, [
{
present: true,
data: { metric: "bp", section: { aggregate: { min: 121, max: 134 } } },
},
{
present: true,
data: { metric: "steps", section: { aggregate: { latest: 10000 } } },
},
]);
expect(out).toBe(content);
expect(out).not.toContain("[unverified]");
});
it("soft-strips a genuinely fabricated figure on a tool turn (the floor still holds)", async () => {
const content = "Your systolic averaged about 138 recently.";
const out = await toolTurn(content, [
{
present: true,
data: { metric: "bp", section: { aggregate: { avgSys30: 128 } } },
},
]);
expect(out).toContain("[unverified]");
expect(out).not.toContain("138");
});
});
+1
View File
@@ -927,6 +927,7 @@ async function handleChatRequest(request: NextRequest): Promise<Response> {
const unverified = findUnverifiedCoachNumbers(
replyText,
authoritativePayloads,
locale,
);
if (unverified.length > 0) {
const { prose: corrected, stripped } = stripUnverifiedNumbers(