From ea2eaf2074aca53c58fb8866de1fbe6ab8d5f2e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 27 Sep 2026 11:44:54 +0200 Subject: [PATCH] feat(coach): stream a step for every read the turn makes Each tool call now emits a running step as it starts and its final status as it settles: found with a server-counted number of readings, empty with the reason, or failed. The no-tools path shows one step for the record summary. Steps carry catalog labels rendered in the request language, closed-enum domains and windows, and integers only; a lab analyte, a value or a model-written argument never reaches one. At most twelve per turn. They persist on the provenance, so a reload shows the same list. The conversation window reaches the step so a call without a window names the one it actually read. The no-tools transcripts gain the summary step; a new transcript pins found, empty and failed steps ahead of the first token. --- .../golden-transcripts.test.ts.snap | 384 ++++++++++++++ .../chat/__tests__/golden-transcripts.test.ts | 55 +- .../__tests__/route-capability-gate.test.ts | 4 +- .../chat/__tests__/route-list-query.test.ts | 4 +- src/lib/ai/coach/turn/__tests__/steps.test.ts | 501 ++++++++++++++++++ src/lib/ai/coach/turn/model.ts | 12 +- src/lib/ai/coach/turn/steps.ts | 270 +++++++++- 7 files changed, 1216 insertions(+), 14 deletions(-) create mode 100644 src/lib/ai/coach/turn/__tests__/steps.test.ts diff --git a/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap b/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap index 261cd55a1..c0575b2dc 100644 --- a/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap +++ b/src/app/api/insights/chat/__tests__/__snapshots__/golden-transcripts.test.ts.snap @@ -816,6 +816,17 @@ exports[`coach chat golden transcripts > 03 no-tools path with the full snapshot "metrics": [ "bp", ], + "steps": [ + { + "count": 1, + "domain": "snapshot", + "id": "s1", + "label": "Reading your record summary", + "labelKey": "coach.step.snapshot", + "status": "done", + "tool": "snapshot", + }, + ], "windows": [ "last30days", ], @@ -844,6 +855,18 @@ exports[`coach chat golden transcripts > 03 no-tools path with the full snapshot ], "contentType": "text/event-stream; charset=utf-8", "frames": [ + { + "step": { + "count": 1, + "domain": "snapshot", + "id": "s1", + "label": "Reading your record summary", + "labelKey": "coach.step.snapshot", + "status": "done", + "tool": "snapshot", + }, + "type": "step", + }, { "token": "Your ", "type": "token", @@ -889,6 +912,17 @@ exports[`coach chat golden transcripts > 03 no-tools path with the full snapshot "metrics": [ "bp", ], + "steps": [ + { + "count": 1, + "domain": "snapshot", + "id": "s1", + "label": "Reading your record summary", + "labelKey": "coach.step.snapshot", + "status": "done", + "tool": "snapshot", + }, + ], "windows": [ "last30days", ], @@ -4004,6 +4038,17 @@ exports[`coach chat golden transcripts > 16b an empty chain served by the legacy "metrics": [ "bp", ], + "steps": [ + { + "count": 1, + "domain": "snapshot", + "id": "s1", + "label": "Reading your record summary", + "labelKey": "coach.step.snapshot", + "status": "done", + "tool": "snapshot", + }, + ], "windows": [ "last30days", ], @@ -4032,6 +4077,18 @@ exports[`coach chat golden transcripts > 16b an empty chain served by the legacy ], "contentType": "text/event-stream; charset=utf-8", "frames": [ + { + "step": { + "count": 1, + "domain": "snapshot", + "id": "s1", + "label": "Reading your record summary", + "labelKey": "coach.step.snapshot", + "status": "done", + "tool": "snapshot", + }, + "type": "step", + }, { "token": "Your ", "type": "token", @@ -4077,6 +4134,17 @@ exports[`coach chat golden transcripts > 16b an empty chain served by the legacy "metrics": [ "bp", ], + "steps": [ + { + "count": 1, + "domain": "snapshot", + "id": "s1", + "label": "Reading your record summary", + "labelKey": "coach.step.snapshot", + "status": "done", + "tool": "snapshot", + }, + ], "windows": [ "last30days", ], @@ -5259,6 +5327,322 @@ exports[`coach chat golden transcripts > 21 the German locale with a stored self } `; +exports[`coach chat golden transcripts > 22 tool mode with live steps: found, empty and failed, all before the first token 1`] = ` +{ + "body": null, + "calls": [ + { + "call": "createConversation", + }, + { + "call": "appendMessage", + "content": "How is my blood pressure?", + "metricSource": null, + "model": null, + "providerType": null, + "role": "user", + "tokensUsed": null, + }, + { + "call": "storeDeterministicFacts", + }, + { + "call": "assertConsentForChain", + "chain": [ + "anthropic", + ], + }, + { + "amount": 1800, + "call": "reserveBudget", + "cap": 2000000, + "dateKey": "2026-09-26", + "owner": "user", + "surface": "coach", + }, + { + "call": "runCoachToolLoop", + "fallbackWindow": "allTime", + "maxTokens": 600, + "messages": [ + "user:0c52b5bc28a6:239", + ], + "providers": [ + "anthropic", + ], + "sharedScope": { + "sources": [ + "bp", + ], + "window": "last30days", + }, + "signal": true, + "system": "30126acc9ad2:1419", + "temperature": 0.4, + "timeoutMs": 180000, + "tools": 2, + }, + { + "actual": 80, + "attribution": { + "reservedOwner": "user", + "servedBy": "anthropic", + }, + "cached": 0, + "call": "reconcileSpend", + "dateKey": "2026-09-26", + "reserved": 1800, + }, + { + "call": "appendMessage", + "content": "Your systolic averaged 128 mmHg.", + "metricSource": { + "groundedFigures": [ + 128, + 82, + 46, + 105, + 35.9375, + 56.09756097560976, + 42, + ], + "metrics": [ + "bp", + ], + "steps": [ + { + "count": 142, + "domain": "bp", + "id": "s1", + "label": "Reading Blood pressure, last 90 days", + "labelKey": "coach.step.readWindow", + "status": "done", + "tool": "get_metric_series", + "window": "last90days", + }, + { + "domain": "sleep", + "id": "s2", + "label": "Reading Sleep, all time", + "labelKey": "coach.step.readWindow", + "reason": "no_data", + "status": "empty", + "tool": "get_sleep", + "window": "allTime", + }, + { + "domain": "labs", + "id": "s3", + "label": "Reading Lab results, last 12 months", + "labelKey": "coach.step.readWindow", + "reason": "retrieval_failed", + "status": "failed", + "tool": "get_labs", + "window": "lastYear", + }, + ], + "toolCalls": [ + { + "name": "get_metric_series", + "present": true, + }, + ], + "windows": [ + "last30days", + ], + }, + "model": "m-tool", + "providerType": "anthropic", + "role": "assistant", + "tokensUsed": 80, + }, + { + "call": "annotate", + "metaKeys": [ + "clusterCount", + "conversationId", + "historyTurns", + "promptChars", + "promptVersion", + "provider", + "snapshotSent", + "tokens", + "toolMode", + "toolsCalled", + ], + "name": "insights.coach.replied", + }, + ], + "contentType": "text/event-stream; charset=utf-8", + "frames": [ + { + "step": { + "domain": "bp", + "id": "s1", + "label": "Reading Blood pressure, last 90 days", + "labelKey": "coach.step.readWindow", + "status": "running", + "tool": "get_metric_series", + "window": "last90days", + }, + "type": "step", + }, + { + "step": { + "domain": "sleep", + "id": "s2", + "label": "Reading Sleep, all time", + "labelKey": "coach.step.readWindow", + "status": "running", + "tool": "get_sleep", + "window": "allTime", + }, + "type": "step", + }, + { + "step": { + "domain": "labs", + "id": "s3", + "label": "Reading Lab results, last 12 months", + "labelKey": "coach.step.readWindow", + "status": "running", + "tool": "get_labs", + "window": "lastYear", + }, + "type": "step", + }, + { + "step": { + "count": 142, + "domain": "bp", + "id": "s1", + "label": "Reading Blood pressure, last 90 days", + "labelKey": "coach.step.readWindow", + "status": "done", + "tool": "get_metric_series", + "window": "last90days", + }, + "type": "step", + }, + { + "step": { + "domain": "sleep", + "id": "s2", + "label": "Reading Sleep, all time", + "labelKey": "coach.step.readWindow", + "reason": "no_data", + "status": "empty", + "tool": "get_sleep", + "window": "allTime", + }, + "type": "step", + }, + { + "step": { + "domain": "labs", + "id": "s3", + "label": "Reading Lab results, last 12 months", + "labelKey": "coach.step.readWindow", + "reason": "retrieval_failed", + "status": "failed", + "tool": "get_labs", + "window": "lastYear", + }, + "type": "step", + }, + { + "token": "Your ", + "type": "token", + }, + { + "token": "systolic ", + "type": "token", + }, + { + "token": "averaged ", + "type": "token", + }, + { + "token": "128 ", + "type": "token", + }, + { + "token": "mmHg.", + "type": "token", + }, + { + "metricSource": { + "groundedFigures": [ + 128, + 82, + 46, + 105, + 35.9375, + 56.09756097560976, + 42, + ], + "metrics": [ + "bp", + ], + "steps": [ + { + "count": 142, + "domain": "bp", + "id": "s1", + "label": "Reading Blood pressure, last 90 days", + "labelKey": "coach.step.readWindow", + "status": "done", + "tool": "get_metric_series", + "window": "last90days", + }, + { + "domain": "sleep", + "id": "s2", + "label": "Reading Sleep, all time", + "labelKey": "coach.step.readWindow", + "reason": "no_data", + "status": "empty", + "tool": "get_sleep", + "window": "allTime", + }, + { + "domain": "labs", + "id": "s3", + "label": "Reading Lab results, last 12 months", + "labelKey": "coach.step.readWindow", + "reason": "retrieval_failed", + "status": "failed", + "tool": "get_labs", + "window": "lastYear", + }, + ], + "toolCalls": [ + { + "name": "get_metric_series", + "present": true, + }, + ], + "windows": [ + "last30days", + ], + }, + "type": "provenance", + }, + { + "conversationId": "c-new", + "messageId": "m2", + "type": "done", + "usage": { + "model": "m-tool", + "totalTokens": 80, + }, + }, + ], + "status": 200, + "thrown": null, +} +`; + exports[`coach chat golden transcripts — refusals before the stream > R1 a fence-drift conversation 404s with an audit row 1`] = ` { "body": null, diff --git a/src/app/api/insights/chat/__tests__/golden-transcripts.test.ts b/src/app/api/insights/chat/__tests__/golden-transcripts.test.ts index f6a48aa2f..aecc3083c 100644 --- a/src/app/api/insights/chat/__tests__/golden-transcripts.test.ts +++ b/src/app/api/insights/chat/__tests__/golden-transcripts.test.ts @@ -120,8 +120,13 @@ vi.mock("@/lib/medications/scheduled-doses", () => ({ vi.mock("@/lib/ai/coach/workout-evidence-builder", () => ({ buildWorkoutEvidenceSection: h.m.buildWorkoutEvidenceSection, })); -vi.mock("@/lib/ai/coach/tools", () => ({ +vi.mock("@/lib/ai/coach/tools", async () => ({ COACH_TOOL_DEFS: [{ name: "get_metric_series" }, { name: "get_sleep" }], + parseCoachToolArgs: ( + await vi.importActual( + "@/lib/ai/coach/tools/definitions", + ) + ).parseCoachToolArgs, MAX_ROUNDS: 3, buildCoachDataInventory: h.m.buildCoachDataInventory, renderDataInventory: h.m.renderDataInventory, @@ -521,6 +526,54 @@ describe("coach chat golden transcripts", () => { reply("Du läufst 30 km pro Woche, dein Blutdruck liegt bei 128 mmHg."); await golden({ message: "Wie ist mein Blutdruck?", locale: "de" }); }); + + it("22 tool mode with live steps: found, empty and failed, all before the first token", async () => { + type Call = { id: string; name: string; arguments: string }; + type Settled = { present: boolean; reason?: string; data?: unknown }; + const calls: Array<[Call, Settled]> = [ + [ + { + id: "a", + name: "get_metric_series", + arguments: '{"metric":"bp","window":"last90days"}', + }, + { + present: true, + data: { + metric: "bp", + section: { aggregate: { coverage: { count: 142 } } }, + }, + }, + ], + [ + { id: "b", name: "get_sleep", arguments: "{}" }, + { present: false, reason: "no_data" }, + ], + [ + { id: "c", name: "get_labs", arguments: '{"analyte":"Ferritin"}' }, + { present: false, reason: "retrieval_failed" }, + ], + ]; + m.runCoachToolLoop.mockImplementation( + async (args: { + onCallStart?: (call: Call, index: number) => void; + onCallSettled?: (call: Call, result: Settled, index: number) => void; + }) => { + calls.forEach(([call], i) => args.onCallStart?.(call, i)); + calls.forEach(([call, result], i) => + args.onCallSettled?.(call, result, i), + ); + return toolLoopResult("Your systolic averaged 128 mmHg."); + }, + ); + const transcript = await golden({ message: "How is my blood pressure?" }); + const types = transcript.frames.map((f) => (f as { type: string }).type); + const firstToken = types.indexOf("token"); + expect(firstToken).toBeGreaterThan(0); + expect(types.lastIndexOf("step")).toBeLessThan(firstToken); + expect(types.filter((t) => t === "step")).toHaveLength(6); + expect(JSON.stringify(transcript)).not.toContain("Ferritin"); + }); }); describe("coach chat golden transcripts — refusals before the stream", () => { diff --git a/src/app/api/insights/chat/__tests__/route-capability-gate.test.ts b/src/app/api/insights/chat/__tests__/route-capability-gate.test.ts index 0ef7fb98f..9bcfdfa39 100644 --- a/src/app/api/insights/chat/__tests__/route-capability-gate.test.ts +++ b/src/app/api/insights/chat/__tests__/route-capability-gate.test.ts @@ -66,7 +66,9 @@ vi.mock("@/lib/ai/provider", () => ({ })); vi.mock("@/lib/ai/consent-guard", () => ({ assertConsentForChain: vi.fn() })); vi.mock("@/lib/ai/prompts/insight-generator", () => ({ PROMPT_VERSION: "x" })); -vi.mock("@/lib/ai/coach/types", () => ({ +// The rest of the module stays real: the step labels read its enums at load. +vi.mock("@/lib/ai/coach/types", async (importOriginal) => ({ + ...(await importOriginal()), coachChatRequestSchema: { safeParse: () => ({ success: false, error: { issues: [] } }), }, diff --git a/src/app/api/insights/chat/__tests__/route-list-query.test.ts b/src/app/api/insights/chat/__tests__/route-list-query.test.ts index b151a2a1b..f22d84152 100644 --- a/src/app/api/insights/chat/__tests__/route-list-query.test.ts +++ b/src/app/api/insights/chat/__tests__/route-list-query.test.ts @@ -53,7 +53,9 @@ vi.mock("@/lib/ai/provider", () => ({ })); vi.mock("@/lib/ai/consent-guard", () => ({ assertConsentForChain: vi.fn() })); vi.mock("@/lib/ai/prompts/insight-generator", () => ({ PROMPT_VERSION: "x" })); -vi.mock("@/lib/ai/coach/types", () => ({ +// The rest of the module stays real: the step labels read its enums at load. +vi.mock("@/lib/ai/coach/types", async (importOriginal) => ({ + ...(await importOriginal()), coachChatRequestSchema: { safeParse: () => ({ success: false }) }, })); vi.mock("@/lib/ai/coach/tools", () => ({ diff --git a/src/lib/ai/coach/turn/__tests__/steps.test.ts b/src/lib/ai/coach/turn/__tests__/steps.test.ts new file mode 100644 index 000000000..5d29e880c --- /dev/null +++ b/src/lib/ai/coach/turn/__tests__/steps.test.ts @@ -0,0 +1,501 @@ +/** + * v1.39.4 — live steps carry catalog labels, closed-enum domains and windows, + * and server-counted integers. Nothing else: not a value, not an argument + * the model wrote, not an analyte name. + * + * The property test drives every tool with fuzzed arguments and fuzzed + * results (seeded, so a failure reproduces) and checks each step against the + * full set of strings the catalog can render. + */ +import { readFileSync } from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { locales, type Locale } from "@/lib/i18n/config"; +import { getServerTranslator } from "@/lib/i18n/server-translator"; +import { + COACH_STEP_LABEL_KEYS, + coachDomainLabelKey, + coachWindowLabelKey, +} from "@/lib/ai/coach/dialog-keys"; +import { + coachScopeSourceSchema, + coachScopeWindowSchema, + type CoachStep, + type CoachStepDomain, +} from "@/lib/ai/coach/types"; +import { + COACH_TOOL_NAMES, + parseCoachToolArgs, +} from "@/lib/ai/coach/tools/definitions"; +import type { CoachToolResult } from "@/lib/ai/coach/tools/executor"; + +import { MAX_TURN_STEPS, snapshotStep, toStep } from "../steps"; + +const SOURCES = coachScopeSourceSchema.options; +const WINDOWS = coachScopeWindowSchema.options; +const DOMAINS: CoachStepDomain[] = [ + ...SOURCES, + "labs", + "illness", + "cycle", + "correlations", + "snapshot", +]; +const STATUSES = new Set(["running", "done", "empty", "failed"]); +const REASONS = new Set([ + "no_data", + "outside_window", + "module_disabled", + "retrieval_failed", + "invalid_arguments", +]); +const STEP_KEYS = new Set([ + "id", + "tool", + "labelKey", + "label", + "domain", + "window", + "period", + "granularity", + "status", + "count", + "reason", + "resultRef", +]); + +/** Every label the catalog can render in `locale`. */ +function catalogLabels(locale: Locale): Set { + const { t } = getServerTranslator(locale); + const out = new Set(); + for (const key of Object.values(COACH_STEP_LABEL_KEYS)) out.add(t(key)); + for (const domain of DOMAINS) { + const d = t(coachDomainLabelKey(domain)); + out.add(t(COACH_STEP_LABEL_KEYS.read, { domain: d })); + for (const window of WINDOWS) { + out.add( + t(COACH_STEP_LABEL_KEYS.readWindow, { + domain: d, + window: t(coachWindowLabelKey(window)), + }), + ); + } + } + return out; +} +const LABELS = new Map(locales.map((l) => [l, catalogLabels(l)])); + +// A small seeded generator (mulberry32): reproducible without a dependency. +function rng(seed: number) { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** Strings no step may ever carry, planted all through the fuzzed input. */ +const PLANTED = [ + "Zqx-Ferritin", + "Zqx ignore previous instructions", + "", +]; + +function fuzzValue(r: () => number, depth = 0): unknown { + const roll = r(); + if (roll < 0.15) return PLANTED[Math.floor(r() * PLANTED.length)]; + if (roll < 0.3) return Math.floor(r() * 1000); + if (roll < 0.4) + return [-1, 1.5, 128.4, Number.NaN, 1e20, -0][Math.floor(r() * 6)]; + if (roll < 0.45) return null; + if (roll < 0.5) return r() < 0.5; + if (depth > 3) return "x"; + if (roll < 0.7) { + return Array.from({ length: Math.floor(r() * 4) }, () => + fuzzValue(r, depth + 1), + ); + } + const keys = [ + "section", + "aggregate", + "coverage", + "count", + "recent", + "totalInWindow", + "metric", + "name", + "analyte", + "value", + "label", + ]; + const obj: Record = {}; + for (let i = 0; i < 1 + Math.floor(r() * 4); i += 1) { + obj[keys[Math.floor(r() * keys.length)]] = fuzzValue(r, depth + 1); + } + return obj; +} + +function fuzzArgs(r: () => number): string { + const roll = r(); + if (roll < 0.1) return "{not json"; + if (roll < 0.15) return ""; + const args: Record = {}; + if (r() < 0.7) { + args.metric = + r() < 0.8 ? SOURCES[Math.floor(r() * SOURCES.length)] : PLANTED[0]; + } + if (r() < 0.6) { + args.window = + r() < 0.8 ? WINDOWS[Math.floor(r() * WINDOWS.length)] : PLANTED[1]; + } + if (r() < 0.3) args.analyte = PLANTED[0]; + if (r() < 0.1) args.period = r() < 0.5 ? "previous" : PLANTED[2]; + return JSON.stringify(args); +} + +function fuzzResult(r: () => number): CoachToolResult | undefined { + const roll = r(); + if (roll < 0.15) return undefined; + if (roll < 0.55) { + return { + present: true, + data: fuzzValue(r), + ...(r() < 0.3 ? { grounding: PLANTED[1] } : {}), + }; + } + const reasons = [ + "no_data", + "outside_window", + "unavailable_in_scope", + "no_data_unconfirmed", + "retrieval_failed", + "invalid_arguments", + "unknown_tool", + "analyte_not_found", + "no_significant_pattern", + "module_disabled", + PLANTED[1], + "__proto__", + "toString", + ]; + return { + present: false, + reason: reasons[Math.floor(r() * reasons.length)], + ...(r() < 0.4 + ? { + searchedWindow: + r() < 0.7 ? WINDOWS[Math.floor(r() * WINDOWS.length)] : PLANTED[0], + } + : {}), + ...(r() < 0.4 + ? { + available: { + count: fuzzValue(r) as number, + firstDate: PLANTED[0], + lastDate: "2024-01-01", + reachableWithWindow: null, + }, + } + : {}), + }; +} + +function assertClean(step: CoachStep, locale: Locale) { + for (const key of Object.keys(step)) expect(STEP_KEYS.has(key)).toBe(true); + expect(step.id).toMatch(/^s([1-9]|1[0-2])$/); + expect( + [...COACH_TOOL_NAMES, "snapshot", "show_result"].includes(step.tool), + ).toBe(true); + expect(Object.values(COACH_STEP_LABEL_KEYS)).toContain(step.labelKey); + expect(LABELS.get(locale)!.has(step.label)).toBe(true); + if (step.domain !== undefined) expect(DOMAINS).toContain(step.domain); + if (step.window !== undefined) expect(WINDOWS).toContain(step.window); + if (step.period !== undefined) { + expect(["current", "previous", "yearAgo"]).toContain(step.period); + } + if (step.granularity !== undefined) { + expect(["day", "week", "month"]).toContain(step.granularity); + } + expect(STATUSES.has(step.status)).toBe(true); + if (step.count !== undefined) { + expect(Number.isSafeInteger(step.count)).toBe(true); + expect(step.count).toBeGreaterThanOrEqual(0); + } + if (step.reason !== undefined) expect(REASONS.has(step.reason)).toBe(true); + const wire = JSON.stringify(step); + for (const planted of PLANTED) expect(wire).not.toContain(planted); + expect(wire.toLowerCase()).not.toContain("zqx"); +} + +describe("toStep — property: only catalog keys, domains, windows and integers", () => { + it("holds for every tool over fuzzed arguments and results in every locale", () => { + const r = rng(0x5eed); + let produced = 0; + for (let i = 0; i < 4000; i += 1) { + const name = COACH_TOOL_NAMES[i % COACH_TOOL_NAMES.length]; + const locale = locales[Math.floor(r() * locales.length)]; + const call = { id: `c${i}`, name, arguments: fuzzArgs(r) }; + const step = toStep({ + call, + index: Math.floor(r() * 14), + parsedArgs: parseCoachToolArgs(call.name, call.arguments), + result: fuzzResult(r), + locale, + ...(r() < 0.5 + ? { fallbackWindow: WINDOWS[Math.floor(r() * WINDOWS.length)] } + : {}), + }); + if (step === null) continue; + produced += 1; + assertClean(step, locale); + } + // The generator must actually exercise the mapper, not null out. + expect(produced).toBeGreaterThan(2000); + }); + + it("covers every tool with a step", () => { + for (const name of COACH_TOOL_NAMES) { + const args = name === "get_metric_series" ? '{"metric":"bp"}' : "{}"; + const step = toStep({ + call: { id: "x", name, arguments: args }, + index: 0, + parsedArgs: parseCoachToolArgs(name, args), + locale: "en", + }); + expect(step, name).not.toBeNull(); + assertClean(step!, "en"); + } + }); +}); + +describe("toStep — mapping", () => { + const bp = { + id: "a", + name: "get_metric_series", + arguments: '{"metric":"bp","window":"last90days"}', + }; + const parsed = parseCoachToolArgs(bp.name, bp.arguments); + + it("a started call is running, with the label naming domain and window", () => { + expect( + toStep({ call: bp, index: 0, parsedArgs: parsed, locale: "en" }), + ).toEqual({ + id: "s1", + tool: "get_metric_series", + labelKey: "coach.step.readWindow", + label: "Reading Blood pressure, last 90 days", + domain: "bp", + window: "last90days", + status: "running", + }); + }); + + it("a found result counts the readings the aggregate covered", () => { + const step = toStep({ + call: bp, + index: 2, + parsedArgs: parsed, + locale: "de", + result: { + present: true, + data: { + metric: "bp", + section: { aggregate: { coverage: { count: 142 }, mean: 128.4 } }, + }, + }, + }); + expect(step).toMatchObject({ id: "s3", status: "done", count: 142 }); + expect(step?.label).toBe("Lese Blutdruck, letzte 90 Tage"); + }); + + it("maps each miss to its status and reason", () => { + const at = (reason: string, extra: Partial = {}) => + toStep({ + call: bp, + index: 0, + parsedArgs: parsed, + locale: "en", + result: { present: false, reason, ...extra }, + }); + expect(at("no_data")).toMatchObject({ status: "empty", reason: "no_data" }); + expect( + at("outside_window", { + available: { + count: 40, + firstDate: "2023-01-01", + lastDate: "2023-06-01", + reachableWithWindow: null, + }, + }), + ).toMatchObject({ status: "empty", reason: "outside_window", count: 40 }); + expect(at("retrieval_failed")).toMatchObject({ + status: "failed", + reason: "retrieval_failed", + }); + expect(at("no_data_unconfirmed")).toMatchObject({ + status: "failed", + reason: "retrieval_failed", + }); + expect(at("invalid_arguments")).toMatchObject({ + status: "failed", + reason: "invalid_arguments", + }); + const inScope = at("unavailable_in_scope"); + expect(inScope?.status).toBe("empty"); + expect(inScope?.reason).toBeUndefined(); + }); + + it("uses the conversation window when the call names none", () => { + const call = { id: "s", name: "get_sleep", arguments: "{}" }; + expect( + toStep({ + call, + index: 0, + parsedArgs: {}, + locale: "en", + fallbackWindow: "last7days", + }), + ).toMatchObject({ domain: "sleep", window: "last7days" }); + }); + + it("never names a lab analyte, and says the fixed year labs read", () => { + const call = { + id: "l", + name: "get_labs", + arguments: '{"analyte":"Ferritin"}', + }; + const step = toStep({ + call, + index: 0, + parsedArgs: parseCoachToolArgs(call.name, call.arguments), + locale: "en", + fallbackWindow: "last7days", + result: { + present: true, + data: { recent: [{ name: "Ferritin", value: 37.5, unit: "ng/mL" }] }, + }, + }); + expect(step).toMatchObject({ + domain: "labs", + window: "lastYear", + status: "done", + count: 1, + }); + expect(JSON.stringify(step)).not.toMatch(/ferritin|ng\/mL|37/i); + }); + + it("gives no step past the cap, for an unknown tool, or an unreadable metric", () => { + expect( + toStep({ + call: bp, + index: MAX_TURN_STEPS, + parsedArgs: parsed, + locale: "en", + }), + ).toBeNull(); + expect( + toStep({ + call: bp, + index: MAX_TURN_STEPS - 1, + parsedArgs: parsed, + locale: "en", + })?.id, + ).toBe("s12"); + expect( + toStep({ + call: { id: "u", name: "get_everything", arguments: "{}" }, + index: 0, + parsedArgs: undefined, + locale: "en", + }), + ).toBeNull(); + expect( + toStep({ + call: { id: "m", name: "get_metric_series", arguments: "{nope" }, + index: 0, + parsedArgs: undefined, + locale: "en", + result: { present: false, reason: "invalid_arguments" }, + }), + ).toBeNull(); + }); +}); + +describe("snapshotStep", () => { + it("is one settled step counting the metrics", () => { + expect(snapshotStep({ metricCount: 5, locale: "en" })).toEqual({ + id: "s1", + tool: "snapshot", + labelKey: "coach.step.snapshot", + label: "Reading your record summary", + domain: "snapshot", + status: "done", + count: 5, + }); + }); + + it("an empty snapshot says so", () => { + expect(snapshotStep({ metricCount: 0, locale: "en" })).toMatchObject({ + status: "empty", + count: 0, + reason: "no_data", + }); + }); +}); + +/** + * A step that started and never settled would spin forever in the UI and be + * persisted as running. The loop's behaviour is tested with mocks in + * `tools-loop-callbacks.test.ts`; this pins the shape that makes it true for + * every call: nothing between `onCallStart` and `onCallSettled` but the + * executor, and the executor turns every failure into a result. + */ +describe("the tool loop settles every call it starts (structural)", () => { + const root = path.resolve(__dirname, "../../../../../.."); + const loop = readFileSync( + path.join(root, "src/lib/ai/coach/tools/loop.ts"), + "utf8", + ); + const executor = readFileSync( + path.join(root, "src/lib/ai/coach/tools/executor.ts"), + "utf8", + ); + + it("fires onCallStart and onCallSettled exactly once each per call", () => { + expect(loop.match(/onCallStart\?\.\(/g)).toHaveLength(1); + expect(loop.match(/onCallSettled\?\.\(/g)).toHaveLength(1); + }); + + it("runs only the executor between the two, with no exit in between", () => { + const start = loop.indexOf("onCallStart?.("); + const settle = loop.indexOf("onCallSettled?.("); + expect(start).toBeGreaterThan(-1); + expect(settle).toBeGreaterThan(start); + const between = loop.slice(start, settle); + expect(between).toMatch(/await\s+executeCoachTool\(/); + expect(between).not.toMatch(/\breturn\b|\bthrow\b|\bcontinue\b|\bbreak\b/); + expect(between).not.toMatch(/\bif\s*\(/); + }); + + it("the executor answers every failure with a result instead of throwing", () => { + const body = executor.slice( + executor.indexOf("export async function executeCoachTool("), + executor.indexOf("async function dispatch("), + ); + expect(body.length).toBeGreaterThan(0); + // The one call that reaches data runs inside a try whose catch returns. + expect(body).toMatch( + /try\s*\{[\s\S]*await dispatch\([\s\S]*\}\s*catch[\s\S]*return \{ present: false, reason: "retrieval_failed" \}/, + ); + expect(body).not.toMatch(/\bthrow\b/); + // JSON parsing is guarded too. + expect(body).toMatch( + /try\s*\{[\s\S]*JSON\.parse\(rawArguments\)[\s\S]*\}\s*catch/, + ); + }); +}); diff --git a/src/lib/ai/coach/turn/model.ts b/src/lib/ai/coach/turn/model.ts index 4c3296383..6c12649c1 100644 --- a/src/lib/ai/coach/turn/model.ts +++ b/src/lib/ai/coach/turn/model.ts @@ -211,6 +211,7 @@ export async function runTurnModel(args: { index, parsedArgs: parseCoachToolArgs(call.name, call.arguments), locale, + fallbackWindow: effectiveScope?.window, }), ), onCallSettled: (call, result, index) => { @@ -220,7 +221,16 @@ export async function runTurnModel(args: { ...(parsedArgs ? { args: parsedArgs } : {}), result, }; - steps.record(toStep({ call, index, parsedArgs, result, locale })); + steps.record( + toStep({ + call, + index, + parsedArgs, + result, + locale, + fallbackWindow: effectiveScope?.window, + }), + ); }, }); // v1.32.1 — the numeric verifier ACTIVATES only when this turn actually diff --git a/src/lib/ai/coach/turn/steps.ts b/src/lib/ai/coach/turn/steps.ts index 72cd3d381..aff8ba98c 100644 --- a/src/lib/ai/coach/turn/steps.ts +++ b/src/lib/ai/coach/turn/steps.ts @@ -4,38 +4,288 @@ * * A step carries a catalog label key, the server-rendered label, the domain, * the window and a server-counted number. Never free text, an analyte name - * or a health value. - * - * Not built yet: both functions answer null, so a turn emits no step frame. + * or a health value: every field is either a closed enum member checked + * here, a catalog string rendered from those enums, or a non-negative + * integer the server counted. Nothing the model wrote and nothing the + * record holds as text can reach it, because nothing is copied through + * without passing one of those checks. */ import type { Locale } from "@/lib/i18n/config"; +import { getServerTranslator } from "@/lib/i18n/server-translator"; import type { AiToolCall } from "@/lib/ai/types"; -import type { CoachStep } from "@/lib/ai/coach/types"; +import { + coachScopeSourceSchema, + coachScopeWindowSchema, + type CoachResultGranularity, + type CoachResultPeriod, + type CoachScopeWindow, + type CoachStep, + type CoachStepDomain, + type CoachStepReason, + type CoachStepStatus, +} from "@/lib/ai/coach/types"; +import { + COACH_STEP_LABEL_KEYS, + coachDomainLabelKey, + coachWindowLabelKey, +} from "@/lib/ai/coach/dialog-keys"; import type { CoachToolResult } from "@/lib/ai/coach/tools/executor"; +import { + isCoachToolName, + type CoachToolName, +} from "@/lib/ai/coach/tools/definitions"; + +/** At most this many steps per turn; later calls run but show no step. */ +export const MAX_TURN_STEPS = 12; + +const SCOPE_SOURCES: ReadonlySet = new Set( + coachScopeSourceSchema.options, +); +const WINDOWS: ReadonlySet = new Set(coachScopeWindowSchema.options); +const PERIODS: ReadonlySet = new Set([ + "current", + "previous", + "yearAgo", +]); +const GRANULARITIES: ReadonlySet = new Set([ + "day", + "week", + "month", +]); + +/** + * The domain each tool reads. `null` means the domain is the validated + * `metric` argument. A record keyed on the tool name, so a new tool does not + * compile until it says what it reads. + */ +const TOOL_DOMAIN: Readonly> = { + get_metric_series: null, + get_glucose_panel: "glucose", + get_sleep: "sleep", + get_medication_compliance: "compliance", + get_labs: "labs", + get_illness_recovery: "illness", + get_workouts: "workouts", + get_cycle: "cycle", + get_correlations: "correlations", +}; + +/** + * The window a tool reads when the call names none. `"fallback"` is the + * conversation's window (the executor's default); `null` is a read with no + * window at all. Labs read a fixed trailing year whatever the conversation + * says, so their step says that year rather than the conversation's window. + */ +const TOOL_WINDOW: Readonly< + Record +> = { + get_metric_series: "fallback", + get_glucose_panel: "fallback", + get_sleep: "fallback", + get_medication_compliance: "fallback", + get_labs: "lastYear", + get_illness_recovery: "fallback", + get_workouts: "fallback", + get_cycle: null, + get_correlations: null, +}; + +/** + * A miss reason from the executor, mapped onto the step. Anything not listed + * (a correlation reader's own "no pattern survived" code, a rows-exist-but- + * no-block verdict) is an empty step with no stated reason: saying "no + * readings" there would be false. + */ +const MISS: Readonly< + Record +> = { + no_data: { status: "empty", reason: "no_data" }, + analyte_not_found: { status: "empty", reason: "no_data" }, + outside_window: { status: "empty", reason: "outside_window" }, + module_disabled: { status: "empty", reason: "module_disabled" }, + retrieval_failed: { status: "failed", reason: "retrieval_failed" }, + no_data_unconfirmed: { status: "failed", reason: "retrieval_failed" }, + invalid_arguments: { status: "failed", reason: "invalid_arguments" }, + unknown_tool: { status: "failed", reason: "invalid_arguments" }, + unsupported_metric: { status: "failed", reason: "invalid_arguments" }, + use_get_glucose_panel: { status: "failed", reason: "invalid_arguments" }, + use_get_medication_compliance: { + status: "failed", + reason: "invalid_arguments", + }, +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** A non-negative integer, or undefined. Never a float: a count, not a value. */ +function asCount(value: unknown): number | undefined { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 + ? value + : undefined; +} + +function pick( + allowed: ReadonlySet, + value: unknown, +): T | undefined { + return typeof value === "string" && allowed.has(value) + ? (value as T) + : undefined; +} + +/** + * How many readings or rows a present result covered, when the result says + * so in a field the server counted: + * - a metric series: the aggregate's coverage count; + * - workouts: the sessions in the window; + * - labs: the biomarkers returned (one row each). + * Anything else carries no count rather than a guess. + */ +function presentCount(tool: CoachToolName, data: unknown): number | undefined { + if (!isRecord(data)) return undefined; + switch (tool) { + case "get_metric_series": { + const section = data.section; + if (!isRecord(section) || !isRecord(section.aggregate)) return undefined; + const coverage = section.aggregate.coverage; + return isRecord(coverage) ? asCount(coverage.count) : undefined; + } + case "get_workouts": + return asCount(data.totalInWindow); + case "get_labs": + return Array.isArray(data.recent) ? data.recent.length : undefined; + default: + return undefined; + } +} + +function render( + locale: Locale, + domain: CoachStepDomain, + window: CoachScopeWindow | undefined, +): { labelKey: string; label: string } { + const { t } = getServerTranslator(locale); + const labelKey = window + ? COACH_STEP_LABEL_KEYS.readWindow + : COACH_STEP_LABEL_KEYS.read; + return { + labelKey, + label: t(labelKey, { + domain: t(coachDomainLabelKey(domain)), + ...(window ? { window: t(coachWindowLabelKey(window)) } : {}), + }), + }; +} /** * The step for one tool call: `running` when `result` is absent (the call * just started), its final status once the result is in. `index` counts * calls across the turn from 0; `parsedArgs` are the schema-validated - * arguments, absent when they did not validate. + * arguments, absent when they did not validate. `fallbackWindow` is the + * conversation's window, the one a call without a window reads. + * + * Null for a call past the cap, for a tool name outside the catalogue, and + * for a metric call whose metric did not validate: there is nothing true to + * label those with. */ -export function toStep(_args: { +export function toStep(args: { call: AiToolCall; index: number; parsedArgs: Record | undefined; result?: CoachToolResult; locale: Locale; + fallbackWindow?: CoachScopeWindow; }): CoachStep | null { - return null; + const { call, index, parsedArgs, result, locale } = args; + if (!Number.isSafeInteger(index) || index < 0 || index >= MAX_TURN_STEPS) { + return null; + } + if (!isCoachToolName(call.name)) return null; + const tool = call.name; + + const domain = + TOOL_DOMAIN[tool] ?? + pick(SCOPE_SOURCES, parsedArgs?.metric); + // A metric series call whose arguments did not validate names no domain + // the executor would read; the executor refuses it and the model is told. + // There is nothing true to show, so it gets no step. + if (!domain) return null; + + const windowRule = TOOL_WINDOW[tool]; + const window = + windowRule === null + ? undefined + : windowRule === "fallback" + ? (pick(WINDOWS, parsedArgs?.window) ?? + pick(WINDOWS, result?.searchedWindow) ?? + pick(WINDOWS, args.fallbackWindow)) + : windowRule; + const period = pick(PERIODS, parsedArgs?.period); + const granularity = pick( + GRANULARITIES, + parsedArgs?.granularity, + ); + + const settled = settle(tool, result); + return { + id: `s${index + 1}`, + tool, + ...render(locale, domain, window), + domain, + ...(window ? { window } : {}), + ...(period ? { period } : {}), + ...(granularity ? { granularity } : {}), + status: settled.status, + ...(settled.count !== undefined ? { count: settled.count } : {}), + ...(settled.reason ? { reason: settled.reason } : {}), + }; +} + +function settle( + tool: CoachToolName, + result: CoachToolResult | undefined, +): { status: CoachStepStatus; count?: number; reason?: CoachStepReason } { + if (!result) return { status: "running" }; + if (result.present) { + return { status: "done", count: presentCount(tool, result.data) }; + } + const miss = (typeof result.reason === "string" && + Object.hasOwn(MISS, result.reason) && + MISS[result.reason]) || { status: "empty" as const }; + // An out-of-window miss knows how much the record holds elsewhere; the + // count rides the step for the method line, the row shows the reason. + const count = + miss.status === "empty" && isRecord(result.available) + ? asCount(result.available.count) + : undefined; + return { + status: miss.status, + ...(count !== undefined ? { count } : {}), + ...(miss.reason ? { reason: miss.reason } : {}), + }; } /** * The single step a no-tools turn shows: the full snapshot, with the number - * of metrics it covered. + * of metrics it covered. The snapshot is already built when this runs, so + * the step is settled from the start. */ -export function snapshotStep(_args: { +export function snapshotStep(args: { metricCount: number; locale: Locale; }): CoachStep | null { - return null; + const { t } = getServerTranslator(args.locale); + const count = asCount(args.metricCount) ?? 0; + return { + id: "s1", + tool: "snapshot", + labelKey: COACH_STEP_LABEL_KEYS.snapshot, + label: t(COACH_STEP_LABEL_KEYS.snapshot), + domain: "snapshot", + status: count > 0 ? "done" : "empty", + count, + ...(count === 0 ? { reason: "no_data" as const } : {}), + }; }