diff --git a/src/lib/ai/coach/__tests__/tools-loop-callbacks.test.ts b/src/lib/ai/coach/__tests__/tools-loop-callbacks.test.ts new file mode 100644 index 000000000..f5f0cc3c4 --- /dev/null +++ b/src/lib/ai/coach/__tests__/tools-loop-callbacks.test.ts @@ -0,0 +1,152 @@ +/** + * v1.39.4 — the loop's progress seam: every call that starts settles, the + * trace keeps only schema-validated arguments, and a forced answer at the + * round cap is reported as such. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { CoachToolResult } from "@/lib/ai/coach/tools/executor"; + +const executeCoachTool = vi.fn<(args?: unknown) => Promise>(); +vi.mock("@/lib/ai/coach/tools/executor", () => ({ + executeCoachTool: (args: unknown) => executeCoachTool(args), +})); +const runRawCompletionWithFallback = vi.fn(); +vi.mock("@/lib/ai/provider-runner", () => ({ + runRawCompletionWithFallback: (args: unknown) => + runRawCompletionWithFallback(args), +})); + +import { HARD_CAP, runCoachToolLoop } from "@/lib/ai/coach/tools/loop"; +import { COACH_TOOL_DEFS } from "@/lib/ai/coach/tools/definitions"; + +type Call = { id: string; name: string; arguments: string }; + +function round(content: string, toolCalls?: Call[]) { + return { + result: { + content, + tokensUsed: 5, + model: "mock", + providerType: "anthropic" as const, + ...(toolCalls + ? { toolCalls, finishReason: "tool_calls" as const } + : { finishReason: "stop" as const }), + }, + workingProvider: { providerType: "anthropic" }, + fallbackHops: [], + }; +} + +const baseArgs = { + userId: "u1", + providers: [], + system: "sys", + messages: [{ role: "user" as const, content: "q" }], + tools: COACH_TOOL_DEFS, +}; + +beforeEach(() => { + executeCoachTool.mockReset(); + runRawCompletionWithFallback.mockReset(); +}); + +describe("runCoachToolLoop — progress callbacks", () => { + it("settles every call that started, misses and invalid calls included, with turn-wide indexes", async () => { + executeCoachTool + .mockResolvedValueOnce({ present: true, data: {} }) + .mockResolvedValueOnce({ present: false, reason: "invalid_arguments" }) + .mockResolvedValueOnce({ present: false, reason: "unknown_tool" }); + runRawCompletionWithFallback + .mockResolvedValueOnce( + round("", [ + { id: "a", name: "get_metric_series", arguments: '{"metric":"bp"}' }, + { id: "b", name: "get_sleep", arguments: "{not json" }, + ]), + ) + .mockResolvedValueOnce( + round("", [{ id: "c", name: "get_everything", arguments: "{}" }]), + ) + .mockResolvedValueOnce(round("answer")); + + const started: Array<[string, number]> = []; + const settled: Array<[string, boolean, number]> = []; + const out = await runCoachToolLoop({ + ...baseArgs, + onCallStart: (call, index) => started.push([call.id, index]), + onCallSettled: (call, result, index) => + settled.push([call.id, result.present, index]), + }); + + expect(started).toEqual([ + ["a", 0], + ["b", 1], + ["c", 2], + ]); + expect([...settled].sort((x, y) => x[2] - y[2])).toEqual([ + ["a", true, 0], + ["b", false, 1], + ["c", false, 2], + ]); + expect(out.forcedFinal).toBe(false); + }); + + it("keeps only schema-validated arguments on the trace", async () => { + executeCoachTool.mockResolvedValue({ present: false }); + runRawCompletionWithFallback + .mockResolvedValueOnce( + round("", [ + { id: "a", name: "get_metric_series", arguments: '{"metric":"bp"}' }, + // Unknown key: the strict schema refuses it. + { + id: "b", + name: "get_metric_series", + arguments: '{"metric":"bp","userId":"someone-else"}', + }, + { id: "c", name: "get_sleep", arguments: "{not json" }, + ]), + ) + .mockResolvedValueOnce(round("answer")); + + const out = await runCoachToolLoop(baseArgs); + const byArgs = out.toolTrace.map((t) => t.args ?? null); + expect(byArgs).toContainEqual({ metric: "bp" }); + expect(byArgs.filter((a) => a === null)).toHaveLength(2); + expect(JSON.stringify(out.toolTrace)).not.toContain("someone-else"); + }); + + it("a throwing callback never breaks the turn", async () => { + executeCoachTool.mockResolvedValue({ present: true, data: {} }); + runRawCompletionWithFallback + .mockResolvedValueOnce( + round("", [{ id: "a", name: "get_sleep", arguments: "{}" }]), + ) + .mockResolvedValueOnce(round("answer")); + const out = await runCoachToolLoop({ + ...baseArgs, + onCallStart: () => { + throw new Error("closed stream"); + }, + onCallSettled: () => { + throw new Error("closed stream"); + }, + }); + expect(out.result.content).toBe("answer"); + expect(out.toolTrace).toHaveLength(1); + }); + + it("reports a forced answer at the round cap", async () => { + executeCoachTool.mockResolvedValue({ present: false }); + runRawCompletionWithFallback.mockImplementation( + (args: { params: { toolChoice?: string } }) => + Promise.resolve( + args.params.toolChoice === "none" + ? round("forced") + : round("", [{ id: "x", name: "get_sleep", arguments: "{}" }]), + ), + ); + const out = await runCoachToolLoop(baseArgs); + expect(out.rounds).toBe(HARD_CAP); + expect(out.forcedFinal).toBe(true); + }); +}); diff --git a/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts b/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts index c19d6e9b6..ed35284c4 100644 --- a/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts +++ b/src/lib/ai/coach/__tests__/tools-loop-miss-payloads.test.ts @@ -79,7 +79,9 @@ describe("coach tool loop — which results reach the grounding guard", () => { const loop = await runCoachToolLoop(baseArgs); // The trace records that the tool ran and found nothing… - expect(loop.toolTrace).toEqual([{ name: "get_sleep", present: false }]); + expect(loop.toolTrace).toEqual([ + { name: "get_sleep", present: false, args: {} }, + ]); // …but the payload set the route grades against is empty, so the route's // `activatingPayloads.length > 0` gate is false and the numeric grounding // guard never runs for this turn. diff --git a/src/lib/ai/coach/__tests__/tools-loop.test.ts b/src/lib/ai/coach/__tests__/tools-loop.test.ts index 1d1e2c899..973c1d751 100644 --- a/src/lib/ai/coach/__tests__/tools-loop.test.ts +++ b/src/lib/ai/coach/__tests__/tools-loop.test.ts @@ -88,7 +88,7 @@ describe("runCoachToolLoop", () => { expect(out.result.content).toBe("Your BP looks steady."); expect(out.totalTokens).toBe(50); // summed across both rounds expect(out.toolTrace).toEqual([ - { name: "get_metric_series", present: true }, + { name: "get_metric_series", present: true, args: { metric: "bp" } }, ]); // v1.21.0 (P6) — the present result's payload is retained for the prose // number-verifier. @@ -278,7 +278,7 @@ describe("shared snapshot scope threading (D5-1)", () => { expect(out.toolResults).toHaveLength(1); expect(out.toolResults[0].available).toMatchObject({ count: 1597 }); expect(out.toolTrace).toEqual([ - { name: "get_glucose_panel", present: false }, + { name: "get_glucose_panel", present: false, args: {} }, ]); }); diff --git a/src/lib/ai/coach/tools/definitions.ts b/src/lib/ai/coach/tools/definitions.ts index ee8e73c22..8207bac1f 100644 --- a/src/lib/ai/coach/tools/definitions.ts +++ b/src/lib/ai/coach/tools/definitions.ts @@ -98,6 +98,44 @@ export const getCycleArgsSchema = z.object({}).strict(); export const getCorrelationsArgsSchema = z.object({}).strict(); +/** + * v1.39.4 — each tool's argument schema, keyed by name. The record type makes + * a new tool name fail to compile until its schema is listed here. + */ +const COACH_TOOL_ARG_SCHEMAS: Record = { + get_metric_series: getMetricSeriesArgsSchema, + get_glucose_panel: getGlucosePanelArgsSchema, + get_sleep: getSleepArgsSchema, + get_medication_compliance: getMedicationComplianceArgsSchema, + get_labs: getLabsArgsSchema, + get_illness_recovery: getIllnessRecoveryArgsSchema, + get_workouts: getWorkoutsArgsSchema, + get_cycle: getCycleArgsSchema, + get_correlations: getCorrelationsArgsSchema, +}; + +/** + * v1.39.4 — a call's arguments as its schema validates them, or undefined + * when the name is unknown, the JSON does not parse, or the schema refuses + * it. Only this validated form ever rides the tool trace; the model's raw + * argument string never does. + */ +export function parseCoachToolArgs( + name: string, + rawArguments: string, +): Record | undefined { + if (!isCoachToolName(name)) return undefined; + let raw: unknown; + try { + raw = rawArguments.trim() === "" ? {} : JSON.parse(rawArguments); + } catch { + return undefined; + } + const parsed = COACH_TOOL_ARG_SCHEMAS[name].safeParse(raw); + if (!parsed.success) return undefined; + return parsed.data as Record; +} + /** * JSON-Schema parameter shapes handed to the provider. Kept hand-written * (rather than generated from Zod) so the wire description the model reads is diff --git a/src/lib/ai/coach/tools/executor.ts b/src/lib/ai/coach/tools/executor.ts index 099c8e1cb..dbeb060f8 100644 --- a/src/lib/ai/coach/tools/executor.ts +++ b/src/lib/ai/coach/tools/executor.ts @@ -99,6 +99,12 @@ export interface CoachToolResult { export interface CoachToolTrace { name: string; present: boolean; + /** + * v1.39.4 — the call's arguments as their schema validated them; absent + * when they did not validate. Turn-internal (steps, method, chips): never + * persisted, the provenance keeps `{ name, present }` only. + */ + args?: Record; } function pickSection( diff --git a/src/lib/ai/coach/tools/loop.ts b/src/lib/ai/coach/tools/loop.ts index cd671c0b6..f657a25b4 100644 --- a/src/lib/ai/coach/tools/loop.ts +++ b/src/lib/ai/coach/tools/loop.ts @@ -26,9 +26,15 @@ import { annotate } from "@/lib/logging/context"; import { runRawCompletionWithFallback } from "@/lib/ai/provider-runner"; import type { ProviderChainResolved } from "@/lib/ai/provider-runner"; -import type { AiMessage, AiToolDef, CompletionResult } from "@/lib/ai/types"; +import type { + AiMessage, + AiToolCall, + AiToolDef, + CompletionResult, +} from "@/lib/ai/types"; import type { ProviderHealthLedger } from "@/lib/ai/provider-health-ledger"; import type { CoachScope, CoachScopeWindow } from "@/lib/ai/coach/types"; +import { parseCoachToolArgs, type CoachToolName } from "./definitions"; import { executeCoachTool, type CoachToolResult, @@ -81,6 +87,18 @@ export interface CoachToolLoopResult { * one honest sentence the fix exists to make sayable. */ toolResults: CoachToolResult[]; + /** + * v1.39.4 — true when the loop reached `HARD_CAP`: the model asked for + * tools in every round it was offered them, so the final answer was forced + * with `toolChoice: "none"` rather than given freely. + */ + forcedFinal: boolean; +} + +/** v1.39.4 — a tool call to run before the first model round. */ +interface CoachSeedCall { + name: CoachToolName; + args: Record; } export async function runCoachToolLoop(args: { @@ -108,6 +126,26 @@ export async function runCoachToolLoop(args: { * shared default holds. */ timeoutMs?: number; + /** + * v1.39.4 — fired as each tool call starts. `index` counts calls across + * the whole turn from 0, so it is stable for a step id. A throwing callback + * is swallowed: progress reporting never breaks the loop. + */ + onCallStart?: (call: AiToolCall, index: number) => void; + /** + * v1.39.4 — fired once for every call that started, when its result is in: + * found, missed, invalid or failed alike (`executeCoachTool` never throws). + */ + onCallSettled?: ( + call: AiToolCall, + result: CoachToolResult, + index: number, + ) => void; + /** + * v1.39.4 — calls to run before round one, reusing the loop's own + * assistant-call and tool-result serialisation. Accepted, not read yet. + */ + seedCalls?: ReadonlyArray; }): Promise { const { userId, @@ -121,6 +159,8 @@ export async function runCoachToolLoop(args: { ledger, signal, timeoutMs, + onCallStart, + onCallSettled, } = args; const messages: AiMessage[] = [...args.messages]; @@ -130,6 +170,7 @@ export async function runCoachToolLoop(args: { let workingProviderType = ""; const toolTrace: CoachToolTrace[] = []; const toolResults: CoachToolResult[] = []; + let callCount = 0; // Round budget: rounds 1..HARD_CAP. On the last allowed round we forbid tool // calls so the model must answer. @@ -184,6 +225,7 @@ export async function runCoachToolLoop(args: { rounds, toolTrace, toolResults, + forcedFinal: isForcedFinal, }; } @@ -197,6 +239,9 @@ export async function runCoachToolLoop(args: { const results = await Promise.all( calls.map(async (call) => { + const index = callCount; + callCount += 1; + notify(() => onCallStart?.(call, index)); const toolResult = await executeCoachTool({ userId, name: call.name, @@ -204,7 +249,13 @@ export async function runCoachToolLoop(args: { fallbackWindow, sharedScope, }); - toolTrace.push({ name: call.name, present: toolResult.present }); + notify(() => onCallSettled?.(call, toolResult, index)); + const validArgs = parseCoachToolArgs(call.name, call.arguments); + toolTrace.push({ + name: call.name, + present: toolResult.present, + ...(validArgs ? { args: validArgs } : {}), + }); // v1.21.0 (P6) — retain the payloads that carried figures for the // post-hoc prose number-verifier (the union of numeric leaves grounds // the figures the model may cite): a present result's `data`, or the @@ -230,3 +281,12 @@ export async function runCoachToolLoop(args: { // defensive guard. throw new Error("coach tool loop exceeded hard cap without a final answer"); } + +/** Run a progress callback; a throw from it never reaches the loop. */ +function notify(callback: () => void): void { + try { + callback(); + } catch { + // Progress is best-effort; the answer is not. + } +}