feat(coach): report tool calls as they start and settle

The retrieval loop takes onCallStart and onCallSettled callbacks, fired
for every call with a turn-wide index; a throwing callback is swallowed.
The trace carries each call's schema-validated arguments (never the raw
string, never persisted), the result says whether the answer was forced
at the round cap, and a seedCalls parameter is accepted but not read yet.
This commit is contained in:
Marc-André Bombeck committed 2026-09-27 10:37:40 +02:00
1 parent a8b3e5d7f4
commit 742ef2c9b0
6 files changed
+263 -5

No files matched your search

@@ -0,0 +1,152 @@
/**
* v1.39.4 — the loop's progress seam: every call that starts settles, the
* trace keeps only schema-validated arguments, and a forced answer at the
* round cap is reported as such.
*/
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { CoachToolResult } from "@/lib/ai/coach/tools/executor";
const executeCoachTool = vi.fn<(args?: unknown) => Promise<CoachToolResult>>();
vi.mock("@/lib/ai/coach/tools/executor", () => ({
executeCoachTool: (args: unknown) => executeCoachTool(args),
}));
const runRawCompletionWithFallback = vi.fn();
vi.mock("@/lib/ai/provider-runner", () => ({
runRawCompletionWithFallback: (args: unknown) =>
runRawCompletionWithFallback(args),
}));
import { HARD_CAP, runCoachToolLoop } from "@/lib/ai/coach/tools/loop";
import { COACH_TOOL_DEFS } from "@/lib/ai/coach/tools/definitions";
type Call = { id: string; name: string; arguments: string };
function round(content: string, toolCalls?: Call[]) {
return {
result: {
content,
tokensUsed: 5,
model: "mock",
providerType: "anthropic" as const,
...(toolCalls
? { toolCalls, finishReason: "tool_calls" as const }
: { finishReason: "stop" as const }),
},
workingProvider: { providerType: "anthropic" },
fallbackHops: [],
};
}
const baseArgs = {
userId: "u1",
providers: [],
system: "sys",
messages: [{ role: "user" as const, content: "q" }],
tools: COACH_TOOL_DEFS,
};
beforeEach(() => {
executeCoachTool.mockReset();
runRawCompletionWithFallback.mockReset();
});
describe("runCoachToolLoop — progress callbacks", () => {
it("settles every call that started, misses and invalid calls included, with turn-wide indexes", async () => {
executeCoachTool
.mockResolvedValueOnce({ present: true, data: {} })
.mockResolvedValueOnce({ present: false, reason: "invalid_arguments" })
.mockResolvedValueOnce({ present: false, reason: "unknown_tool" });
runRawCompletionWithFallback
.mockResolvedValueOnce(
round("", [
{ id: "a", name: "get_metric_series", arguments: '{"metric":"bp"}' },
{ id: "b", name: "get_sleep", arguments: "{not json" },
]),
)
.mockResolvedValueOnce(
round("", [{ id: "c", name: "get_everything", arguments: "{}" }]),
)
.mockResolvedValueOnce(round("answer"));
const started: Array<[string, number]> = [];
const settled: Array<[string, boolean, number]> = [];
const out = await runCoachToolLoop({
...baseArgs,
onCallStart: (call, index) => started.push([call.id, index]),
onCallSettled: (call, result, index) =>
settled.push([call.id, result.present, index]),
});
expect(started).toEqual([
["a", 0],
["b", 1],
["c", 2],
]);
expect([...settled].sort((x, y) => x[2] - y[2])).toEqual([
["a", true, 0],
["b", false, 1],
["c", false, 2],
]);
expect(out.forcedFinal).toBe(false);
});
it("keeps only schema-validated arguments on the trace", async () => {
executeCoachTool.mockResolvedValue({ present: false });
runRawCompletionWithFallback
.mockResolvedValueOnce(
round("", [
{ id: "a", name: "get_metric_series", arguments: '{"metric":"bp"}' },
// Unknown key: the strict schema refuses it.
{
id: "b",
name: "get_metric_series",
arguments: '{"metric":"bp","userId":"someone-else"}',
},
{ id: "c", name: "get_sleep", arguments: "{not json" },
]),
)
.mockResolvedValueOnce(round("answer"));
const out = await runCoachToolLoop(baseArgs);
const byArgs = out.toolTrace.map((t) => t.args ?? null);
expect(byArgs).toContainEqual({ metric: "bp" });
expect(byArgs.filter((a) => a === null)).toHaveLength(2);
expect(JSON.stringify(out.toolTrace)).not.toContain("someone-else");
});
it("a throwing callback never breaks the turn", async () => {
executeCoachTool.mockResolvedValue({ present: true, data: {} });
runRawCompletionWithFallback
.mockResolvedValueOnce(
round("", [{ id: "a", name: "get_sleep", arguments: "{}" }]),
)
.mockResolvedValueOnce(round("answer"));
const out = await runCoachToolLoop({
...baseArgs,
onCallStart: () => {
throw new Error("closed stream");
},
onCallSettled: () => {
throw new Error("closed stream");
},
});
expect(out.result.content).toBe("answer");
expect(out.toolTrace).toHaveLength(1);
});
it("reports a forced answer at the round cap", async () => {
executeCoachTool.mockResolvedValue({ present: false });
runRawCompletionWithFallback.mockImplementation(
(args: { params: { toolChoice?: string } }) =>
Promise.resolve(
args.params.toolChoice === "none"
? round("forced")
: round("", [{ id: "x", name: "get_sleep", arguments: "{}" }]),
),
);
const out = await runCoachToolLoop(baseArgs);
expect(out.rounds).toBe(HARD_CAP);
expect(out.forcedFinal).toBe(true);
});
});
@@ -79,7 +79,9 @@ describe("coach tool loop — which results reach the grounding guard", () => {
const loop = await runCoachToolLoop(baseArgs);
// The trace records that the tool ran and found nothing…
expect(loop.toolTrace).toEqual([{ name: "get_sleep", present: false }]);
expect(loop.toolTrace).toEqual([
{ name: "get_sleep", present: false, args: {} },
]);
// …but the payload set the route grades against is empty, so the route's
// `activatingPayloads.length > 0` gate is false and the numeric grounding
// guard never runs for this turn.
@@ -88,7 +88,7 @@ describe("runCoachToolLoop", () => {
expect(out.result.content).toBe("Your BP looks steady.");
expect(out.totalTokens).toBe(50); // summed across both rounds
expect(out.toolTrace).toEqual([
{ name: "get_metric_series", present: true },
{ name: "get_metric_series", present: true, args: { metric: "bp" } },
]);
// v1.21.0 (P6) — the present result's payload is retained for the prose
// number-verifier.
@@ -278,7 +278,7 @@ describe("shared snapshot scope threading (D5-1)", () => {
expect(out.toolResults).toHaveLength(1);
expect(out.toolResults[0].available).toMatchObject({ count: 1597 });
expect(out.toolTrace).toEqual([
{ name: "get_glucose_panel", present: false },
{ name: "get_glucose_panel", present: false, args: {} },
]);
});
+38
View File
@@ -98,6 +98,44 @@ export const getCycleArgsSchema = z.object({}).strict();
export const getCorrelationsArgsSchema = z.object({}).strict();
/**
* v1.39.4 — each tool's argument schema, keyed by name. The record type makes
* a new tool name fail to compile until its schema is listed here.
*/
const COACH_TOOL_ARG_SCHEMAS: Record<CoachToolName, z.ZodType> = {
get_metric_series: getMetricSeriesArgsSchema,
get_glucose_panel: getGlucosePanelArgsSchema,
get_sleep: getSleepArgsSchema,
get_medication_compliance: getMedicationComplianceArgsSchema,
get_labs: getLabsArgsSchema,
get_illness_recovery: getIllnessRecoveryArgsSchema,
get_workouts: getWorkoutsArgsSchema,
get_cycle: getCycleArgsSchema,
get_correlations: getCorrelationsArgsSchema,
};
/**
* v1.39.4 — a call's arguments as its schema validates them, or undefined
* when the name is unknown, the JSON does not parse, or the schema refuses
* it. Only this validated form ever rides the tool trace; the model's raw
* argument string never does.
*/
export function parseCoachToolArgs(
name: string,
rawArguments: string,
): Record<string, unknown> | undefined {
if (!isCoachToolName(name)) return undefined;
let raw: unknown;
try {
raw = rawArguments.trim() === "" ? {} : JSON.parse(rawArguments);
} catch {
return undefined;
}
const parsed = COACH_TOOL_ARG_SCHEMAS[name].safeParse(raw);
if (!parsed.success) return undefined;
return parsed.data as Record<string, unknown>;
}
/**
* JSON-Schema parameter shapes handed to the provider. Kept hand-written
* (rather than generated from Zod) so the wire description the model reads is
+6
View File
@@ -99,6 +99,12 @@ export interface CoachToolResult {
export interface CoachToolTrace {
name: string;
present: boolean;
/**
* v1.39.4 — the call's arguments as their schema validated them; absent
* when they did not validate. Turn-internal (steps, method, chips): never
* persisted, the provenance keeps `{ name, present }` only.
*/
args?: Record<string, unknown>;
}
function pickSection(
+62 -2
View File
@@ -26,9 +26,15 @@
import { annotate } from "@/lib/logging/context";
import { runRawCompletionWithFallback } from "@/lib/ai/provider-runner";
import type { ProviderChainResolved } from "@/lib/ai/provider-runner";
import type { AiMessage, AiToolDef, CompletionResult } from "@/lib/ai/types";
import type {
AiMessage,
AiToolCall,
AiToolDef,
CompletionResult,
} from "@/lib/ai/types";
import type { ProviderHealthLedger } from "@/lib/ai/provider-health-ledger";
import type { CoachScope, CoachScopeWindow } from "@/lib/ai/coach/types";
import { parseCoachToolArgs, type CoachToolName } from "./definitions";
import {
executeCoachTool,
type CoachToolResult,
@@ -81,6 +87,18 @@ export interface CoachToolLoopResult {
* one honest sentence the fix exists to make sayable.
*/
toolResults: CoachToolResult[];
/**
* v1.39.4 — true when the loop reached `HARD_CAP`: the model asked for
* tools in every round it was offered them, so the final answer was forced
* with `toolChoice: "none"` rather than given freely.
*/
forcedFinal: boolean;
}
/** v1.39.4 — a tool call to run before the first model round. */
interface CoachSeedCall {
name: CoachToolName;
args: Record<string, unknown>;
}
export async function runCoachToolLoop(args: {
@@ -108,6 +126,26 @@ export async function runCoachToolLoop(args: {
* shared default holds.
*/
timeoutMs?: number;
/**
* v1.39.4 — fired as each tool call starts. `index` counts calls across
* the whole turn from 0, so it is stable for a step id. A throwing callback
* is swallowed: progress reporting never breaks the loop.
*/
onCallStart?: (call: AiToolCall, index: number) => void;
/**
* v1.39.4 — fired once for every call that started, when its result is in:
* found, missed, invalid or failed alike (`executeCoachTool` never throws).
*/
onCallSettled?: (
call: AiToolCall,
result: CoachToolResult,
index: number,
) => void;
/**
* v1.39.4 — calls to run before round one, reusing the loop's own
* assistant-call and tool-result serialisation. Accepted, not read yet.
*/
seedCalls?: ReadonlyArray<CoachSeedCall>;
}): Promise<CoachToolLoopResult> {
const {
userId,
@@ -121,6 +159,8 @@ export async function runCoachToolLoop(args: {
ledger,
signal,
timeoutMs,
onCallStart,
onCallSettled,
} = args;
const messages: AiMessage[] = [...args.messages];
@@ -130,6 +170,7 @@ export async function runCoachToolLoop(args: {
let workingProviderType = "";
const toolTrace: CoachToolTrace[] = [];
const toolResults: CoachToolResult[] = [];
let callCount = 0;
// Round budget: rounds 1..HARD_CAP. On the last allowed round we forbid tool
// calls so the model must answer.
@@ -184,6 +225,7 @@ export async function runCoachToolLoop(args: {
rounds,
toolTrace,
toolResults,
forcedFinal: isForcedFinal,
};
}
@@ -197,6 +239,9 @@ export async function runCoachToolLoop(args: {
const results = await Promise.all(
calls.map(async (call) => {
const index = callCount;
callCount += 1;
notify(() => onCallStart?.(call, index));
const toolResult = await executeCoachTool({
userId,
name: call.name,
@@ -204,7 +249,13 @@ export async function runCoachToolLoop(args: {
fallbackWindow,
sharedScope,
});
toolTrace.push({ name: call.name, present: toolResult.present });
notify(() => onCallSettled?.(call, toolResult, index));
const validArgs = parseCoachToolArgs(call.name, call.arguments);
toolTrace.push({
name: call.name,
present: toolResult.present,
...(validArgs ? { args: validArgs } : {}),
});
// v1.21.0 (P6) — retain the payloads that carried figures for the
// post-hoc prose number-verifier (the union of numeric leaves grounds
// the figures the model may cite): a present result's `data`, or the
@@ -230,3 +281,12 @@ export async function runCoachToolLoop(args: {
// defensive guard.
throw new Error("coach tool loop exceeded hard cap without a final answer");
}
/** Run a progress callback; a throw from it never reaches the loop. */
function notify(callback: () => void): void {
try {
callback();
} catch {
// Progress is best-effort; the answer is not.
}
}