mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 20:42:43 +00:00
feat(coach): report tool calls as they start and settle
The retrieval loop takes onCallStart and onCallSettled callbacks, fired for every call with a turn-wide index; a throwing callback is swallowed. The trace carries each call's schema-validated arguments (never the raw string, never persisted), the result says whether the answer was forced at the round cap, and a seedCalls parameter is accepted but not read yet.
This commit is contained in:
1 parent
a8b3e5d7f4
commit
742ef2c9b0
6 files changed
+263
-5
No files matched your search
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* v1.39.4 — the loop's progress seam: every call that starts settles, the
|
||||
* trace keeps only schema-validated arguments, and a forced answer at the
|
||||
* round cap is reported as such.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { CoachToolResult } from "@/lib/ai/coach/tools/executor";
|
||||
|
||||
const executeCoachTool = vi.fn<(args?: unknown) => Promise<CoachToolResult>>();
|
||||
vi.mock("@/lib/ai/coach/tools/executor", () => ({
|
||||
executeCoachTool: (args: unknown) => executeCoachTool(args),
|
||||
}));
|
||||
const runRawCompletionWithFallback = vi.fn();
|
||||
vi.mock("@/lib/ai/provider-runner", () => ({
|
||||
runRawCompletionWithFallback: (args: unknown) =>
|
||||
runRawCompletionWithFallback(args),
|
||||
}));
|
||||
|
||||
import { HARD_CAP, runCoachToolLoop } from "@/lib/ai/coach/tools/loop";
|
||||
import { COACH_TOOL_DEFS } from "@/lib/ai/coach/tools/definitions";
|
||||
|
||||
type Call = { id: string; name: string; arguments: string };
|
||||
|
||||
function round(content: string, toolCalls?: Call[]) {
|
||||
return {
|
||||
result: {
|
||||
content,
|
||||
tokensUsed: 5,
|
||||
model: "mock",
|
||||
providerType: "anthropic" as const,
|
||||
...(toolCalls
|
||||
? { toolCalls, finishReason: "tool_calls" as const }
|
||||
: { finishReason: "stop" as const }),
|
||||
},
|
||||
workingProvider: { providerType: "anthropic" },
|
||||
fallbackHops: [],
|
||||
};
|
||||
}
|
||||
|
||||
const baseArgs = {
|
||||
userId: "u1",
|
||||
providers: [],
|
||||
system: "sys",
|
||||
messages: [{ role: "user" as const, content: "q" }],
|
||||
tools: COACH_TOOL_DEFS,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
executeCoachTool.mockReset();
|
||||
runRawCompletionWithFallback.mockReset();
|
||||
});
|
||||
|
||||
describe("runCoachToolLoop — progress callbacks", () => {
|
||||
it("settles every call that started, misses and invalid calls included, with turn-wide indexes", async () => {
|
||||
executeCoachTool
|
||||
.mockResolvedValueOnce({ present: true, data: {} })
|
||||
.mockResolvedValueOnce({ present: false, reason: "invalid_arguments" })
|
||||
.mockResolvedValueOnce({ present: false, reason: "unknown_tool" });
|
||||
runRawCompletionWithFallback
|
||||
.mockResolvedValueOnce(
|
||||
round("", [
|
||||
{ id: "a", name: "get_metric_series", arguments: '{"metric":"bp"}' },
|
||||
{ id: "b", name: "get_sleep", arguments: "{not json" },
|
||||
]),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
round("", [{ id: "c", name: "get_everything", arguments: "{}" }]),
|
||||
)
|
||||
.mockResolvedValueOnce(round("answer"));
|
||||
|
||||
const started: Array<[string, number]> = [];
|
||||
const settled: Array<[string, boolean, number]> = [];
|
||||
const out = await runCoachToolLoop({
|
||||
...baseArgs,
|
||||
onCallStart: (call, index) => started.push([call.id, index]),
|
||||
onCallSettled: (call, result, index) =>
|
||||
settled.push([call.id, result.present, index]),
|
||||
});
|
||||
|
||||
expect(started).toEqual([
|
||||
["a", 0],
|
||||
["b", 1],
|
||||
["c", 2],
|
||||
]);
|
||||
expect([...settled].sort((x, y) => x[2] - y[2])).toEqual([
|
||||
["a", true, 0],
|
||||
["b", false, 1],
|
||||
["c", false, 2],
|
||||
]);
|
||||
expect(out.forcedFinal).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps only schema-validated arguments on the trace", async () => {
|
||||
executeCoachTool.mockResolvedValue({ present: false });
|
||||
runRawCompletionWithFallback
|
||||
.mockResolvedValueOnce(
|
||||
round("", [
|
||||
{ id: "a", name: "get_metric_series", arguments: '{"metric":"bp"}' },
|
||||
// Unknown key: the strict schema refuses it.
|
||||
{
|
||||
id: "b",
|
||||
name: "get_metric_series",
|
||||
arguments: '{"metric":"bp","userId":"someone-else"}',
|
||||
},
|
||||
{ id: "c", name: "get_sleep", arguments: "{not json" },
|
||||
]),
|
||||
)
|
||||
.mockResolvedValueOnce(round("answer"));
|
||||
|
||||
const out = await runCoachToolLoop(baseArgs);
|
||||
const byArgs = out.toolTrace.map((t) => t.args ?? null);
|
||||
expect(byArgs).toContainEqual({ metric: "bp" });
|
||||
expect(byArgs.filter((a) => a === null)).toHaveLength(2);
|
||||
expect(JSON.stringify(out.toolTrace)).not.toContain("someone-else");
|
||||
});
|
||||
|
||||
it("a throwing callback never breaks the turn", async () => {
|
||||
executeCoachTool.mockResolvedValue({ present: true, data: {} });
|
||||
runRawCompletionWithFallback
|
||||
.mockResolvedValueOnce(
|
||||
round("", [{ id: "a", name: "get_sleep", arguments: "{}" }]),
|
||||
)
|
||||
.mockResolvedValueOnce(round("answer"));
|
||||
const out = await runCoachToolLoop({
|
||||
...baseArgs,
|
||||
onCallStart: () => {
|
||||
throw new Error("closed stream");
|
||||
},
|
||||
onCallSettled: () => {
|
||||
throw new Error("closed stream");
|
||||
},
|
||||
});
|
||||
expect(out.result.content).toBe("answer");
|
||||
expect(out.toolTrace).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("reports a forced answer at the round cap", async () => {
|
||||
executeCoachTool.mockResolvedValue({ present: false });
|
||||
runRawCompletionWithFallback.mockImplementation(
|
||||
(args: { params: { toolChoice?: string } }) =>
|
||||
Promise.resolve(
|
||||
args.params.toolChoice === "none"
|
||||
? round("forced")
|
||||
: round("", [{ id: "x", name: "get_sleep", arguments: "{}" }]),
|
||||
),
|
||||
);
|
||||
const out = await runCoachToolLoop(baseArgs);
|
||||
expect(out.rounds).toBe(HARD_CAP);
|
||||
expect(out.forcedFinal).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -79,7 +79,9 @@ describe("coach tool loop — which results reach the grounding guard", () => {
|
||||
const loop = await runCoachToolLoop(baseArgs);
|
||||
|
||||
// The trace records that the tool ran and found nothing…
|
||||
expect(loop.toolTrace).toEqual([{ name: "get_sleep", present: false }]);
|
||||
expect(loop.toolTrace).toEqual([
|
||||
{ name: "get_sleep", present: false, args: {} },
|
||||
]);
|
||||
// …but the payload set the route grades against is empty, so the route's
|
||||
// `activatingPayloads.length > 0` gate is false and the numeric grounding
|
||||
// guard never runs for this turn.
|
||||
|
||||
@@ -88,7 +88,7 @@ describe("runCoachToolLoop", () => {
|
||||
expect(out.result.content).toBe("Your BP looks steady.");
|
||||
expect(out.totalTokens).toBe(50); // summed across both rounds
|
||||
expect(out.toolTrace).toEqual([
|
||||
{ name: "get_metric_series", present: true },
|
||||
{ name: "get_metric_series", present: true, args: { metric: "bp" } },
|
||||
]);
|
||||
// v1.21.0 (P6) — the present result's payload is retained for the prose
|
||||
// number-verifier.
|
||||
@@ -278,7 +278,7 @@ describe("shared snapshot scope threading (D5-1)", () => {
|
||||
expect(out.toolResults).toHaveLength(1);
|
||||
expect(out.toolResults[0].available).toMatchObject({ count: 1597 });
|
||||
expect(out.toolTrace).toEqual([
|
||||
{ name: "get_glucose_panel", present: false },
|
||||
{ name: "get_glucose_panel", present: false, args: {} },
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -98,6 +98,44 @@ export const getCycleArgsSchema = z.object({}).strict();
|
||||
|
||||
export const getCorrelationsArgsSchema = z.object({}).strict();
|
||||
|
||||
/**
|
||||
* v1.39.4 — each tool's argument schema, keyed by name. The record type makes
|
||||
* a new tool name fail to compile until its schema is listed here.
|
||||
*/
|
||||
const COACH_TOOL_ARG_SCHEMAS: Record<CoachToolName, z.ZodType> = {
|
||||
get_metric_series: getMetricSeriesArgsSchema,
|
||||
get_glucose_panel: getGlucosePanelArgsSchema,
|
||||
get_sleep: getSleepArgsSchema,
|
||||
get_medication_compliance: getMedicationComplianceArgsSchema,
|
||||
get_labs: getLabsArgsSchema,
|
||||
get_illness_recovery: getIllnessRecoveryArgsSchema,
|
||||
get_workouts: getWorkoutsArgsSchema,
|
||||
get_cycle: getCycleArgsSchema,
|
||||
get_correlations: getCorrelationsArgsSchema,
|
||||
};
|
||||
|
||||
/**
|
||||
* v1.39.4 — a call's arguments as its schema validates them, or undefined
|
||||
* when the name is unknown, the JSON does not parse, or the schema refuses
|
||||
* it. Only this validated form ever rides the tool trace; the model's raw
|
||||
* argument string never does.
|
||||
*/
|
||||
export function parseCoachToolArgs(
|
||||
name: string,
|
||||
rawArguments: string,
|
||||
): Record<string, unknown> | undefined {
|
||||
if (!isCoachToolName(name)) return undefined;
|
||||
let raw: unknown;
|
||||
try {
|
||||
raw = rawArguments.trim() === "" ? {} : JSON.parse(rawArguments);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = COACH_TOOL_ARG_SCHEMAS[name].safeParse(raw);
|
||||
if (!parsed.success) return undefined;
|
||||
return parsed.data as Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON-Schema parameter shapes handed to the provider. Kept hand-written
|
||||
* (rather than generated from Zod) so the wire description the model reads is
|
||||
|
||||
@@ -99,6 +99,12 @@ export interface CoachToolResult {
|
||||
export interface CoachToolTrace {
|
||||
name: string;
|
||||
present: boolean;
|
||||
/**
|
||||
* v1.39.4 — the call's arguments as their schema validated them; absent
|
||||
* when they did not validate. Turn-internal (steps, method, chips): never
|
||||
* persisted, the provenance keeps `{ name, present }` only.
|
||||
*/
|
||||
args?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
function pickSection(
|
||||
|
||||
@@ -26,9 +26,15 @@
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { runRawCompletionWithFallback } from "@/lib/ai/provider-runner";
|
||||
import type { ProviderChainResolved } from "@/lib/ai/provider-runner";
|
||||
import type { AiMessage, AiToolDef, CompletionResult } from "@/lib/ai/types";
|
||||
import type {
|
||||
AiMessage,
|
||||
AiToolCall,
|
||||
AiToolDef,
|
||||
CompletionResult,
|
||||
} from "@/lib/ai/types";
|
||||
import type { ProviderHealthLedger } from "@/lib/ai/provider-health-ledger";
|
||||
import type { CoachScope, CoachScopeWindow } from "@/lib/ai/coach/types";
|
||||
import { parseCoachToolArgs, type CoachToolName } from "./definitions";
|
||||
import {
|
||||
executeCoachTool,
|
||||
type CoachToolResult,
|
||||
@@ -81,6 +87,18 @@ export interface CoachToolLoopResult {
|
||||
* one honest sentence the fix exists to make sayable.
|
||||
*/
|
||||
toolResults: CoachToolResult[];
|
||||
/**
|
||||
* v1.39.4 — true when the loop reached `HARD_CAP`: the model asked for
|
||||
* tools in every round it was offered them, so the final answer was forced
|
||||
* with `toolChoice: "none"` rather than given freely.
|
||||
*/
|
||||
forcedFinal: boolean;
|
||||
}
|
||||
|
||||
/** v1.39.4 — a tool call to run before the first model round. */
|
||||
interface CoachSeedCall {
|
||||
name: CoachToolName;
|
||||
args: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export async function runCoachToolLoop(args: {
|
||||
@@ -108,6 +126,26 @@ export async function runCoachToolLoop(args: {
|
||||
* shared default holds.
|
||||
*/
|
||||
timeoutMs?: number;
|
||||
/**
|
||||
* v1.39.4 — fired as each tool call starts. `index` counts calls across
|
||||
* the whole turn from 0, so it is stable for a step id. A throwing callback
|
||||
* is swallowed: progress reporting never breaks the loop.
|
||||
*/
|
||||
onCallStart?: (call: AiToolCall, index: number) => void;
|
||||
/**
|
||||
* v1.39.4 — fired once for every call that started, when its result is in:
|
||||
* found, missed, invalid or failed alike (`executeCoachTool` never throws).
|
||||
*/
|
||||
onCallSettled?: (
|
||||
call: AiToolCall,
|
||||
result: CoachToolResult,
|
||||
index: number,
|
||||
) => void;
|
||||
/**
|
||||
* v1.39.4 — calls to run before round one, reusing the loop's own
|
||||
* assistant-call and tool-result serialisation. Accepted, not read yet.
|
||||
*/
|
||||
seedCalls?: ReadonlyArray<CoachSeedCall>;
|
||||
}): Promise<CoachToolLoopResult> {
|
||||
const {
|
||||
userId,
|
||||
@@ -121,6 +159,8 @@ export async function runCoachToolLoop(args: {
|
||||
ledger,
|
||||
signal,
|
||||
timeoutMs,
|
||||
onCallStart,
|
||||
onCallSettled,
|
||||
} = args;
|
||||
|
||||
const messages: AiMessage[] = [...args.messages];
|
||||
@@ -130,6 +170,7 @@ export async function runCoachToolLoop(args: {
|
||||
let workingProviderType = "";
|
||||
const toolTrace: CoachToolTrace[] = [];
|
||||
const toolResults: CoachToolResult[] = [];
|
||||
let callCount = 0;
|
||||
|
||||
// Round budget: rounds 1..HARD_CAP. On the last allowed round we forbid tool
|
||||
// calls so the model must answer.
|
||||
@@ -184,6 +225,7 @@ export async function runCoachToolLoop(args: {
|
||||
rounds,
|
||||
toolTrace,
|
||||
toolResults,
|
||||
forcedFinal: isForcedFinal,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -197,6 +239,9 @@ export async function runCoachToolLoop(args: {
|
||||
|
||||
const results = await Promise.all(
|
||||
calls.map(async (call) => {
|
||||
const index = callCount;
|
||||
callCount += 1;
|
||||
notify(() => onCallStart?.(call, index));
|
||||
const toolResult = await executeCoachTool({
|
||||
userId,
|
||||
name: call.name,
|
||||
@@ -204,7 +249,13 @@ export async function runCoachToolLoop(args: {
|
||||
fallbackWindow,
|
||||
sharedScope,
|
||||
});
|
||||
toolTrace.push({ name: call.name, present: toolResult.present });
|
||||
notify(() => onCallSettled?.(call, toolResult, index));
|
||||
const validArgs = parseCoachToolArgs(call.name, call.arguments);
|
||||
toolTrace.push({
|
||||
name: call.name,
|
||||
present: toolResult.present,
|
||||
...(validArgs ? { args: validArgs } : {}),
|
||||
});
|
||||
// v1.21.0 (P6) — retain the payloads that carried figures for the
|
||||
// post-hoc prose number-verifier (the union of numeric leaves grounds
|
||||
// the figures the model may cite): a present result's `data`, or the
|
||||
@@ -230,3 +281,12 @@ export async function runCoachToolLoop(args: {
|
||||
// defensive guard.
|
||||
throw new Error("coach tool loop exceeded hard cap without a final answer");
|
||||
}
|
||||
|
||||
/** Run a progress callback; a throw from it never reaches the loop. */
|
||||
function notify(callback: () => void): void {
|
||||
try {
|
||||
callback();
|
||||
} catch {
|
||||
// Progress is best-effort; the answer is not.
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user