mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-12 22:26:13 +00:00
Marc-André Bombeck
8e74bc05c8
Unit tests for the turn budget and the no-progress brake, the budgeted loop (every stop reason, the round-by-round ledger, repeated calls, questions, comparisons, reasoning and provider state), the trail recorder and its screen, clarifications through the tool and the brake, the comparison table and chart, the trail column and its owner-only read, the reply frames, and the turn's frames end to end through the route. An integration test proves the trail against a real database. Eval scenarios cover a multi-round why question, a period comparison with its chart, a question asked through the tool, and keep the charts the existing table scenarios already showed. Take the effective reasoning level from the route rather than resolving it in the turn; the turn only caps it when the operator pays.
452 lines
15 KiB
TypeScript
452 lines
15 KiB
TypeScript
/**
|
|
* v1.41 — the budgeted loop: every stop reason forces a final round without
|
|
* tools and with the reason line, the ledger is reserved and settled round
|
|
* by round, a repeated call is never run, a question ends the turn, the
|
|
* reasoning level and the provider state reach the provider, and the trail
|
|
* opens its thinking entry before the provider is called.
|
|
*/
|
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
|
|
import type { CoachToolResult } from "@/lib/ai/coach/tools/executor";
|
|
|
|
const executeCoachTool = vi.fn<(args?: unknown) => Promise<CoachToolResult>>();
|
|
vi.mock("@/lib/ai/coach/tools/executor", () => ({
|
|
executeCoachTool: (args: unknown) => executeCoachTool(args),
|
|
}));
|
|
const runRawCompletionWithFallback = vi.fn();
|
|
vi.mock("@/lib/ai/provider-runner", () => ({
|
|
runRawCompletionWithFallback: (args: unknown) =>
|
|
runRawCompletionWithFallback(args),
|
|
}));
|
|
const runDialogTool = vi.fn();
|
|
vi.mock("@/lib/ai/coach/tools/dialog-tools", async (importOriginal) => ({
|
|
...(await importOriginal<object>()),
|
|
runDialogTool: (args: unknown) => runDialogTool(args),
|
|
}));
|
|
const runCompareSeries = vi.fn();
|
|
vi.mock("@/lib/ai/coach/tools/compare-series", async (importOriginal) => ({
|
|
...(await importOriginal<object>()),
|
|
runCompareSeries: (args: unknown) => runCompareSeries(args),
|
|
}));
|
|
|
|
import { runCoachToolLoop, type RoundSpend } from "@/lib/ai/coach/tools/loop";
|
|
import { createTurnBudget } from "@/lib/ai/coach/tools/turn-budget";
|
|
import { COACH_TOOL_DEFS } from "@/lib/ai/coach/tools/definitions";
|
|
import { UNBOUNDED_REACH } from "@/lib/ai/coach/history-reach";
|
|
import { createActivityRecorder } from "@/lib/ai/coach/activity/recorder";
|
|
import type { CoachActivity, CoachClarification } from "@/lib/ai/coach/types";
|
|
|
|
type Call = { id: string; name: string; arguments: string };
|
|
|
|
function round(
|
|
content: string,
|
|
toolCalls?: Call[],
|
|
extra: Record<string, unknown> = {},
|
|
) {
|
|
return {
|
|
result: {
|
|
content,
|
|
tokensUsed: 100,
|
|
model: "mock",
|
|
providerType: "codex" as const,
|
|
...(toolCalls
|
|
? { toolCalls, finishReason: "tool_calls" as const }
|
|
: { finishReason: "stop" as const }),
|
|
...extra,
|
|
},
|
|
workingProvider: { providerType: "codex" },
|
|
fallbackHops: [],
|
|
};
|
|
}
|
|
|
|
const table = (metric: string, window = "last30days"): Call => ({
|
|
id: `${metric}-${window}`,
|
|
name: "get_metric_table",
|
|
arguments: JSON.stringify({ metric, window }),
|
|
});
|
|
|
|
const roomy = () =>
|
|
createTurnBudget({
|
|
payer: "user",
|
|
initialInputTokens: 10,
|
|
limits: { tokens: 1e9, wallMs: 1e9, maxRounds: 12 },
|
|
});
|
|
|
|
const baseArgs = {
|
|
userId: "u1",
|
|
providers: [],
|
|
system: "sys",
|
|
messages: [{ role: "user" as const, content: "why?" }],
|
|
tools: COACH_TOOL_DEFS,
|
|
reach: UNBOUNDED_REACH,
|
|
};
|
|
|
|
function spend(admit: boolean[] = []): RoundSpend & {
|
|
reserveRound: ReturnType<typeof vi.fn>;
|
|
settleRound: ReturnType<typeof vi.fn>;
|
|
} {
|
|
const reserveRound = vi.fn(async () => admit.shift() ?? true);
|
|
const settleRound = vi.fn(async () => {});
|
|
return { reserveRound, settleRound };
|
|
}
|
|
|
|
function lastParams() {
|
|
const calls = runRawCompletionWithFallback.mock.calls;
|
|
return calls[calls.length - 1][0].params as {
|
|
system: string;
|
|
toolChoice?: string;
|
|
tools?: unknown[];
|
|
};
|
|
}
|
|
|
|
beforeEach(() => {
|
|
executeCoachTool.mockReset();
|
|
runRawCompletionWithFallback.mockReset();
|
|
runDialogTool.mockReset();
|
|
runCompareSeries.mockReset();
|
|
executeCoachTool.mockResolvedValue({ present: true, data: { mean: 1 } });
|
|
});
|
|
|
|
describe("stop reasons", () => {
|
|
it("answers from the reserve when the token budget runs out", async () => {
|
|
runRawCompletionWithFallback
|
|
.mockResolvedValueOnce(round("", [table("bp")]))
|
|
.mockResolvedValueOnce(round("Answer from what I have."));
|
|
const out = await runCoachToolLoop({
|
|
...baseArgs,
|
|
budget: createTurnBudget({
|
|
payer: "operator",
|
|
initialInputTokens: 10,
|
|
limits: { tokens: 500, wallMs: 1e9, maxRounds: 12 },
|
|
}),
|
|
});
|
|
expect(out.stop).toEqual({ reason: "budget", rounds: 2 });
|
|
expect(out.forcedFinal).toBe(true);
|
|
expect(lastParams().toolChoice).toBe("none");
|
|
expect(lastParams().tools).toBeUndefined();
|
|
expect(lastParams().system).toMatch(
|
|
/FINAL ROUND: this turn's token budget/,
|
|
);
|
|
expect(out.result.content).toBe("Answer from what I have.");
|
|
});
|
|
|
|
it("answers when the wall time would run out", async () => {
|
|
let now = 0;
|
|
runRawCompletionWithFallback.mockImplementation(async () => {
|
|
now += 50_000;
|
|
return lastParamsSafe() === "none"
|
|
? round("late answer")
|
|
: round("", [table("bp")]);
|
|
});
|
|
function lastParamsSafe() {
|
|
const calls = runRawCompletionWithFallback.mock.calls;
|
|
return calls.length > 0
|
|
? (calls[calls.length - 1][0].params.toolChoice as string)
|
|
: "auto";
|
|
}
|
|
const out = await runCoachToolLoop({
|
|
...baseArgs,
|
|
budget: createTurnBudget({
|
|
payer: "operator",
|
|
initialInputTokens: 10,
|
|
now: () => now,
|
|
limits: { tokens: 1e9, wallMs: 90_000, maxRounds: 12 },
|
|
}),
|
|
});
|
|
expect(out.stop?.reason).toBe("time");
|
|
expect(out.forcedFinal).toBe(true);
|
|
});
|
|
|
|
it("answers when the daily ledger refuses the next round", async () => {
|
|
runRawCompletionWithFallback
|
|
.mockResolvedValueOnce(round("", [table("bp")]))
|
|
.mockResolvedValueOnce(round("", [table("weight")]))
|
|
.mockResolvedValueOnce(round("Ledger closed, here is what I have."));
|
|
const ledger = spend([true, false]);
|
|
const out = await runCoachToolLoop({
|
|
...baseArgs,
|
|
budget: roomy(),
|
|
spend: ledger,
|
|
});
|
|
expect(out.stop).toEqual({ reason: "budget", rounds: 3 });
|
|
expect(ledger.reserveRound).toHaveBeenCalledTimes(2);
|
|
// Three rounds settled; only the last against the final reserve.
|
|
expect(ledger.settleRound.mock.calls.map((c) => c[0].final)).toEqual([
|
|
false,
|
|
false,
|
|
true,
|
|
]);
|
|
expect(ledger.settleRound.mock.calls[0][0]).toMatchObject({
|
|
tokens: 100,
|
|
cachedTokens: 0,
|
|
servedBy: "codex",
|
|
});
|
|
});
|
|
|
|
it("does not offer to keep looking after a no-progress stop", async () => {
|
|
executeCoachTool.mockResolvedValue({ present: false, reason: "no_data" });
|
|
runRawCompletionWithFallback
|
|
.mockResolvedValueOnce(round("", [table("bp")]))
|
|
.mockResolvedValueOnce(round("", [table("weight")]))
|
|
.mockResolvedValueOnce(round("Nothing recorded for those."));
|
|
const out = await runCoachToolLoop({ ...baseArgs, budget: roomy() });
|
|
expect(out.stop).toEqual({ reason: "no_progress", rounds: 3 });
|
|
expect(out.forcedFinal).toBe(false);
|
|
expect(lastParams().system).toMatch(/brought nothing new/);
|
|
});
|
|
});
|
|
|
|
describe("repeated calls", () => {
|
|
it("never runs a call twice and tells the model which call it repeats", async () => {
|
|
runRawCompletionWithFallback
|
|
.mockResolvedValueOnce(round("", [table("bp")]))
|
|
.mockResolvedValueOnce(
|
|
round("", [{ ...table("bp"), id: "again" }, table("weight")]),
|
|
)
|
|
.mockResolvedValueOnce(round("done"));
|
|
const out = await runCoachToolLoop({ ...baseArgs, budget: roomy() });
|
|
expect(executeCoachTool).toHaveBeenCalledTimes(2);
|
|
const third = runRawCompletionWithFallback.mock.calls[2][0].params
|
|
.messages as Array<{
|
|
role: string;
|
|
toolCallId?: string;
|
|
content: string;
|
|
}>;
|
|
const repeat = third.find(
|
|
(m) => m.role === "tool" && m.toolCallId === "again",
|
|
);
|
|
expect(JSON.parse(repeat!.content)).toEqual({
|
|
present: false,
|
|
reason: "duplicate",
|
|
duplicateOf: "bp-last30days",
|
|
});
|
|
expect(out.toolTrace.map((t) => t.name)).toEqual([
|
|
"get_metric_table",
|
|
"get_metric_table",
|
|
]);
|
|
});
|
|
|
|
it("brakes when a whole round only repeats", async () => {
|
|
runRawCompletionWithFallback
|
|
.mockResolvedValueOnce(round("", [table("bp")]))
|
|
.mockResolvedValueOnce(round("", [{ ...table("bp"), id: "x" }]))
|
|
.mockResolvedValueOnce(round("enough"));
|
|
const out = await runCoachToolLoop({ ...baseArgs, budget: roomy() });
|
|
expect(out.stop).toEqual({ reason: "no_progress", rounds: 3 });
|
|
expect(executeCoachTool).toHaveBeenCalledTimes(1);
|
|
});
|
|
});
|
|
|
|
describe("dialog and comparison tools", () => {
|
|
const CLARIFY: CoachClarification = {
|
|
kind: "metric",
|
|
choices: [
|
|
{
|
|
id: "c1",
|
|
labelKey: "insights.coach.metric.resting_hr",
|
|
label: "Resting heart rate",
|
|
value: { metric: "resting_hr" },
|
|
},
|
|
{
|
|
id: "c2",
|
|
labelKey: "insights.coach.metric.walking_hr",
|
|
label: "Walking heart rate",
|
|
value: { metric: "walking_hr" },
|
|
},
|
|
],
|
|
freeText: true,
|
|
assumption: "c1",
|
|
};
|
|
|
|
it("ends the turn on a question, with no prose round", async () => {
|
|
runDialogTool.mockResolvedValue({
|
|
kind: "ask",
|
|
result: { asked: true },
|
|
question: "Resting or walking pulse? Otherwise I'll use resting.",
|
|
clarification: CLARIFY,
|
|
});
|
|
runRawCompletionWithFallback.mockResolvedValueOnce(
|
|
round("", [
|
|
{
|
|
id: "q",
|
|
name: "ask_clarification",
|
|
arguments: JSON.stringify({ kind: "metric", question: "x?" }),
|
|
},
|
|
]),
|
|
);
|
|
const out = await runCoachToolLoop({
|
|
...baseArgs,
|
|
budget: roomy(),
|
|
dialog: {
|
|
userId: "u1",
|
|
conversationId: "c1",
|
|
locale: "en",
|
|
userMessage: "how is my pulse?",
|
|
inventory: [],
|
|
conversationWindowSet: false,
|
|
},
|
|
});
|
|
expect(runRawCompletionWithFallback).toHaveBeenCalledTimes(1);
|
|
expect(out.clarification?.clarification).toEqual(CLARIFY);
|
|
expect(out.result.content).toBe(
|
|
"Resting or walking pulse? Otherwise I'll use resting.",
|
|
);
|
|
expect(out.stop).toBeUndefined();
|
|
});
|
|
|
|
it("keeps a question the brake declined as an assumption and goes on", async () => {
|
|
runDialogTool.mockResolvedValue({
|
|
kind: "declined",
|
|
result: { declined: "rate", assume: "Resting heart rate" },
|
|
clarification: CLARIFY,
|
|
});
|
|
runRawCompletionWithFallback
|
|
.mockResolvedValueOnce(
|
|
round("", [
|
|
{
|
|
id: "q",
|
|
name: "ask_clarification",
|
|
arguments: JSON.stringify({ kind: "metric", question: "x?" }),
|
|
},
|
|
]),
|
|
)
|
|
.mockResolvedValueOnce(round("Assumed resting pulse: steady."));
|
|
const out = await runCoachToolLoop({
|
|
...baseArgs,
|
|
budget: roomy(),
|
|
dialog: {
|
|
userId: "u1",
|
|
conversationId: "c1",
|
|
locale: "en",
|
|
userMessage: "how is my pulse?",
|
|
inventory: [],
|
|
conversationWindowSet: false,
|
|
},
|
|
});
|
|
expect(out.clarification).toBeUndefined();
|
|
expect(out.declinedClarifications).toEqual([CLARIFY]);
|
|
expect(out.result.content).toBe("Assumed resting pulse: steady.");
|
|
});
|
|
|
|
it("declines every dialog tool when the turn gives the loop no dialog context", async () => {
|
|
runRawCompletionWithFallback
|
|
.mockResolvedValueOnce(
|
|
round("", [
|
|
{
|
|
id: "r",
|
|
name: "remember_fact",
|
|
arguments: JSON.stringify({
|
|
category: "goal",
|
|
fact: "x",
|
|
why: "y",
|
|
}),
|
|
},
|
|
]),
|
|
)
|
|
.mockResolvedValueOnce(round("ok"));
|
|
const out = await runCoachToolLoop({ ...baseArgs, budget: roomy() });
|
|
expect(runDialogTool).not.toHaveBeenCalled();
|
|
expect(out.memoryNote).toBeUndefined();
|
|
});
|
|
|
|
it("runs compare_series outside the executor", async () => {
|
|
runCompareSeries.mockResolvedValue({
|
|
present: true,
|
|
data: { mode: "periods" },
|
|
});
|
|
runRawCompletionWithFallback
|
|
.mockResolvedValueOnce(
|
|
round("", [
|
|
{
|
|
id: "cmp",
|
|
name: "compare_series",
|
|
arguments: JSON.stringify({ mode: "periods", metric: "bp" }),
|
|
},
|
|
]),
|
|
)
|
|
.mockResolvedValueOnce(round("Higher than last month."));
|
|
const out = await runCoachToolLoop({ ...baseArgs, budget: roomy() });
|
|
expect(runCompareSeries).toHaveBeenCalledTimes(1);
|
|
expect(executeCoachTool).not.toHaveBeenCalled();
|
|
expect(out.toolTrace).toEqual([
|
|
{
|
|
name: "compare_series",
|
|
present: true,
|
|
args: { mode: "periods", metric: "bp" },
|
|
},
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe("reasoning, provider state and the trail", () => {
|
|
it("sends the reasoning level and hands the provider state back on the next round", async () => {
|
|
const state = {
|
|
providerType: "codex",
|
|
model: "gpt",
|
|
items: [{ enc: "x" }],
|
|
};
|
|
runRawCompletionWithFallback
|
|
.mockResolvedValueOnce(round("", [table("bp")], { providerState: state }))
|
|
.mockResolvedValueOnce(round("done"));
|
|
await runCoachToolLoop({
|
|
...baseArgs,
|
|
budget: roomy(),
|
|
reasoning: { effort: "medium", summaries: true },
|
|
});
|
|
const first = runRawCompletionWithFallback.mock.calls[0][0].params;
|
|
expect(first.reasoning).toEqual({ effort: "medium", summaries: true });
|
|
expect(typeof first.onReasoning).toBe("function");
|
|
const second = runRawCompletionWithFallback.mock.calls[1][0].params
|
|
.messages as Array<{ role: string; providerState?: unknown }>;
|
|
expect(second.find((m) => m.role === "assistant")?.providerState).toEqual(
|
|
state,
|
|
);
|
|
});
|
|
|
|
it("opens the thinking entry before the provider is called, and fills it from the reasoning", async () => {
|
|
const events: string[] = [];
|
|
const frames: CoachActivity[] = [];
|
|
const activity = createActivityRecorder({
|
|
emit: (a) => {
|
|
frames.push(a);
|
|
events.push(`${a.phase}:${a.status}`);
|
|
},
|
|
screen: { locale: "en", figures: () => [] },
|
|
});
|
|
runRawCompletionWithFallback.mockImplementationOnce(
|
|
async (args: {
|
|
params: { onReasoning?: (e: { kind: string; text: string }) => void };
|
|
}) => {
|
|
events.push("provider");
|
|
args.params.onReasoning?.({
|
|
kind: "title",
|
|
text: "**Weighing the weeks**",
|
|
});
|
|
args.params.onReasoning?.({
|
|
kind: "text",
|
|
text: "Sleep first, then the pulse.",
|
|
});
|
|
return round("Steady.");
|
|
},
|
|
);
|
|
await runCoachToolLoop({
|
|
...baseArgs,
|
|
budget: roomy(),
|
|
activity,
|
|
locale: "en",
|
|
reasoning: { effort: "low", summaries: true },
|
|
});
|
|
expect(events[0]).toBe("thinking:running");
|
|
expect(events[1]).toBe("provider");
|
|
const done = frames.filter((f) => f.phase === "thinking").at(-1)!;
|
|
expect(done.status).toBe("done");
|
|
expect(done.title).toBe("Weighing the weeks");
|
|
expect(done.text).toBe("Sleep first, then the pulse.");
|
|
// The persisted metadata carries no model text.
|
|
expect(JSON.stringify(activity.meta())).not.toContain("Weighing");
|
|
expect(activity.trail()?.entries[0]).toMatchObject({
|
|
title: "Weighing the weeks",
|
|
});
|
|
});
|
|
});
|