diff --git a/src/lib/ai/__tests__/reasoning-passthrough.test.ts b/src/lib/ai/__tests__/reasoning-passthrough.test.ts new file mode 100644 index 000000000..16886899f --- /dev/null +++ b/src/lib/ai/__tests__/reasoning-passthrough.test.ts @@ -0,0 +1,128 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { AiReasoningEvent, CompletionParams } from "../types"; + +/** + * v1.41 — the chain runner hands `reasoning` and `onReasoning` to every + * candidate unchanged, reports reasoning as counts only, and the mock client + * reasons deterministically for the loop's tests. + */ +const annotations = vi.hoisted(() => [] as Array>); +vi.mock("@/lib/logging/context", () => ({ + annotate: (fields: { meta?: Record }) => { + if (fields.meta) annotations.push(fields.meta); + }, +})); +vi.mock("../coach/budget", async () => { + const actual = + await vi.importActual("../coach/budget"); + return { + ...actual, + readDailySpend: vi.fn(async () => ({ total: 0, operator: 0 })), + }; +}); + +import { MockAIProvider } from "../mock-client"; +import { + clearLastWorkingProviderCache, + runRawCompletionWithFallback, +} from "../provider-runner"; +import { createInMemoryProviderHealthLedger } from "../provider-health-ledger"; + +function params(extra: Partial = {}): CompletionParams { + return { + system: "s", + messages: [{ role: "user", content: "u" }], + ...extra, + }; +} + +beforeEach(() => { + annotations.length = 0; + clearLastWorkingProviderCache(); +}); + +describe("MockAIProvider reasoning", () => { + it("reasons deterministically when asked, with state for the next round", async () => { + const mock = new MockAIProvider({ providerType: "codex", responses: "a" }); + const events: AiReasoningEvent[] = []; + const first = await mock.generateCompletion( + params({ + reasoning: { effort: "medium", summaries: true }, + onReasoning: (e) => events.push(e), + }), + ); + expect(events).toEqual([ + { kind: "title", text: "Reading the request" }, + { + kind: "text", + text: "**Reading the request**\nMock reasoning for call 1.", + }, + { kind: "done", text: "" }, + ]); + expect(first.reasoning).toEqual({ + summary: ["**Reading the request**\nMock reasoning for call 1."], + tokens: 8, + }); + expect(first.providerState).toEqual({ + providerType: "codex", + model: "mock-model", + items: [{ type: "mock-reasoning", call: 1 }], + }); + }); + + it("stays silent without a request, and reports nothing for off", async () => { + const mock = new MockAIProvider({ responses: "a" }); + const onReasoning = vi.fn(); + const plain = await mock.generateCompletion(params({ onReasoning })); + expect(onReasoning).not.toHaveBeenCalled(); + expect(plain.reasoning).toBeUndefined(); + const off = await mock.generateCompletion( + params({ reasoning: { effort: "off", summaries: true }, onReasoning }), + ); + expect(off.reasoning).toEqual({ summary: [], tokens: null }); + expect(off.providerState).toBeUndefined(); + }); +}); + +describe("Chain runner reasoning pass-through", () => { + it("hands the same reasoning request to the fallback hop and counts it", async () => { + const failing = new MockAIProvider({ + providerType: "anthropic", + rejectWith: Object.assign(new Error("down"), { httpStatus: 503 }), + }); + const working = new MockAIProvider({ + providerType: "codex", + responses: "ok", + }); + const events: AiReasoningEvent[] = []; + const onReasoning = (e: AiReasoningEvent) => events.push(e); + const reasoning = { effort: "high" as const, summaries: true }; + + const out = await runRawCompletionWithFallback({ + userId: "u1", + providers: [ + { providerType: "anthropic", instance: failing }, + { providerType: "codex", instance: working }, + ], + params: params({ reasoning, onReasoning }), + surface: "coach", + ledger: createInMemoryProviderHealthLedger(), + }); + + expect(failing.calls[0].reasoning).toBe(reasoning); + expect(working.calls[0].reasoning).toBe(reasoning); + expect(working.calls[0].onReasoning).toBe(onReasoning); + expect(events.map((e) => e.kind)).toEqual(["title", "text", "done"]); + expect(out.result.providerState?.items).toHaveLength(1); + + const success = annotations.find((m) => "ai_reasoning_spend" in m); + expect(success).toMatchObject({ + ai_reasoning_spend: 8, + ai_reasoning_summary_count: 1, + }); + // Counts only: no model text, no provider state on the wide event. + const flat = JSON.stringify(annotations); + expect(flat).not.toContain("mock-reasoning"); + expect(flat).not.toContain("Reading the request"); + }); +}); diff --git a/src/lib/ai/mock-client.ts b/src/lib/ai/mock-client.ts index 6cb7a2b65..1b596662b 100644 --- a/src/lib/ai/mock-client.ts +++ b/src/lib/ai/mock-client.ts @@ -4,6 +4,7 @@ import type { CompletionParams, CompletionResult, } from "./types"; +import { reasoningTitleOf } from "./reasoning/support"; /** * Deterministic mock AI provider for tests. @@ -52,6 +53,14 @@ export interface MockAIProviderOptions { finishReason?: CompletionResult["finishReason"]; /** v1.20.0 — cachedInputTokens echoed on the result. Defaults to null. */ cachedInputTokens?: number | null; + /** + * v1.41 — what a call that asked for reasoning reports. Unset: a fixed + * summary per call (`**Reading the request**` + the call number), 8 tokens, + * and one `providerState` item naming the call. `false`: the mock never + * reasons, like a provider without a reasoning stream. Calls without + * `params.reasoning` (or with `off`) never get either, as on a real client. + */ + reasoning?: { summary: string[]; tokens: number | null } | false; } /** Type guard: a per-call array of tool-call sets. */ @@ -82,6 +91,7 @@ export class MockAIProvider implements AIProvider { private readonly cachedInputTokens: number | null; private readonly model: string; private readonly rejectWith: Error | undefined; + private readonly reasoning: MockAIProviderOptions["reasoning"]; private callIdx = 0; constructor(opts: MockAIProviderOptions = {}) { @@ -90,6 +100,7 @@ export class MockAIProvider implements AIProvider { this.rejectWith = opts.rejectWith; this.finishReason = opts.finishReason; this.cachedInputTokens = opts.cachedInputTokens ?? null; + this.reasoning = opts.reasoning; if (isToolCallMatrix(opts.toolCalls)) { this.toolCalls = opts.toolCalls; } else if (Array.isArray(opts.toolCalls)) { @@ -127,6 +138,7 @@ export class MockAIProvider implements AIProvider { ? this.toolCalls[Math.min(this.callIdx, this.toolCalls.length - 1)] : undefined; this.callIdx++; + const reasoned = this.reason(params, this.callIdx); return { content: this.responses[idx], tokensUsed: this.tokens[tokIdx] ?? null, @@ -135,6 +147,41 @@ export class MockAIProvider implements AIProvider { providerType: this.type, ...(toolCalls ? { toolCalls } : {}), finishReason: this.finishReason, + ...reasoned, + }; + } + + /** + * v1.41 — deterministic reasoning for a call that asked for it: the same + * events a real client reports (title, text, done) and a `providerState` + * the next round can be checked for. + */ + private reason( + params: CompletionParams, + call: number, + ): Pick { + if (!params.reasoning) return {}; + if (params.reasoning.effort === "off" || this.reasoning === false) { + params.onReasoning?.({ kind: "done", text: "" }); + return { reasoning: { summary: [], tokens: null } }; + } + const reasoning = this.reasoning ?? { + summary: [`**Reading the request**\nMock reasoning for call ${call}.`], + tokens: 8, + }; + for (const text of reasoning.summary) { + const title = reasoningTitleOf(text); + if (title) params.onReasoning?.({ kind: "title", text: title }); + params.onReasoning?.({ kind: "text", text }); + } + params.onReasoning?.({ kind: "done", text: "" }); + return { + reasoning: { summary: [...reasoning.summary], tokens: reasoning.tokens }, + providerState: { + providerType: this.type, + model: this.model, + items: [{ type: "mock-reasoning", call }], + }, }; } diff --git a/src/lib/ai/provider-runner.ts b/src/lib/ai/provider-runner.ts index 7bf914106..860da4973 100644 --- a/src/lib/ai/provider-runner.ts +++ b/src/lib/ai/provider-runner.ts @@ -413,6 +413,10 @@ async function runRawChain( surface: BudgetSurface; ledger?: ProviderHealthLedger; }, + // v1.41 — `params` reaches every candidate unchanged, `reasoning` and + // `onReasoning` included. A fallback hop may see `providerState` another + // provider made on an earlier round; each client re-inserts only its own + // and drops the rest, so nothing is filtered here. invoke: (candidate: ProviderChainResolved) => Promise, successMeta?: ( candidate: ProviderChainResolved, @@ -494,6 +498,16 @@ async function runRawChain( ai_chain_working_provider: candidate.providerType, ai_chain_fallback_count: hops.length, ...(successMeta ? successMeta(candidate) : {}), + // v1.41 — counts only. The summaries are model text and the + // provider state is opaque; neither ever reaches a wide event. The + // reasoning-token count is `spend`: a key naming tokens is blanked + // by the central redactor. + ...(result.reasoning + ? { + ai_reasoning_spend: result.reasoning.tokens, + ai_reasoning_summary_count: result.reasoning.summary.length, + } + : {}), }, }); return {