From 34afa3a4f0ce63e4be53a7ff9ef0ff81b6b1786a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 6 Oct 2026 19:26:30 +0200 Subject: [PATCH] feat(ai): reason deterministically in the mock and count reasoning in the chain runner The mock reports a fixed summary, title and provider state for calls that ask for reasoning. The runner passes reasoning and onReasoning to every candidate unchanged and adds the reasoning token and summary counts to the wide event, never the text or the state. --- .../__tests__/reasoning-passthrough.test.ts | 128 ++++++++++++++++++ src/lib/ai/mock-client.ts | 47 +++++++ src/lib/ai/provider-runner.ts | 14 ++ 3 files changed, 189 insertions(+) create mode 100644 src/lib/ai/__tests__/reasoning-passthrough.test.ts diff --git a/src/lib/ai/__tests__/reasoning-passthrough.test.ts b/src/lib/ai/__tests__/reasoning-passthrough.test.ts new file mode 100644 index 000000000..16886899f --- /dev/null +++ b/src/lib/ai/__tests__/reasoning-passthrough.test.ts @@ -0,0 +1,128 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { AiReasoningEvent, CompletionParams } from "../types"; + +/** + * v1.41 — the chain runner hands `reasoning` and `onReasoning` to every + * candidate unchanged, reports reasoning as counts only, and the mock client + * reasons deterministically for the loop's tests. + */ +const annotations = vi.hoisted(() => [] as Array>); +vi.mock("@/lib/logging/context", () => ({ + annotate: (fields: { meta?: Record }) => { + if (fields.meta) annotations.push(fields.meta); + }, +})); +vi.mock("../coach/budget", async () => { + const actual = + await vi.importActual("../coach/budget"); + return { + ...actual, + readDailySpend: vi.fn(async () => ({ total: 0, operator: 0 })), + }; +}); + +import { MockAIProvider } from "../mock-client"; +import { + clearLastWorkingProviderCache, + runRawCompletionWithFallback, +} from "../provider-runner"; +import { createInMemoryProviderHealthLedger } from "../provider-health-ledger"; + +function params(extra: Partial = {}): CompletionParams { + return { + system: "s", + messages: [{ role: "user", content: "u" }], + ...extra, + }; +} + +beforeEach(() => { + annotations.length = 0; + clearLastWorkingProviderCache(); +}); + +describe("MockAIProvider reasoning", () => { + it("reasons deterministically when asked, with state for the next round", async () => { + const mock = new MockAIProvider({ providerType: "codex", responses: "a" }); + const events: AiReasoningEvent[] = []; + const first = await mock.generateCompletion( + params({ + reasoning: { effort: "medium", summaries: true }, + onReasoning: (e) => events.push(e), + }), + ); + expect(events).toEqual([ + { kind: "title", text: "Reading the request" }, + { + kind: "text", + text: "**Reading the request**\nMock reasoning for call 1.", + }, + { kind: "done", text: "" }, + ]); + expect(first.reasoning).toEqual({ + summary: ["**Reading the request**\nMock reasoning for call 1."], + tokens: 8, + }); + expect(first.providerState).toEqual({ + providerType: "codex", + model: "mock-model", + items: [{ type: "mock-reasoning", call: 1 }], + }); + }); + + it("stays silent without a request, and reports nothing for off", async () => { + const mock = new MockAIProvider({ responses: "a" }); + const onReasoning = vi.fn(); + const plain = await mock.generateCompletion(params({ onReasoning })); + expect(onReasoning).not.toHaveBeenCalled(); + expect(plain.reasoning).toBeUndefined(); + const off = await mock.generateCompletion( + params({ reasoning: { effort: "off", summaries: true }, onReasoning }), + ); + expect(off.reasoning).toEqual({ summary: [], tokens: null }); + expect(off.providerState).toBeUndefined(); + }); +}); + +describe("Chain runner reasoning pass-through", () => { + it("hands the same reasoning request to the fallback hop and counts it", async () => { + const failing = new MockAIProvider({ + providerType: "anthropic", + rejectWith: Object.assign(new Error("down"), { httpStatus: 503 }), + }); + const working = new MockAIProvider({ + providerType: "codex", + responses: "ok", + }); + const events: AiReasoningEvent[] = []; + const onReasoning = (e: AiReasoningEvent) => events.push(e); + const reasoning = { effort: "high" as const, summaries: true }; + + const out = await runRawCompletionWithFallback({ + userId: "u1", + providers: [ + { providerType: "anthropic", instance: failing }, + { providerType: "codex", instance: working }, + ], + params: params({ reasoning, onReasoning }), + surface: "coach", + ledger: createInMemoryProviderHealthLedger(), + }); + + expect(failing.calls[0].reasoning).toBe(reasoning); + expect(working.calls[0].reasoning).toBe(reasoning); + expect(working.calls[0].onReasoning).toBe(onReasoning); + expect(events.map((e) => e.kind)).toEqual(["title", "text", "done"]); + expect(out.result.providerState?.items).toHaveLength(1); + + const success = annotations.find((m) => "ai_reasoning_spend" in m); + expect(success).toMatchObject({ + ai_reasoning_spend: 8, + ai_reasoning_summary_count: 1, + }); + // Counts only: no model text, no provider state on the wide event. + const flat = JSON.stringify(annotations); + expect(flat).not.toContain("mock-reasoning"); + expect(flat).not.toContain("Reading the request"); + }); +}); diff --git a/src/lib/ai/mock-client.ts b/src/lib/ai/mock-client.ts index 6cb7a2b65..1b596662b 100644 --- a/src/lib/ai/mock-client.ts +++ b/src/lib/ai/mock-client.ts @@ -4,6 +4,7 @@ import type { CompletionParams, CompletionResult, } from "./types"; +import { reasoningTitleOf } from "./reasoning/support"; /** * Deterministic mock AI provider for tests. @@ -52,6 +53,14 @@ export interface MockAIProviderOptions { finishReason?: CompletionResult["finishReason"]; /** v1.20.0 — cachedInputTokens echoed on the result. Defaults to null. */ cachedInputTokens?: number | null; + /** + * v1.41 — what a call that asked for reasoning reports. Unset: a fixed + * summary per call (`**Reading the request**` + the call number), 8 tokens, + * and one `providerState` item naming the call. `false`: the mock never + * reasons, like a provider without a reasoning stream. Calls without + * `params.reasoning` (or with `off`) never get either, as on a real client. + */ + reasoning?: { summary: string[]; tokens: number | null } | false; } /** Type guard: a per-call array of tool-call sets. */ @@ -82,6 +91,7 @@ export class MockAIProvider implements AIProvider { private readonly cachedInputTokens: number | null; private readonly model: string; private readonly rejectWith: Error | undefined; + private readonly reasoning: MockAIProviderOptions["reasoning"]; private callIdx = 0; constructor(opts: MockAIProviderOptions = {}) { @@ -90,6 +100,7 @@ export class MockAIProvider implements AIProvider { this.rejectWith = opts.rejectWith; this.finishReason = opts.finishReason; this.cachedInputTokens = opts.cachedInputTokens ?? null; + this.reasoning = opts.reasoning; if (isToolCallMatrix(opts.toolCalls)) { this.toolCalls = opts.toolCalls; } else if (Array.isArray(opts.toolCalls)) { @@ -127,6 +138,7 @@ export class MockAIProvider implements AIProvider { ? this.toolCalls[Math.min(this.callIdx, this.toolCalls.length - 1)] : undefined; this.callIdx++; + const reasoned = this.reason(params, this.callIdx); return { content: this.responses[idx], tokensUsed: this.tokens[tokIdx] ?? null, @@ -135,6 +147,41 @@ export class MockAIProvider implements AIProvider { providerType: this.type, ...(toolCalls ? { toolCalls } : {}), finishReason: this.finishReason, + ...reasoned, + }; + } + + /** + * v1.41 — deterministic reasoning for a call that asked for it: the same + * events a real client reports (title, text, done) and a `providerState` + * the next round can be checked for. + */ + private reason( + params: CompletionParams, + call: number, + ): Pick { + if (!params.reasoning) return {}; + if (params.reasoning.effort === "off" || this.reasoning === false) { + params.onReasoning?.({ kind: "done", text: "" }); + return { reasoning: { summary: [], tokens: null } }; + } + const reasoning = this.reasoning ?? { + summary: [`**Reading the request**\nMock reasoning for call ${call}.`], + tokens: 8, + }; + for (const text of reasoning.summary) { + const title = reasoningTitleOf(text); + if (title) params.onReasoning?.({ kind: "title", text: title }); + params.onReasoning?.({ kind: "text", text }); + } + params.onReasoning?.({ kind: "done", text: "" }); + return { + reasoning: { summary: [...reasoning.summary], tokens: reasoning.tokens }, + providerState: { + providerType: this.type, + model: this.model, + items: [{ type: "mock-reasoning", call }], + }, }; } diff --git a/src/lib/ai/provider-runner.ts b/src/lib/ai/provider-runner.ts index 7bf914106..860da4973 100644 --- a/src/lib/ai/provider-runner.ts +++ b/src/lib/ai/provider-runner.ts @@ -413,6 +413,10 @@ async function runRawChain( surface: BudgetSurface; ledger?: ProviderHealthLedger; }, + // v1.41 — `params` reaches every candidate unchanged, `reasoning` and + // `onReasoning` included. A fallback hop may see `providerState` another + // provider made on an earlier round; each client re-inserts only its own + // and drops the rest, so nothing is filtered here. invoke: (candidate: ProviderChainResolved) => Promise, successMeta?: ( candidate: ProviderChainResolved, @@ -494,6 +498,16 @@ async function runRawChain( ai_chain_working_provider: candidate.providerType, ai_chain_fallback_count: hops.length, ...(successMeta ? successMeta(candidate) : {}), + // v1.41 — counts only. The summaries are model text and the + // provider state is opaque; neither ever reaches a wide event. The + // reasoning-token count is `spend`: a key naming tokens is blanked + // by the central redactor. + ...(result.reasoning + ? { + ai_reasoning_spend: result.reasoning.tokens, + ai_reasoning_summary_count: result.reasoning.summary.length, + } + : {}), }, }); return {