fix(coach): make the daily token budget provider-aware

The daily AI-token gate was a flat 25,000/day for every provider. That cap
exists to bound the OPERATOR's API bill, but a ChatGPT-OAuth (Codex) or BYOK
chain egresses on the USER's own plan/key and costs the operator nothing, so
gating it on the operator-cost ceiling is wrong. A single gpt-5.x reasoning
turn legitimately reports 20k-40k gross total_tokens (re-sent system prompt +
inventory + tool defs per round + hidden reasoning, summed across rounds), so
one or two turns exhausted the flat cap and locked the user out of a plan they
pay for.

Classify the chain's cost owner by its primary provider via resolveDailyCap:
an admin-openai primary (operator pays) keeps the operator-cost cap; every
user-egress primary (codex/openai/anthropic/local) gets a generous
abuse-only ceiling. Threaded into the Coach chat gate and both OCR-extract
modes.

Raise the operator-cost cap 25,000 -> 200,000 so the operator-key path also
survives a normal day of reasoning turns, keeping gross-token accounting
rather than reworking the reconcile math.

Subtract cached input tokens at reconcile: the Responses-API gross total still
includes prompt-cached input the user did not re-pay for. The codex client
already parsed cached_tokens; sum it through the tool loop and bill
total_tokens minus cached.
This commit is contained in:
Marc-André Bombeck committed 2026-06-26 12:53:04 +02:00
1 parent 9a3049a6cf
commit 3783348506
9 files changed
+262 -7

No files matched your search

@@ -94,6 +94,7 @@ vi.mock("@/lib/ai/coach/budget", () => ({
buildDateKey: vi.fn(),
enforceBudget: vi.fn(),
recordSpend: vi.fn(),
resolveDailyCap: vi.fn(() => 200_000),
}));
vi.mock("@/lib/ai/coach/refusal", () => ({ detectRefusal: vi.fn() }));
vi.mock("@/lib/ai/coach/system-prompt", () => ({
@@ -100,6 +100,7 @@ vi.mock("@/lib/ai/coach/budget", () => ({
buildDateKey: vi.fn(() => "2026-06-21"),
reserveBudget: vi.fn(async () => ({ allowed: true, reserved: 1500 })),
reconcileSpend: vi.fn(async () => undefined),
resolveDailyCap: vi.fn(() => 2_000_000),
}));
vi.mock("@/lib/ai/coach/refusal", () => ({
detectRefusal: vi.fn(() => ({ refuse: false })),
@@ -111,6 +111,7 @@ vi.mock("@/lib/ai/coach/budget", () => ({
buildDateKey: vi.fn(() => "2026-06-21"),
reserveBudget,
reconcileSpend,
resolveDailyCap: vi.fn(() => 2_000_000),
}));
const { detectRefusal } = vi.hoisted(() => ({
@@ -245,8 +246,14 @@ describe("coach chat — tool-mode routing (F1)", () => {
{ providerType: "anthropic", instance: {} },
]);
await post(chatReq({ message: "How is my BP?" }));
// maxTokens (1500) × MAX_ROUNDS (2).
expect(reserveBudget).toHaveBeenCalledWith("u1", 3000, "2026-06-21");
// maxTokens (1500) × MAX_ROUNDS (2). The 4th arg is the provider-aware
// daily cap (F1) — mocked to the user-plan ceiling for this BYOK chain.
expect(reserveBudget).toHaveBeenCalledWith(
"u1",
3000,
"2026-06-21",
2_000_000,
);
});
it("persists the tool trace onto provenance", async () => {
@@ -321,8 +328,14 @@ describe("coach chat — tool-mode routing (F1)", () => {
expect(runCoachToolLoop).not.toHaveBeenCalled();
expect(runRawCompletionWithFallback).toHaveBeenCalledTimes(1);
// Single-round budget reservation in the fallback path.
expect(reserveBudget).toHaveBeenCalledWith("u1", 1500, "2026-06-21");
// Single-round budget reservation in the fallback path; the 4th arg is the
// provider-aware daily cap (F1).
expect(reserveBudget).toHaveBeenCalledWith(
"u1",
1500,
"2026-06-21",
2_000_000,
);
// The legacy path ships the snapshot figures in the user turn.
const params = (
(runRawCompletionWithFallback.mock.calls[0] as unknown[])[0] as {
+14
View File
@@ -64,6 +64,7 @@ import {
buildDateKey,
reserveBudget,
reconcileSpend,
resolveDailyCap,
} from "@/lib/ai/coach/budget";
import { detectRefusal } from "@/lib/ai/coach/refusal";
import {
@@ -503,6 +504,12 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}.`;
// so reserve the per-call ceiling × the round count up front and reconcile
// the SUMMED actual tokens afterwards. The atomic reserve/reconcile
// primitives are unchanged; only the reserved amount scales.
// v1.21.0 (F1) — the daily ceiling is the OPERATOR's cost cap only when the
// chain egresses via the operator's own key (`admin-openai` primary). A
// ChatGPT-OAuth/Codex or BYOK chain runs on the user's OWN plan/key and costs
// the operator nothing, so it gets the generous user-plan ceiling — gating it
// on the operator-cost cap would lock the user out of a plan they pay for.
const dailyCap = resolveDailyCap(chain);
const reqDateKey = buildDateKey();
const reservation = await reserveBudget(
userId,
@@ -510,6 +517,7 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}.`;
? AI_BUDGETS.coach.maxTokens * MAX_ROUNDS
: AI_BUDGETS.coach.maxTokens,
reqDateKey,
dailyCap,
);
if (!reservation.allowed) {
annotate({
@@ -526,6 +534,9 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}.`;
// prose number-verifier. Empty on the no-tools path.
let toolResultPayloads: unknown[] = [];
let totalTokensSpent: number;
// v1.21.0 (F3) — cached-input tokens to subtract at reconcile (prompt-cached
// input the user did not re-pay for must not be billed to the daily meter).
let cachedTokensSpent = 0;
try {
if (toolMode) {
// v1.20.0 (F1) — base context: the full system prompt + a tool-mode
@@ -565,6 +576,7 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}. Fetch
toolTrace = loop.toolTrace;
toolResultPayloads = (loop.toolResults ?? []).map((r) => r.data);
totalTokensSpent = loop.totalTokens;
cachedTokensSpent = loop.cachedTokens;
} else {
const fallback = await runRawCompletionWithFallback({
userId,
@@ -587,6 +599,7 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}. Fetch
result = fallback.result;
workingProviderType = fallback.workingProvider.providerType;
totalTokensSpent = result.tokensUsed ?? 0;
cachedTokensSpent = result.cachedInputTokens ?? 0;
}
} catch (err) {
// The provider chain failed outright — no tokens were billed, so refund
@@ -643,6 +656,7 @@ Reply now as the assistant, in ${locale === "de" ? "German" : "English"}. Fetch
reservation.reserved,
totalTokensSpent,
reqDateKey,
cachedTokensSpent,
).catch(() => {
// Ledger reconcile is best-effort; a failure leaves the conservative
// reservation in place (never an undercount) and never breaks the turn.
@@ -41,6 +41,7 @@ vi.mock("@/lib/ai/coach/budget", () => ({
buildDateKey: vi.fn(() => "2026-06-26"),
reserveBudget: vi.fn(),
reconcileSpend: vi.fn().mockResolvedValue(undefined),
resolveDailyCap: vi.fn(() => 200_000),
}));
vi.mock("@/lib/labs/ocr-extract", async () => {
const actual = await vi.importActual<typeof import("@/lib/labs/ocr-extract")>(
@@ -106,6 +107,8 @@ describe("POST /api/labs/ocr/extract — text mode budget", () => {
"user-1",
AI_BUDGETS.ocrExtractText.maxTokens,
"2026-06-26",
// F1 — the provider-aware daily cap (mocked) is threaded as the 4th arg.
200_000,
);
});
+9
View File
@@ -35,6 +35,7 @@ import {
buildDateKey,
reconcileSpend,
reserveBudget,
resolveDailyCap,
} from "@/lib/ai/coach/budget";
import { prisma } from "@/lib/db";
import {
@@ -142,11 +143,15 @@ async function handleTextExtract(
// a vision call, so it reserves the proportionate text ceiling rather than
// the vision budget. Over-charging the vision rate for a text call would
// exhaust the day budget against spend that never happened.
// v1.21.0 (F1) — the operator-cost cap applies only when the picked provider
// egresses on the operator's own key; a BYOK / Codex / local pick runs on the
// user's own plan and gets the generous user-plan ceiling.
const dateKey = buildDateKey();
const reservation = await reserveBudget(
userId,
AI_BUDGETS.ocrExtractText.maxTokens,
dateKey,
resolveDailyCap([{ providerType: pick.entry.providerType }]),
);
if (!reservation.allowed) {
annotate({
@@ -234,11 +239,15 @@ async function handleVisionExtract(
}
// 4. Reserve the day's budget BEFORE the provider call (atomic, TOCTOU-safe).
// v1.21.0 (F1) — the operator-cost cap applies only when the picked vision
// provider egresses on the operator's own key; a BYOK / Codex pick runs on
// the user's own plan and gets the generous user-plan ceiling.
const dateKey = buildDateKey();
const reservation = await reserveBudget(
userId,
AI_BUDGETS.ocrExtract.maxTokens,
dateKey,
resolveDailyCap([{ providerType: pick.entry.providerType }]),
);
if (!reservation.allowed) {
annotate({