From 18ed57b8671ca1d49f44aafbbb543056f66c968f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 11 Sep 2026 11:11:01 +0200 Subject: [PATCH] fix(ai): every surface keeps a ceiling on the day's mixed total The owner split left an operator-funded chain with no ceiling at all. Its arm compared operator_tokens, and that counter returns to roughly zero on every reconcile the user's own plan settled: on the 2026-09-11 evidence the shared key answered 500 and codex served, so the counter never grew, every job reservation was admitted forever, and a runaway client loop could write an unbounded row. A pre-existing concurrency contract caught it from the other side, two reservations that must not both pass one cap both passing. reserveBudget now refuses when the owner's counter is over its cap OR the day's mixed total is over the abuse ceiling, and a background surface gets its share of that ceiling too, so background spend is bounded on the total and not only under a counter the operator's key never fills. The surface is a required argument, so the type checker enumerated every reservation site. The hop guard learns the same lesson: an operator-funded fallback hop is now rationed at the calling surface's ceiling rather than the full operator cap. A job admitted under the job share could previously spend the whole 200 000 once its chain fell back onto the shared key, which is the original defect reached by the fallback path. The job share applies only where the operator pays for the chain. Multiplying whichever ceiling the chain resolved halved background generation for a self-hoster on a local model, who costs the operator nothing. Refusals now name the ceiling they hit and whose it was. A refusal used to emit the day's mixed total and nothing else, which on an operator refusal is not the counter that tripped. --- src/app/api/ai/test/route.ts | 1 + .../documents/inbound/[id]/extract/route.ts | 3 + .../api/documents/inbound/[id]/index/route.ts | 1 + .../documents/inbound/[id]/suggest/route.ts | 2 + .../documents/inbound/[id]/summary/route.ts | 2 + .../chat/__tests__/route-tool-mode.test.ts | 2 + src/app/api/insights/chat/route.ts | 17 ++- .../labs/ocr/extract/__tests__/route.test.ts | 2 + src/app/api/labs/ocr/extract/route.ts | 2 + .../extract/__tests__/route.test.ts | 3 + src/app/api/medications/extract/route.ts | 2 + .../__tests__/gateway-embedded-error.test.ts | 3 + src/lib/ai/__tests__/provider-runner.test.ts | 125 ++++++++++++++++++ src/lib/ai/coach/__tests__/budget.test.ts | 119 ++++++++++++++++- .../self-context-questions-consent.test.ts | 1 + src/lib/ai/coach/budget.ts | 108 +++++++++++++-- src/lib/ai/coach/self-context-questions.ts | 1 + src/lib/ai/coach/tools/loop.ts | 2 + src/lib/ai/provider-runner.ts | 33 ++++- src/lib/documents/auto-stage-labs.ts | 1 + src/lib/documents/fenced-chat.ts | 2 + src/lib/documents/index-document.ts | 1 + .../__tests__/status-provider-budget.test.ts | 13 ++ src/lib/insights/briefing-provider.ts | 20 ++- src/lib/insights/status-provider.ts | 25 +++- src/lib/jobs/coach-nudge-ai.ts | 1 + .../jobs/document-content-index-backfill.ts | 1 + src/lib/jobs/document-summary.ts | 1 + tests/integration/coach-budget-owner.test.ts | 99 ++++++++++++++ .../coach-budget-reservation.test.ts | 77 ++++++++++- 30 files changed, 645 insertions(+), 25 deletions(-) diff --git a/src/app/api/ai/test/route.ts b/src/app/api/ai/test/route.ts index 389938c71..27265df91 100644 --- a/src/app/api/ai/test/route.ts +++ b/src/app/api/ai/test/route.ts @@ -119,6 +119,7 @@ export const POST = apiHandler(async (request: NextRequest) => { providerType: provider.type === "admin-key" ? "admin-openai" : "local", }, ]), + "coach", ); if (!reservation.allowed) { annotate({ action: { name: "ai.test.budget_exceeded" } }); diff --git a/src/app/api/documents/inbound/[id]/extract/route.ts b/src/app/api/documents/inbound/[id]/extract/route.ts index 75b43c825..118717268 100644 --- a/src/app/api/documents/inbound/[id]/extract/route.ts +++ b/src/app/api/documents/inbound/[id]/extract/route.ts @@ -256,6 +256,7 @@ async function handleTextExtract( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { await refundDocumentAiSlot(userId); @@ -359,6 +360,7 @@ async function handleStoredExtract( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { await refundDocumentAiSlot(userId); @@ -479,6 +481,7 @@ async function handleVisionExtract( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { await refundDocumentAiSlot(userId); diff --git a/src/app/api/documents/inbound/[id]/index/route.ts b/src/app/api/documents/inbound/[id]/index/route.ts index 87f37415a..04530af0c 100644 --- a/src/app/api/documents/inbound/[id]/index/route.ts +++ b/src/app/api/documents/inbound/[id]/index/route.ts @@ -251,6 +251,7 @@ async function handleVisionIndex( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { await refundDocumentAiSlot(userId); diff --git a/src/app/api/documents/inbound/[id]/suggest/route.ts b/src/app/api/documents/inbound/[id]/suggest/route.ts index 61bd2499d..0fe5d57a7 100644 --- a/src/app/api/documents/inbound/[id]/suggest/route.ts +++ b/src/app/api/documents/inbound/[id]/suggest/route.ts @@ -168,6 +168,7 @@ async function handleTextSuggest( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { await refundDocumentAiSlot(userId); @@ -263,6 +264,7 @@ async function handleVisionSuggest( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { await refundDocumentAiSlot(userId); diff --git a/src/app/api/documents/inbound/[id]/summary/route.ts b/src/app/api/documents/inbound/[id]/summary/route.ts index f0903b787..f207faf12 100644 --- a/src/app/api/documents/inbound/[id]/summary/route.ts +++ b/src/app/api/documents/inbound/[id]/summary/route.ts @@ -347,6 +347,7 @@ async function handleTextSummary( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { await refundDocumentAiSlot(userId); @@ -438,6 +439,7 @@ async function handleVisionSummary( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { await refundDocumentAiSlot(userId); diff --git a/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts b/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts index f0b475076..2dac8489d 100644 --- a/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts +++ b/src/app/api/insights/chat/__tests__/route-tool-mode.test.ts @@ -326,6 +326,7 @@ describe("coach chat — tool-mode routing (F1)", () => { "2026-06-21", 2_000_000, "user", + "coach", ); }); @@ -569,6 +570,7 @@ describe("coach chat — tool-mode routing (F1)", () => { "2026-06-21", 2_000_000, "user", + "coach", ); // The legacy path ships the snapshot figures in the user turn. const params = ( diff --git a/src/app/api/insights/chat/route.ts b/src/app/api/insights/chat/route.ts index 5f6c036ce..c0880ce9d 100644 --- a/src/app/api/insights/chat/route.ts +++ b/src/app/api/insights/chat/route.ts @@ -606,11 +606,25 @@ async function handleChatRequest(request: NextRequest): Promise { reqDateKey, dailyCap, resolveCostOwner(chain), + "coach", ); if (!reservation.allowed) { + // v1.38.19 (Wave E, fix round 1) — say WHICH ceiling refused and WHOSE. + // `totalAfter` alone is the day's mixed total; on an operator refusal that + // is not the counter that tripped, and reading `totalAfter: 1200000` + // against a 200 000 ceiling is what turned the 06:42Z incident into a + // production log dig. Integers and two small enums — no key, host or model + // name is added. annotate({ action: { name: "coach.budget.exceeded" }, - meta: { totalAfter: reservation.totalAfter }, + meta: { + owner: reservation.owner, + surface: "coach", + limit: reservation.limit, + cap: dailyCap, + totalAfter: reservation.totalAfter, + operatorAfter: reservation.operatorAfter, + }, }); return streamProviderError({ code: "coach.budget.exceeded" }); } @@ -748,6 +762,7 @@ async function handleChatRequest(request: NextRequest): Promise { // returned in full so every guard below still runs on the complete text. let streamedDeltas = 0; const fallback = await runStreamingRawCompletionWithFallback({ + surface: "coach", userId, providers: chain, onDelta: () => { diff --git a/src/app/api/labs/ocr/extract/__tests__/route.test.ts b/src/app/api/labs/ocr/extract/__tests__/route.test.ts index d5b1dca60..9471f6ac1 100644 --- a/src/app/api/labs/ocr/extract/__tests__/route.test.ts +++ b/src/app/api/labs/ocr/extract/__tests__/route.test.ts @@ -122,6 +122,8 @@ describe("POST /api/labs/ocr/extract — text mode budget", () => { 200_000, // Wave E — and the cost owner that cap is enforced against as the 5th. "operator", + // Wave E fix round 1 — the surface the ceiling is rationed for, 6th. + "coach", ); }); diff --git a/src/app/api/labs/ocr/extract/route.ts b/src/app/api/labs/ocr/extract/route.ts index 7dfd230f6..901b5b0f5 100644 --- a/src/app/api/labs/ocr/extract/route.ts +++ b/src/app/api/labs/ocr/extract/route.ts @@ -185,6 +185,7 @@ async function handleTextExtract( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { annotate({ @@ -289,6 +290,7 @@ async function handleVisionExtract( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { annotate({ diff --git a/src/app/api/medications/extract/__tests__/route.test.ts b/src/app/api/medications/extract/__tests__/route.test.ts index b0d454ef5..d80ee7565 100644 --- a/src/app/api/medications/extract/__tests__/route.test.ts +++ b/src/app/api/medications/extract/__tests__/route.test.ts @@ -122,6 +122,7 @@ beforeEach(() => { totalAfter: 700, owner: "user", operatorAfter: 0, + limit: null, }); vi.mocked(resolveDailyCap).mockReturnValue(1_234_567); vi.mocked(resolveProviderChain).mockResolvedValue([ @@ -279,6 +280,7 @@ describe("POST /api/medications/extract — daily budget", () => { "2026-05-28", 1_234_567, "user", + "coach", ); }); @@ -290,6 +292,7 @@ describe("POST /api/medications/extract — daily budget", () => { totalAfter: 9_999_999, owner: "user", operatorAfter: 0, + limit: "owner-cap", }); const res = await POST(postReq({ text: "Mounjaro 5mg weekly" }) as never); diff --git a/src/app/api/medications/extract/route.ts b/src/app/api/medications/extract/route.ts index 883de5d55..4fe35ca8a 100644 --- a/src/app/api/medications/extract/route.ts +++ b/src/app/api/medications/extract/route.ts @@ -212,6 +212,7 @@ async function handleExtract(request: NextRequest): Promise { dateKey, resolveDailyCap(chain), resolveCostOwner(chain), + "coach", ); if (!reservation.allowed) { annotate({ @@ -224,6 +225,7 @@ async function handleExtract(request: NextRequest): Promise { let completion; try { completion = await runRawCompletionWithFallback({ + surface: "coach", userId, providers: chain, params: singleUserTurn({ diff --git a/src/lib/ai/__tests__/gateway-embedded-error.test.ts b/src/lib/ai/__tests__/gateway-embedded-error.test.ts index 2c7180c45..391710c13 100644 --- a/src/lib/ai/__tests__/gateway-embedded-error.test.ts +++ b/src/lib/ai/__tests__/gateway-embedded-error.test.ts @@ -280,6 +280,7 @@ describe("the chain runner and the health ledger", () => { const failure = (await runRawCompletionWithFallback({ userId: "u1", + surface: "coach", providers: [ { providerType: "openai-compatible", @@ -318,6 +319,7 @@ describe("the chain runner and the health ledger", () => { const failure = (await runRawCompletionWithFallback({ userId: "u2", + surface: "coach", providers: [ { providerType: "openai-compatible", @@ -350,6 +352,7 @@ describe("the chain runner and the health ledger", () => { const outcome = await runRawCompletionWithFallback({ userId: "u3", + surface: "coach", providers: [ { providerType: "openai-compatible", diff --git a/src/lib/ai/__tests__/provider-runner.test.ts b/src/lib/ai/__tests__/provider-runner.test.ts index bb5509aed..3ff5d781a 100644 --- a/src/lib/ai/__tests__/provider-runner.test.ts +++ b/src/lib/ai/__tests__/provider-runner.test.ts @@ -173,6 +173,7 @@ describe("runRawCompletionWithFallback — last-working cache", () => { // First call: codex fails, openai succeeds. await runRawCompletionWithFallback({ userId: "u-cache", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "openai", instance: openai }, @@ -186,6 +187,7 @@ describe("runRawCompletionWithFallback — last-working cache", () => { // never re-invoked even though it appears first in the input chain. await runRawCompletionWithFallback({ userId: "u-cache", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "openai", instance: openai }, @@ -216,6 +218,7 @@ describe("runRawCompletionWithFallback — last-working cache", () => { // First call: codex fails, openai succeeds, cache = openai. await runRawCompletionWithFallback({ userId: "u-ttl", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "openai", instance: openai }, @@ -232,6 +235,7 @@ describe("runRawCompletionWithFallback — last-working cache", () => { // and this time codex's second scripted entry succeeds). await runRawCompletionWithFallback({ userId: "u-ttl", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "openai", instance: openai }, @@ -248,6 +252,7 @@ describe("runRawCompletionWithFallback — last-working cache", () => { }); await runRawCompletionWithFallback({ userId: "u-clear", + surface: "coach", providers: [{ providerType: "openai", instance: ok }], params: singleUserTurn({ system: "s", user: "u" }), }); @@ -269,6 +274,7 @@ describe("runRawCompletionWithFallback — legacy route shim", () => { }); const result = await runRawCompletionWithFallback({ userId: "u-raw", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "admin-openai", instance: openai }, @@ -296,6 +302,7 @@ describe("runRawCompletionWithFallback — legacy route shim", () => { }); const result = await runRawCompletionWithFallback({ userId: "u-raw-auth-advance", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "openai", instance: openai }, @@ -324,6 +331,7 @@ describe("runRawCompletionWithFallback — legacy route shim", () => { try { await runRawCompletionWithFallback({ userId: "u-raw-allfail", + surface: "coach", providers: [ { providerType: "codex", instance: a }, { providerType: "admin-openai", instance: b }, @@ -361,6 +369,7 @@ describe("runRawCompletionWithFallback — legacy route shim", () => { try { await runRawCompletionWithFallback({ userId: "u-raw-codex400", + surface: "coach", providers: [{ providerType: "codex", instance: codex }], params: singleUserTurn({ system: "s", user: "u" }), }); @@ -383,6 +392,7 @@ describe("runRawCompletionWithFallback — empty input", () => { try { await runRawCompletionWithFallback({ userId: "u-empty", + surface: "coach", providers: [], params: singleUserTurn({ system: "s", user: "u" }), }); @@ -417,6 +427,7 @@ describe("provider-health ledger — auth-failure negative cache", () => { // First call: codex 401 → openai succeeds. Ledger now benches codex. await runRawCompletionWithFallback({ userId: "u-neg", + surface: "coach", providers, params: singleUserTurn({ system: "s", user: "u" }), ledger, @@ -434,6 +445,7 @@ describe("provider-health ledger — auth-failure negative cache", () => { const codexCallsBefore = codex.callCount; await runRawCompletionWithFallback({ userId: "u-neg", + surface: "coach", providers, params: singleUserTurn({ system: "s", user: "u" }), ledger, @@ -453,6 +465,7 @@ describe("provider-health ledger — auth-failure negative cache", () => { // A direct success on codex (e.g. user re-linked) clears the row. await runRawCompletionWithFallback({ userId: "u-clear-neg", + surface: "coach", providers: [{ providerType: "codex", instance: codex }], params: singleUserTurn({ system: "s", user: "u" }), ledger, @@ -488,6 +501,7 @@ describe("provider-health ledger — local model as guaranteed floor", () => { const result = await runRawCompletionWithFallback({ userId: "u-floor", + surface: "coach", providers: [ { providerType: "codex", instance: dead("codex") }, { providerType: "openai", instance: dead("admin-key") }, @@ -517,6 +531,7 @@ describe("provider-health ledger — local model as guaranteed floor", () => { try { await runRawCompletionWithFallback({ userId: "u-local-hard", + surface: "coach", providers: [{ providerType: "local", instance: local }], params: singleUserTurn({ system: "s", user: "u" }), ledger, @@ -531,6 +546,7 @@ describe("provider-health ledger — local model as guaranteed floor", () => { clearLastWorkingProviderCache(); const result = await runRawCompletionWithFallback({ userId: "u-local-hard", + surface: "coach", providers: [{ providerType: "local", instance: local }], params: singleUserTurn({ system: "s", user: "u" }), ledger, @@ -554,6 +570,7 @@ describe("AllProvidersFailedError — primaryCredentialExpired", () => { try { await runRawCompletionWithFallback({ userId: "u-primary-auth", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "admin-openai", instance: openai }, @@ -582,6 +599,7 @@ describe("AllProvidersFailedError — primaryCredentialExpired", () => { try { await runRawCompletionWithFallback({ userId: "u-primary-5xx", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "admin-openai", instance: openai }, @@ -637,6 +655,7 @@ describe("runStreamingRawCompletionWithFallback", () => { const deltas: string[] = []; const result = await runStreamingRawCompletionWithFallback({ userId: "u-stream", + surface: "coach", providers: [{ providerType: "local", instance: provider }], params: singleUserTurn({ system: "s", user: "u", timeoutMs: 12345 }), onDelta: (d) => deltas.push(d), @@ -659,6 +678,7 @@ describe("runStreamingRawCompletionWithFallback", () => { const deltas: string[] = []; const result = await runStreamingRawCompletionWithFallback({ userId: "u-nostream", + surface: "coach", providers: [{ providerType: "local", instance: provider }], params: singleUserTurn({ system: "s", user: "u" }), onDelta: (d) => deltas.push(d), @@ -677,6 +697,7 @@ describe("runStreamingRawCompletionWithFallback", () => { const good = new StreamingProvider("ok"); const result = await runStreamingRawCompletionWithFallback({ userId: "u-cascade", + surface: "coach", providers: [ { providerType: "openai", instance: broken }, { providerType: "local", instance: good }, @@ -711,6 +732,7 @@ describe("runRawCompletionWithFallback — operator-cost cap at hop time", () => await expect( runRawCompletionWithFallback({ userId: "u-cap", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "admin-openai", instance: adminOpenai }, @@ -741,6 +763,7 @@ describe("runRawCompletionWithFallback — operator-cost cap at hop time", () => const outcome = await runRawCompletionWithFallback({ userId: "u-headroom", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "admin-openai", instance: adminOpenai }, @@ -768,6 +791,7 @@ describe("runRawCompletionWithFallback — operator-cost cap at hop time", () => const outcome = await runRawCompletionWithFallback({ userId: "u-owner-split", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "admin-openai", instance: adminOpenai }, @@ -790,6 +814,106 @@ describe("runRawCompletionWithFallback — operator-cost cap at hop time", () => await expect( runRawCompletionWithFallback({ userId: "u-owner-exhausted", + surface: "coach", + providers: [ + { providerType: "codex", instance: codex }, + { providerType: "admin-openai", instance: adminOpenai }, + ], + params, + }), + ).rejects.toMatchObject({ + attempts: [ + expect.objectContaining({ providerType: "codex" }), + expect.objectContaining({ + providerType: "admin-openai", + failureReason: "operator-cost-cap-exhausted", + }), + ], + }); + expect(adminOpenai.callCount).toBe(0); + }); + + // v1.38.19 (Wave E, fix round 1) — the hop guard knows which SURFACE asked. + // + // Without this the job share was enforced at reservation time only: a + // background reservation admitted under `resolveDailyCapFor("job", …)` could + // then reach an operator-funded FALLBACK hop and spend against the full + // 200 000. That is the original defect by another route — chain + // `[codex, admin-openai]` reserves under the user-plan ceiling, `codex` 429s, + // every job falls back onto the shared key, and background work fills the + // operator's ceiling before the user opens the chat. + it("refuses a JOB's admin-* fallback hop at the job share", async () => { + budgetState.spent = 150_000; + budgetState.operatorSpent = 150_000; + const codex = new ScriptedProvider({ + type: "codex", + script: [{ ok: false, error: err(500) }], + }); + const adminOpenai = new ScriptedProvider({ script: [{ ok: true }] }); + + await expect( + runRawCompletionWithFallback({ + userId: "u-job-share", + surface: "job", + providers: [ + { providerType: "codex", instance: codex }, + { providerType: "admin-openai", instance: adminOpenai }, + ], + params, + }), + ).rejects.toMatchObject({ + attempts: [ + expect.objectContaining({ providerType: "codex" }), + expect.objectContaining({ + providerType: "admin-openai", + failureReason: "operator-cost-cap-exhausted", + }), + ], + }); + expect(adminOpenai.callCount).toBe(0); + }); + + it("still lets the interactive chat take that same hop", async () => { + // The whole point of the share: at 150 000 operator tokens the background + // work is done for the day and the person waiting is not. + budgetState.spent = 150_000; + budgetState.operatorSpent = 150_000; + const codex = new ScriptedProvider({ + type: "codex", + script: [{ ok: false, error: err(500) }], + }); + const adminOpenai = new ScriptedProvider({ script: [{ ok: true }] }); + + const outcome = await runRawCompletionWithFallback({ + userId: "u-job-share-chat", + surface: "coach", + providers: [ + { providerType: "codex", instance: codex }, + { providerType: "admin-openai", instance: adminOpenai }, + ], + params, + }); + expect(outcome.workingProvider.providerType).toBe("admin-openai"); + expect(adminOpenai.callCount).toBe(1); + }); + + it("bounds the operator hop on the day's mixed total as well", async () => { + // `operator_tokens` returns to ~0 on every reconcile the user's own plan + // settled, so the operator arm alone leaves a runaway loop unbounded. The + // abuse ceiling on the total is checked at the hop for the same reason it + // is checked at reservation time. + budgetState.spent = 2_000_000; + budgetState.operatorSpent = 0; + const codex = new ScriptedProvider({ + type: "codex", + script: [{ ok: false, error: err(500) }], + }); + const adminOpenai = new ScriptedProvider({ script: [{ ok: true }] }); + + await expect( + runRawCompletionWithFallback({ + userId: "u-total-ceiling", + surface: "coach", providers: [ { providerType: "codex", instance: codex }, { providerType: "admin-openai", instance: adminOpenai }, @@ -813,6 +937,7 @@ describe("runRawCompletionWithFallback — operator-cost cap at hop time", () => const openai = new ScriptedProvider({ script: [{ ok: true }] }); const outcome = await runRawCompletionWithFallback({ userId: "u-own-key", + surface: "coach", providers: [{ providerType: "openai", instance: openai }], params, }); diff --git a/src/lib/ai/coach/__tests__/budget.test.ts b/src/lib/ai/coach/__tests__/budget.test.ts index 62bb3ad76..2a7f764b4 100644 --- a/src/lib/ai/coach/__tests__/budget.test.ts +++ b/src/lib/ai/coach/__tests__/budget.test.ts @@ -8,6 +8,7 @@ import { resolveCostOwner, resolveDailyCap, resolveDailyCapFor, + resolveTotalCapFor, } from "../budget"; vi.mock("@/lib/db", () => ({ @@ -69,6 +70,7 @@ describe("reserveBudget — ledger clamps", () => { "2026-05-10", OPERATOR_COST_CAP, "operator", + "coach", ); expect(res.reserved).toBe(0); }); @@ -81,6 +83,7 @@ describe("reserveBudget — ledger clamps", () => { "2026-05-10", OPERATOR_COST_CAP, "operator", + "coach", ); expect(res.reserved).toBe(0); }); @@ -93,6 +96,7 @@ describe("reserveBudget — ledger clamps", () => { "2026-05-10", OPERATOR_COST_CAP, "operator", + "coach", ); expect(res.reserved).toBe(12); }); @@ -165,6 +169,7 @@ describe("reserveBudget cap (F1 — user-plan path not locked out)", () => { "2026-05-10", resolveDailyCap([{ providerType: "codex" }]), resolveCostOwner([{ providerType: "codex" }]), + "coach", ); expect(res.allowed).toBe(true); // The reservation upsert ran; no refund executeRaw fired. @@ -184,6 +189,7 @@ describe("reserveBudget cap (F1 — user-plan path not locked out)", () => { "2026-05-10", resolveDailyCap([{ providerType: "admin-openai" }]), resolveCostOwner([{ providerType: "admin-openai" }]), + "coach", ); expect(res.allowed).toBe(false); // Refund of the reservation fired on refusal. @@ -227,10 +233,37 @@ describe("resolveDailyCapFor (Wave E — the job share)", () => { ); }); - it("shares the user-plan ceiling the same way", () => { + it("does not ration a chain the operator does not pay for", () => { + // The share exists to protect the OPERATOR's invoice. A self-hoster on a + // local model, or a user on their own ChatGPT plan, costs the operator + // nothing — halving their background generation is a silent downgrade with + // no bill behind it. expect(resolveDailyCapFor("job", [{ providerType: "codex" }])).toBe( + USER_PLAN_CAP, + ); + expect(resolveDailyCapFor("job", [{ providerType: "local" }])).toBe( + USER_PLAN_CAP, + ); + }); +}); + +describe("resolveTotalCapFor (Wave E, fix round 1 — the abuse ceiling)", () => { + it("keeps the user-plan ceiling on the day's mixed total for both owners", () => { + // The operator arm compares `operator_tokens`, a counter that returns to + // ~0 on every reconcile the user's own plan settled. Without a ceiling on + // the total, an operator-primary chain has no ceiling at all: every job + // reservation is admitted forever and a runaway client loop writes an + // unbounded row. + expect(resolveTotalCapFor("coach", "operator")).toBe(USER_PLAN_CAP); + expect(resolveTotalCapFor("coach", "user")).toBe(USER_PLAN_CAP); + }); + + it("gives a background surface its share of the total ceiling too", () => { + expect(resolveTotalCapFor("job", "operator")).toBe( USER_PLAN_CAP * JOB_SURFACE_SHARE, ); + // Nothing to ration when the operator is not paying. + expect(resolveTotalCapFor("job", "user")).toBe(USER_PLAN_CAP); }); }); @@ -260,6 +293,7 @@ describe("reserveBudget cap by cost owner (Wave E)", () => { "2026-09-11", OPERATOR_COST_CAP, "operator", + "coach", ); expect(res.allowed).toBe(true); expect(res.operatorAfter).toBe(151_200); @@ -280,6 +314,7 @@ describe("reserveBudget cap by cost owner (Wave E)", () => { "2026-09-11", OPERATOR_COST_CAP, "operator", + "coach", ); expect(res.allowed).toBe(false); expect(prismaMock.$executeRaw).toHaveBeenCalled(); @@ -296,9 +331,91 @@ describe("reserveBudget cap by cost owner (Wave E)", () => { "2026-09-11", USER_PLAN_CAP, "user", + "coach", ); expect(res.allowed).toBe(false); }); + + // v1.38.19 (Wave E, fix round 1) — the abuse ceiling survives the owner split. + // + // The first cut compared `operator_tokens` and NOTHING else for an + // operator-funded chain. On the 2026-09-11 shape that counter never grows: + // `admin-openai` 500s, `codex` serves, and every reconcile moves the + // reservation back out of the operator's column. So an operator-primary chain + // had no ceiling at all — every job reservation admitted forever, and a + // runaway client loop free to write an unbounded row. + it("refuses a JOB on an operator chain once the day's TOTAL fills its share", async () => { + prismaMock.$queryRaw.mockResolvedValue([ + { total_tokens: 1_900_000 + 1_200, operator_tokens: 0 }, + ]); + const { reserveBudget, resolveDailyCapFor } = await import("../budget"); + const res = await reserveBudget( + "u", + 1_200, + "2026-09-11", + resolveDailyCapFor("job", [{ providerType: "admin-openai" }]), + "operator", + "job", + ); + expect(res.allowed).toBe(false); + expect(res.limit).toBe("total-cap"); + expect(prismaMock.$executeRaw).toHaveBeenCalled(); + }); + + it("still admits the interactive chat on that same day", async () => { + prismaMock.$queryRaw.mockResolvedValue([ + { total_tokens: 1_900_000 + 1_200, operator_tokens: 0 }, + ]); + const { reserveBudget } = await import("../budget"); + const res = await reserveBudget( + "u", + 1_200, + "2026-09-11", + OPERATOR_COST_CAP, + "operator", + "coach", + ); + expect(res.allowed).toBe(true); + expect(res.limit).toBeNull(); + }); + + it("refuses an operator chain once the TOTAL reaches the abuse ceiling", async () => { + prismaMock.$queryRaw.mockResolvedValue([ + { total_tokens: USER_PLAN_CAP + 1_200, operator_tokens: 0 }, + ]); + const { reserveBudget } = await import("../budget"); + const res = await reserveBudget( + "u", + 1_200, + "2026-09-11", + OPERATOR_COST_CAP, + "operator", + "coach", + ); + expect(res.allowed).toBe(false); + expect(res.limit).toBe("total-cap"); + }); + + it("names the owner cap when that is the ceiling that tripped", async () => { + prismaMock.$queryRaw.mockResolvedValue([ + { + total_tokens: 300_000 + 1_200, + operator_tokens: OPERATOR_COST_CAP + 1_200, + }, + ]); + const { reserveBudget } = await import("../budget"); + const res = await reserveBudget( + "u", + 1_200, + "2026-09-11", + OPERATOR_COST_CAP, + "operator", + "coach", + ); + expect(res.allowed).toBe(false); + expect(res.limit).toBe("owner-cap"); + expect(res.operatorAfter).toBe(OPERATOR_COST_CAP); + }); }); describe("reconcileSpend cached-token subtraction (F3)", () => { diff --git a/src/lib/ai/coach/__tests__/self-context-questions-consent.test.ts b/src/lib/ai/coach/__tests__/self-context-questions-consent.test.ts index f3910f854..5bc67d264 100644 --- a/src/lib/ai/coach/__tests__/self-context-questions-consent.test.ts +++ b/src/lib/ai/coach/__tests__/self-context-questions-consent.test.ts @@ -246,6 +246,7 @@ describe("deriveClarifyingQuestions — atomic budget", () => { "2026-07-18", 200_000, "operator", + "coach", ); expect(budgetMocks.reconcileSpend).toHaveBeenCalledWith( "user-1", diff --git a/src/lib/ai/coach/budget.ts b/src/lib/ai/coach/budget.ts index 09e80ab59..e2b77969c 100644 --- a/src/lib/ai/coach/budget.ts +++ b/src/lib/ai/coach/budget.ts @@ -17,6 +17,16 @@ import { prisma } from "@/lib/db"; import type { ProviderChainType } from "@/lib/ai/provider-chain"; +/** + * v1.38.19 (Wave E, fix round 1) — the two raw statements the ledger owns can + * run on the singleton client OR inside an open transaction. The arrival + * reaction reserves its tokens in the same transaction that links the + * reservation to the claimed marker, and used to carry its own copy of the + * upsert to do it; that copy drifted twice. One implementation, an executor + * argument, no second copy to drift. + */ +export type BudgetExecutor = Pick; + /** * Operator-cost daily ceiling — the cap that protects the OPERATOR's LLM bill. * @@ -80,6 +90,35 @@ export function resolveDailyCap( */ export const JOB_SURFACE_SHARE = 0.5; +/** + * v1.38.19 (Wave E) — which kind of caller is asking. `"coach"` is every + * interactive surface (the chat, the extraction routes, the document routes, + * the connection probe): a person is waiting. `"job"` is the automatic + * generators, which run with nobody waiting and must leave the interactive + * surfaces a day of their own. + */ +export type BudgetSurface = "coach" | "job"; + +/** + * v1.38.19 (Wave E, fix round 1) — the job share rations the OPERATOR's + * invoice, so it applies only when the operator is paying. + * + * The first cut multiplied whichever ceiling `resolveDailyCap` returned, which + * halved background generation for a self-hoster on a local model — zero + * marginal cost to anyone, and an unannounced downgrade for exactly the + * audience this project is built for. The share's whole justification is the + * operator's bill; it does not transfer to a chain the operator does not fund. + */ +function applyJobShare( + surface: BudgetSurface, + chain: ReadonlyArray<{ providerType: ProviderChainType }>, + cap: number, +): number { + return surface === "job" && resolveCostOwner(chain) === "operator" + ? Math.floor(cap * JOB_SURFACE_SHARE) + : cap; +} + /** * v1.38.19 (Wave E) — the daily ceiling for one SURFACE on a chain. `"coach"` * (every interactive surface: the chat, the extraction routes, the document @@ -89,11 +128,33 @@ export const JOB_SURFACE_SHARE = 0.5; * claim. */ export function resolveDailyCapFor( - surface: "coach" | "job", + surface: BudgetSurface, chain: ReadonlyArray<{ providerType: ProviderChainType }>, ): number { - const cap = resolveDailyCap(chain); - return surface === "job" ? Math.floor(cap * JOB_SURFACE_SHARE) : cap; + return applyJobShare(surface, chain, resolveDailyCap(chain)); +} + +/** + * v1.38.19 (Wave E, fix round 1) — the ABUSE ceiling on the day's mixed total, + * for one surface and one cost owner. + * + * `resolveDailyCapFor` answers "how much of the owner's ceiling may this + * surface claim", and for an operator-funded chain that ceiling is measured + * against `operator_tokens` — a counter that stays near zero whenever the + * operator's key fails and the user's own plan serves the fallback. That is + * the correct answer for the operator's invoice and a useless one for the row: + * on the 2026-09-11 evidence the operator counter never grew, so nothing + * bounded the 1.0–1.45 M tokens a day of background generation, and nothing + * bounded a runaway client loop either. `USER_PLAN_CAP` stays the ceiling on + * the total in BOTH arms, and a background surface gets its share of that too. + */ +export function resolveTotalCapFor( + surface: BudgetSurface, + owner: BudgetCostOwner, +): number { + return surface === "job" && owner === "operator" + ? Math.floor(USER_PLAN_CAP * JOB_SURFACE_SHARE) + : USER_PLAN_CAP; } /** @@ -211,6 +272,17 @@ export interface ReserveBudgetResult { owner: BudgetCostOwner; /** The day's operator-funded spend AFTER this reservation. */ operatorAfter: number; + /** + * v1.38.19 (Wave E, fix round 1) — which ceiling refused this reservation, + * `null` when it was admitted. `"owner-cap"` is the cost owner's daily + * ceiling (the operator's invoice, or the user's own plan); `"total-cap"` is + * the abuse ceiling on the day's mixed total. The refusal annotations carry + * it: before this, every refusal printed one number, and on an operator + * refusal that number was the day's mixed total rather than the counter that + * tripped — the same unreadable signal that made the 06:42Z diagnosis a + * production log dig. + */ + limit: "owner-cap" | "total-cap" | null; } export async function reserveBudget( @@ -219,6 +291,8 @@ export async function reserveBudget( dateKey: string, cap: number, owner: BudgetCostOwner, + surface: BudgetSurface, + db: BudgetExecutor = prisma, ): Promise { const reserved = Number.isFinite(estimatedTokens) && estimatedTokens > 0 @@ -234,7 +308,7 @@ export async function reserveBudget( // requests serialise on the row's unique (user_id, date_key) constraint, so // each observes a distinct post-increment total — they cannot both read a // sub-cap value and both proceed. - const rows = await prisma.$queryRaw< + const rows = await db.$queryRaw< { total_tokens: number; operator_tokens: number }[] >` INSERT INTO coach_usage (id, user_id, date_key, total_tokens, operator_tokens, message_count, created_at, updated_at) @@ -261,20 +335,37 @@ export async function reserveBudget( const priorTotal = totalAfter - reserved; const priorOperator = operatorAfter - operatorReserved; const prior = owner === "operator" ? priorOperator : priorTotal; - if (prior >= cap) { + // v1.38.19 (Wave E, fix round 1) — BOTH ceilings, always. The owner's cap + // protects whoever pays; the total cap is the abuse ceiling that keeps a + // runaway loop — or a day of background generation the operator's key never + // served — from writing an unbounded row. Dropping the second one for + // operator chains left them with no ceiling at all, because their counter + // returns to ~0 on every reconcile the user's own plan settled. + const totalCap = resolveTotalCapFor(surface, owner); + const limit: ReserveBudgetResult["limit"] = + prior >= cap ? "owner-cap" : priorTotal >= totalCap ? "total-cap" : null; + if (limit !== null) { // Already over before this request — refund the reservation + the // message-count bump and refuse. - await refundReservation(userId, reserved, operatorReserved, dateKey); + await refundReservation(userId, reserved, operatorReserved, dateKey, db); return { allowed: false, reserved, totalAfter: priorTotal, owner, operatorAfter: priorOperator, + limit, }; } - return { allowed: true, reserved, totalAfter, owner, operatorAfter }; + return { + allowed: true, + reserved, + totalAfter, + owner, + operatorAfter, + limit: null, + }; } /** @@ -357,8 +448,9 @@ async function refundReservation( reserved: number, operatorReserved: number, dateKey: string, + db: BudgetExecutor = prisma, ): Promise { - await prisma.$executeRaw` + await db.$executeRaw` UPDATE coach_usage SET total_tokens = GREATEST(0, total_tokens - ${reserved}), operator_tokens = GREATEST(0, operator_tokens - ${operatorReserved}), diff --git a/src/lib/ai/coach/self-context-questions.ts b/src/lib/ai/coach/self-context-questions.ts index e9b3022ea..34cd3b0e2 100644 --- a/src/lib/ai/coach/self-context-questions.ts +++ b/src/lib/ai/coach/self-context-questions.ts @@ -237,6 +237,7 @@ export async function deriveClarifyingQuestions( dateKey, resolveDailyCap(chain), resolveCostOwner(chain), + "coach", ); if (!reservation.allowed) return fallback(); diff --git a/src/lib/ai/coach/tools/loop.ts b/src/lib/ai/coach/tools/loop.ts index 61076777f..cd671c0b6 100644 --- a/src/lib/ai/coach/tools/loop.ts +++ b/src/lib/ai/coach/tools/loop.ts @@ -141,6 +141,8 @@ export async function runCoachToolLoop(args: { const fallback = await runRawCompletionWithFallback({ userId, providers, + // The tool loop IS the Coach chat: a person is waiting on every round. + surface: "coach", ledger, params: { system, diff --git a/src/lib/ai/provider-runner.ts b/src/lib/ai/provider-runner.ts index e414423fd..dd2dfec4b 100644 --- a/src/lib/ai/provider-runner.ts +++ b/src/lib/ai/provider-runner.ts @@ -8,8 +8,10 @@ import { import { annotate } from "@/lib/logging/context"; import { isOperatorFundedProvider, - OPERATOR_COST_CAP, readDailySpend, + resolveDailyCapFor, + resolveTotalCapFor, + type BudgetSurface, } from "@/lib/ai/coach/budget"; /** @@ -399,6 +401,7 @@ async function runRawChain( userId: string; providers: ProviderChainResolved[]; params: CompletionParams; + surface: BudgetSurface; ledger?: ProviderHealthLedger; }, invoke: (candidate: ProviderChainResolved) => Promise, @@ -406,7 +409,7 @@ async function runRawChain( candidate: ProviderChainResolved, ) => Record, ): Promise { - const { userId, providers } = args; + const { userId, providers, surface } = args; const ledger = args.ledger ?? postgresProviderHealthLedger; if (providers.length === 0) { @@ -438,7 +441,21 @@ async function runRawChain( // v1.38.19 (Wave E) — the OPERATOR-funded share, not the day's total: a // day filled by the user's own plan must not close the operator's // fallback hop on money the operator never spent. - if (spent.operator >= OPERATOR_COST_CAP) { + // + // v1.38.19 (Wave E, fix round 1) — and at the SURFACE's ceiling, not the + // full one. A background reservation is rationed at the job share when it + // is taken; without this the hop it reaches by fallback could still spend + // against the whole operator cap, which is how the original defect stayed + // reachable. Chain `[codex, admin-openai]` reserves under the user-plan + // ceiling, `codex` 429s all morning, every job falls back onto the shared + // key, and background work walks `operator_tokens` to 200 000 before the + // user opens the chat. The abuse ceiling on the day's mixed total is + // checked here for the same reason it is checked at reservation time. + const hopCap = resolveDailyCapFor(surface, [ + { providerType: candidate.providerType }, + ]); + const hopTotalCap = resolveTotalCapFor(surface, "operator"); + if (spent.operator >= hopCap || spent.total >= hopTotalCap) { const hop: FallbackHop = { providerType: candidate.providerType, attempt: i + 1, @@ -452,6 +469,8 @@ async function runRawChain( [`ai_chain_hop_${i + 1}_provider`]: candidate.providerType, [`ai_chain_hop_${i + 1}_reason`]: "operator-cost-cap-exhausted", operator_cap_spent: spent.operator, + operator_cap_limit: hopCap, + ai_chain_surface: surface, }, }); continue; @@ -520,6 +539,12 @@ export async function runStreamingRawCompletionWithFallback(args: { userId: string; providers: ProviderChainResolved[]; params: CompletionParams; + /** + * v1.38.19 (Wave E, fix round 1) — which ceiling an operator-funded FALLBACK + * hop is rationed against. `"job"` gets the background share, exactly as the + * reservation did; `"coach"` gets the whole ceiling. + */ + surface: BudgetSurface; /** Called once per streamed token/chunk for providers that stream. */ onDelta: (delta: string) => void; /** See `RunWithFallbackParams.ledger`. */ @@ -566,6 +591,8 @@ export async function runRawCompletionWithFallback(args: { userId: string; providers: ProviderChainResolved[]; params: CompletionParams; + /** See {@link runStreamingRawCompletionWithFallback}'s `surface`. */ + surface: BudgetSurface; /** See `RunWithFallbackParams.ledger`. */ ledger?: ProviderHealthLedger; }): Promise { diff --git a/src/lib/documents/auto-stage-labs.ts b/src/lib/documents/auto-stage-labs.ts index 2188d4795..d77c9391f 100644 --- a/src/lib/documents/auto-stage-labs.ts +++ b/src/lib/documents/auto-stage-labs.ts @@ -289,6 +289,7 @@ export async function maybeAutoStageLabFacts( { providerType: provider.pick.entry.providerType }, ]), provider.costOwner, + "job", ); if (!reservation.allowed) return { staged: false, reason: "budget" }; diff --git a/src/lib/documents/fenced-chat.ts b/src/lib/documents/fenced-chat.ts index 9e70b7863..9b8e63e04 100644 --- a/src/lib/documents/fenced-chat.ts +++ b/src/lib/documents/fenced-chat.ts @@ -309,6 +309,7 @@ export async function streamFencedReply( dateKey, resolveDailyCap([{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "coach", ); if (!reservation.allowed) { annotate({ action: { name: "documents.chat.budget.exceeded" } }); @@ -371,6 +372,7 @@ Reply now as the assistant, grounded ONLY in the documents above, in ${ let result; try { const fallback = await runStreamingRawCompletionWithFallback({ + surface: "coach", userId, // Single-provider — the document-ordered pick, no cascade. providers: [pick!.entry], diff --git a/src/lib/documents/index-document.ts b/src/lib/documents/index-document.ts index 1c222c7c8..76b0a28a3 100644 --- a/src/lib/documents/index-document.ts +++ b/src/lib/documents/index-document.ts @@ -236,6 +236,7 @@ async function tryProviderIndex( dateKey, provider.dailyCap, provider.costOwner, + "coach", ); // Budget exhausted → fall through to the free local path rather than stall; // a text-layer PDF stays searchable even once the AI allowance is spent. diff --git a/src/lib/insights/__tests__/status-provider-budget.test.ts b/src/lib/insights/__tests__/status-provider-budget.test.ts index fa3568716..c756c3dac 100644 --- a/src/lib/insights/__tests__/status-provider-budget.test.ts +++ b/src/lib/insights/__tests__/status-provider-budget.test.ts @@ -151,6 +151,7 @@ describe("runStatusCompletion — ledger accounting", () => { "2026-09-11", resolveDailyCap(chatChain), resolveCostOwner(chatChain), + "coach", ); expect(chat.allowed).toBe(true); }); @@ -176,9 +177,21 @@ describe("runStatusCompletion — ledger accounting", () => { // "no provider configured" assessment, which a budget refusal is not. expect(runRawCompletionWithFallback).not.toHaveBeenCalled(); expect(result.kind).toBe("error"); + // v1.38.19 (Wave E, fix round 1) — the refusal has to SAY which ceiling it + // hit and whose. `{ totalAfter }` alone was the day's mixed total, which on + // an operator refusal is not the counter that tripped; and the shared + // `error` outcome cannot distinguish an exhausted background share from a + // dead provider, so the annotation is the only place that can. expect(annotate).toHaveBeenCalledWith( expect.objectContaining({ action: { name: "insights.status.budget_exceeded" }, + meta: expect.objectContaining({ + owner: "operator", + surface: "job", + limit: "owner-cap", + cap: OPERATOR_COST_CAP * 0.5, + operatorAfter: OPERATOR_COST_CAP, + }), }), ); // The refused reservation was refunded, not left on the row. diff --git a/src/lib/insights/briefing-provider.ts b/src/lib/insights/briefing-provider.ts index fc248e038..16c2a2c42 100644 --- a/src/lib/insights/briefing-provider.ts +++ b/src/lib/insights/briefing-provider.ts @@ -118,18 +118,29 @@ export async function runBriefingCompletion( args.maxTokens + Math.ceil((args.systemPrompt.length + args.userPrompt.length) / 4); + // v1.38.19 (Wave E) — a background surface: half the day's ceiling when the + // operator funds the chain. + const jobCap = resolveDailyCapFor("job", args.chain); const reservation = await reserveBudget( args.userId, estimatedTokens, dateKey, - // v1.38.19 (Wave E) — a background surface: half the day's ceiling. - resolveDailyCapFor("job", args.chain), + jobCap, resolveCostOwner(args.chain), + "job", ); if (!reservation.allowed) { annotate({ action: { name: "insights.briefing.budget_exceeded" }, - meta: { stage: args.stage, totalAfter: reservation.totalAfter }, + meta: { + stage: args.stage, + owner: reservation.owner, + surface: "job", + limit: reservation.limit, + cap: jobCap, + totalAfter: reservation.totalAfter, + operatorAfter: reservation.operatorAfter, + }, }); throw new BriefingBudgetExceededError(args.stage, reservation.totalAfter); } @@ -138,6 +149,9 @@ export async function runBriefingCompletion( try { outcome = await runRawCompletionWithFallback({ userId: args.userId, + // A background generator: an operator-funded fallback hop is rationed at + // the job share, exactly as the reservation above was. + surface: "job", providers: args.chain, params: singleUserTurn({ system: args.systemPrompt, diff --git a/src/lib/insights/status-provider.ts b/src/lib/insights/status-provider.ts index 109f7145d..6f63fba14 100644 --- a/src/lib/insights/status-provider.ts +++ b/src/lib/insights/status-provider.ts @@ -219,13 +219,16 @@ export async function runStatusCompletion( const dateKey = buildDateKey(); const estimatedTokens = maxTokens + Math.ceil((systemPrompt.length + userPrompt.length) / 4); + // v1.38.19 (Wave E) — a background surface: half the day's ceiling when the + // operator funds the chain. + const jobCap = resolveDailyCapFor("job", chain); const reservation = await reserveBudget( userId, estimatedTokens, dateKey, - // v1.38.19 (Wave E) — a background surface: half the day's ceiling. - resolveDailyCapFor("job", chain), + jobCap, resolveCostOwner(chain), + "job", ); if (!reservation.allowed) { // Over the day's ceiling. Reported as `error` — a TRANSIENT miss the caller @@ -233,9 +236,22 @@ export async function runStatusCompletion( // which callers cache as the settled "no provider configured" assessment. // The distinct annotation keeps the refusal observable even though the // result shape is shared. + // v1.38.19 (Wave E, fix round 1) — the shared `error` outcome cannot say + // WHY the generation stopped, so the annotation has to. `surface: "job"` + // plus `cap` separates "this job used up the background share" from "the + // whole day is spent", and `limit` says which counter tripped. The operator + // sees the same split on the admin provider-health card. annotate({ action: { name: "insights.status.budget_exceeded" }, - meta: { cacheAction, totalAfter: reservation.totalAfter }, + meta: { + cacheAction, + owner: reservation.owner, + surface: "job", + limit: reservation.limit, + cap: jobCap, + totalAfter: reservation.totalAfter, + operatorAfter: reservation.operatorAfter, + }, }); return { kind: "error" }; } @@ -245,6 +261,9 @@ export async function runStatusCompletion( runRawCompletionWithFallback({ userId, providers: chain, + // A background generator: an operator-funded fallback hop is rationed + // at the job share, exactly as the reservation above was. + surface: "job", params: singleUserTurn({ system: systemPrompt, user: userPrompt, diff --git a/src/lib/jobs/coach-nudge-ai.ts b/src/lib/jobs/coach-nudge-ai.ts index d9d8b7b1e..b991fb4ed 100644 --- a/src/lib/jobs/coach-nudge-ai.ts +++ b/src/lib/jobs/coach-nudge-ai.ts @@ -202,6 +202,7 @@ export const composeNudgeWithAI: ComposeNudgeWithAI = async (params) => { dateKey, dailyCap, resolveCostOwner(chain), + "job", ); if (!reservation.allowed) { annotate({ action: { name: "coach.nudge.ai.budget_exceeded" } }); diff --git a/src/lib/jobs/document-content-index-backfill.ts b/src/lib/jobs/document-content-index-backfill.ts index bdec599f1..d8e4ad718 100644 --- a/src/lib/jobs/document-content-index-backfill.ts +++ b/src/lib/jobs/document-content-index-backfill.ts @@ -236,6 +236,7 @@ export async function runContentIndexBackfillForUser( dateKey, dailyCap, resolveCostOwner([{ providerType: pick.entry.providerType }]), + "job", ); if (!reservation.allowed) { reason = "budget-reached"; diff --git a/src/lib/jobs/document-summary.ts b/src/lib/jobs/document-summary.ts index f4c1307d9..647115ee5 100644 --- a/src/lib/jobs/document-summary.ts +++ b/src/lib/jobs/document-summary.ts @@ -194,6 +194,7 @@ export async function runDocumentSummaryJob( // v1.38.19 (Wave E) — a background surface: half the day's ceiling. resolveDailyCapFor("job", [{ providerType: pick.entry.providerType }]), resolveCostOwner([{ providerType: pick.entry.providerType }]), + "job", ); // Budget exhausted → skip (no local fallback for a summary). if (!reservation.allowed) { diff --git a/tests/integration/coach-budget-owner.test.ts b/tests/integration/coach-budget-owner.test.ts index 95f81f8a5..08344af12 100644 --- a/tests/integration/coach-budget-owner.test.ts +++ b/tests/integration/coach-budget-owner.test.ts @@ -53,6 +53,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => { dateKey, OPERATOR_COST_CAP, "operator", + "coach", ); expect(res.allowed).toBe(true); await reconcileSpend(userId, res.reserved, 3_000, dateKey, 0, { @@ -80,6 +81,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => { dateKey, OPERATOR_COST_CAP, "operator", + "coach", ); await reconcileSpend(userId, res.reserved, 3_000, dateKey, 0, { servedBy: "codex", @@ -114,6 +116,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => { dateKey, USER_PLAN_CAP, "user", + "coach", ); await reconcileSpend(userId, res.reserved, 4_000, dateKey, 0, { servedBy: "codex", @@ -142,6 +145,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => { dateKey, USER_PLAN_CAP, "user", + "coach", ); await reconcileSpend(userId, res.reserved, 2_500, dateKey, 0, { servedBy: "admin-openai", @@ -178,6 +182,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => { dateKey, OPERATOR_COST_CAP, "operator", + "coach", ); expect(res.allowed).toBe(true); expect(res.operatorAfter).toBe(153_000); @@ -205,6 +210,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => { dateKey, OPERATOR_COST_CAP, "operator", + "coach", ); expect(res.allowed).toBe(false); @@ -213,6 +219,98 @@ describe("coach_usage.operator_tokens (real Postgres)", () => { expect(row?.operatorTokens).toBe(OPERATOR_COST_CAP); }); + // v1.38.19 (Wave E, fix round 1) — the incident, end to end. + // + // The 06:42Z row verbatim: 1.2 M tokens on the day, none of them the + // operator's, on the chain `[admin-openai, codex, openai-compatible]`. The + // chat must open. Before the wave it was refused because the ledger had no + // owner dimension; this pins that the refusal cannot come back. + it("does not refuse the chat for a day of background spend on the user's own plan", async () => { + const { reserveBudget, resolveDailyCap, resolveCostOwner } = + await import("@/lib/ai/coach/budget"); + const userId = await seedUser(); + const dateKey = "2026-09-11"; + const chain = [ + { providerType: "admin-openai" as const }, + { providerType: "codex" as const }, + { providerType: "openai-compatible" as const }, + ]; + + await getPrismaClient().coachUsage.create({ + data: { + userId, + dateKey, + totalTokens: 1_200_000, + operatorTokens: 0, + messageCount: 240, + }, + }); + + const chat = await reserveBudget( + userId, + 3_000, + dateKey, + resolveDailyCap(chain), + resolveCostOwner(chain), + "coach", + ); + expect(chat.allowed).toBe(true); + expect(chat.limit).toBeNull(); + }); + + it("refuses the BACKGROUND generation on that same day", async () => { + // The other half of the promise: the interactive day is protected, and the + // automatic work that filled the row is the thing that stops. The operator + // counter is empty, so only the abuse ceiling on the total can say no — + // which is the ceiling the first cut dropped for operator chains. + const { reserveBudget, resolveDailyCapFor, resolveCostOwner } = + await import("@/lib/ai/coach/budget"); + const userId = await seedUser(); + const dateKey = "2026-09-11"; + const chain = [ + { providerType: "admin-openai" as const }, + { providerType: "codex" as const }, + ]; + + await getPrismaClient().coachUsage.create({ + data: { + userId, + dateKey, + totalTokens: 1_900_000, + operatorTokens: 0, + messageCount: 380, + }, + }); + + const job = await reserveBudget( + userId, + 3_000, + dateKey, + resolveDailyCapFor("job", chain), + resolveCostOwner(chain), + "job", + ); + expect(job.allowed).toBe(false); + expect(job.limit).toBe("total-cap"); + + // Refused means refunded: the row is exactly as it was. + const row = await readRow(userId, dateKey); + expect(row?.totalTokens).toBe(1_900_000); + expect(row?.operatorTokens).toBe(0); + expect(row?.messageCount).toBe(380); + + // ...and the person waiting still gets their turn. + const chat = await reserveBudget( + userId, + 3_000, + dateKey, + 200_000, + "operator", + "coach", + ); + expect(chat.allowed).toBe(true); + }); + it("refunds both counters when a refused reservation is rolled back", async () => { const { reserveBudget, OPERATOR_COST_CAP } = await import("@/lib/ai/coach/budget"); @@ -235,6 +333,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => { dateKey, OPERATOR_COST_CAP, "operator", + "coach", ); expect(res.allowed).toBe(false); diff --git a/tests/integration/coach-budget-reservation.test.ts b/tests/integration/coach-budget-reservation.test.ts index 1ff211fc0..1f0bec052 100644 --- a/tests/integration/coach-budget-reservation.test.ts +++ b/tests/integration/coach-budget-reservation.test.ts @@ -37,13 +37,27 @@ describe("reserveBudget (real Postgres)", () => { // Pre-spend 900 so a single 600-token reservation stays under the cap // but two concurrent ones cannot both be admitted. + // + // v1.38.19 (Wave E) — the 900 sits on BOTH counters. The ledger grew a cost + // dimension: an operator-owned reservation is measured against + // `operator_tokens`, so a fixture that pre-spends only the mixed total no + // longer states its own premise ("900 already spent under this cap") and + // would prove nothing about concurrency. The contract this test exists for + // — two concurrent reservations cannot both pass one ceiling — is + // unchanged, and so is every assertion below. await getPrismaClient().coachUsage.create({ - data: { userId, dateKey, totalTokens: 900, messageCount: 1 }, + data: { + userId, + dateKey, + totalTokens: 900, + operatorTokens: 900, + messageCount: 1, + }, }); const [a, b] = await Promise.all([ - reserveBudget(userId, 600, dateKey, cap, "operator"), - reserveBudget(userId, 600, dateKey, cap, "operator"), + reserveBudget(userId, 600, dateKey, cap, "operator", "coach"), + reserveBudget(userId, 600, dateKey, cap, "operator", "coach"), ]); // Exactly one is admitted: the first to commit sees prior 900 (< cap) and @@ -58,6 +72,36 @@ describe("reserveBudget (real Postgres)", () => { where: { userId_dateKey: { userId, dateKey } }, }); expect(row?.totalTokens).toBe(1500); + expect(row?.operatorTokens).toBe(1500); + }); + + it("lets at most one of two concurrent requests push past the TOTAL ceiling", async () => { + // The same guarantee on the other arm. An operator-funded chain whose key + // keeps failing leaves `operator_tokens` near zero all day, so the abuse + // ceiling on the mixed total is the only ceiling left — and it has to + // serialise exactly like the owner's does. + const { reserveBudget, USER_PLAN_CAP } = + await import("@/lib/ai/coach/budget"); + const userId = await seedUser(); + const dateKey = "2026-06-23"; + + await getPrismaClient().coachUsage.create({ + data: { + userId, + dateKey, + totalTokens: USER_PLAN_CAP - 100, + operatorTokens: 0, + messageCount: 1, + }, + }); + + const [a, b] = await Promise.all([ + reserveBudget(userId, 600, dateKey, 200_000, "operator", "coach"), + reserveBudget(userId, 600, dateKey, 200_000, "operator", "coach"), + ]); + + expect([a, b].filter((r) => r.allowed).length).toBe(1); + expect([a, b].find((r) => !r.allowed)?.limit).toBe("total-cap"); }); it("admits the first request of a fresh day via the upsert create branch", async () => { @@ -65,7 +109,14 @@ describe("reserveBudget (real Postgres)", () => { const userId = await seedUser(); const dateKey = "2026-06-20"; - const res = await reserveBudget(userId, 600, dateKey, 25_000, "operator"); + const res = await reserveBudget( + userId, + 600, + dateKey, + 25_000, + "operator", + "coach", + ); expect(res.allowed).toBe(true); const row = await getPrismaClient().coachUsage.findUnique({ @@ -81,7 +132,14 @@ describe("reserveBudget (real Postgres)", () => { const userId = await seedUser(); const dateKey = "2026-06-21"; - const res = await reserveBudget(userId, 600, dateKey, 25_000, "operator"); + const res = await reserveBudget( + userId, + 600, + dateKey, + 25_000, + "operator", + "coach", + ); await reconcileSpend(userId, res.reserved, 120, dateKey, 0, { servedBy: "admin-openai", reservedOwner: res.owner, @@ -102,7 +160,14 @@ describe("reserveBudget (real Postgres)", () => { // Provider returned 450 tokens but the reply was empty/sentinel: the // reconcile must still record the 450 burned, not zero. - const res = await reserveBudget(userId, 600, dateKey, 25_000, "operator"); + const res = await reserveBudget( + userId, + 600, + dateKey, + 25_000, + "operator", + "coach", + ); await reconcileSpend(userId, res.reserved, 450, dateKey, 0, { servedBy: "admin-openai", reservedOwner: res.owner,