fix(ai): every surface keeps a ceiling on the day's mixed total

The owner split left an operator-funded chain with no ceiling at all. Its arm
compared operator_tokens, and that counter returns to roughly zero on every
reconcile the user's own plan settled: on the 2026-09-11 evidence the shared
key answered 500 and codex served, so the counter never grew, every job
reservation was admitted forever, and a runaway client loop could write an
unbounded row. A pre-existing concurrency contract caught it from the other
side, two reservations that must not both pass one cap both passing.

reserveBudget now refuses when the owner's counter is over its cap OR the day's
mixed total is over the abuse ceiling, and a background surface gets its share
of that ceiling too, so background spend is bounded on the total and not only
under a counter the operator's key never fills. The surface is a required
argument, so the type checker enumerated every reservation site.

The hop guard learns the same lesson: an operator-funded fallback hop is now
rationed at the calling surface's ceiling rather than the full operator cap. A
job admitted under the job share could previously spend the whole 200 000 once
its chain fell back onto the shared key, which is the original defect reached
by the fallback path.

The job share applies only where the operator pays for the chain. Multiplying
whichever ceiling the chain resolved halved background generation for a
self-hoster on a local model, who costs the operator nothing.

Refusals now name the ceiling they hit and whose it was. A refusal used to emit
the day's mixed total and nothing else, which on an operator refusal is not the
counter that tripped.
This commit is contained in:
Marc-André Bombeck committed 2026-09-11 11:11:01 +02:00
1 parent ff51b72cd1
commit 18ed57b867
30 files changed
+645 -25

No files matched your search

@@ -53,6 +53,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => {
dateKey,
OPERATOR_COST_CAP,
"operator",
"coach",
);
expect(res.allowed).toBe(true);
await reconcileSpend(userId, res.reserved, 3_000, dateKey, 0, {
@@ -80,6 +81,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => {
dateKey,
OPERATOR_COST_CAP,
"operator",
"coach",
);
await reconcileSpend(userId, res.reserved, 3_000, dateKey, 0, {
servedBy: "codex",
@@ -114,6 +116,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => {
dateKey,
USER_PLAN_CAP,
"user",
"coach",
);
await reconcileSpend(userId, res.reserved, 4_000, dateKey, 0, {
servedBy: "codex",
@@ -142,6 +145,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => {
dateKey,
USER_PLAN_CAP,
"user",
"coach",
);
await reconcileSpend(userId, res.reserved, 2_500, dateKey, 0, {
servedBy: "admin-openai",
@@ -178,6 +182,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => {
dateKey,
OPERATOR_COST_CAP,
"operator",
"coach",
);
expect(res.allowed).toBe(true);
expect(res.operatorAfter).toBe(153_000);
@@ -205,6 +210,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => {
dateKey,
OPERATOR_COST_CAP,
"operator",
"coach",
);
expect(res.allowed).toBe(false);
@@ -213,6 +219,98 @@ describe("coach_usage.operator_tokens (real Postgres)", () => {
expect(row?.operatorTokens).toBe(OPERATOR_COST_CAP);
});
// v1.38.19 (Wave E, fix round 1) — the incident, end to end.
//
// The 06:42Z row verbatim: 1.2 M tokens on the day, none of them the
// operator's, on the chain `[admin-openai, codex, openai-compatible]`. The
// chat must open. Before the wave it was refused because the ledger had no
// owner dimension; this pins that the refusal cannot come back.
it("does not refuse the chat for a day of background spend on the user's own plan", async () => {
const { reserveBudget, resolveDailyCap, resolveCostOwner } =
await import("@/lib/ai/coach/budget");
const userId = await seedUser();
const dateKey = "2026-09-11";
const chain = [
{ providerType: "admin-openai" as const },
{ providerType: "codex" as const },
{ providerType: "openai-compatible" as const },
];
await getPrismaClient().coachUsage.create({
data: {
userId,
dateKey,
totalTokens: 1_200_000,
operatorTokens: 0,
messageCount: 240,
},
});
const chat = await reserveBudget(
userId,
3_000,
dateKey,
resolveDailyCap(chain),
resolveCostOwner(chain),
"coach",
);
expect(chat.allowed).toBe(true);
expect(chat.limit).toBeNull();
});
it("refuses the BACKGROUND generation on that same day", async () => {
// The other half of the promise: the interactive day is protected, and the
// automatic work that filled the row is the thing that stops. The operator
// counter is empty, so only the abuse ceiling on the total can say no —
// which is the ceiling the first cut dropped for operator chains.
const { reserveBudget, resolveDailyCapFor, resolveCostOwner } =
await import("@/lib/ai/coach/budget");
const userId = await seedUser();
const dateKey = "2026-09-11";
const chain = [
{ providerType: "admin-openai" as const },
{ providerType: "codex" as const },
];
await getPrismaClient().coachUsage.create({
data: {
userId,
dateKey,
totalTokens: 1_900_000,
operatorTokens: 0,
messageCount: 380,
},
});
const job = await reserveBudget(
userId,
3_000,
dateKey,
resolveDailyCapFor("job", chain),
resolveCostOwner(chain),
"job",
);
expect(job.allowed).toBe(false);
expect(job.limit).toBe("total-cap");
// Refused means refunded: the row is exactly as it was.
const row = await readRow(userId, dateKey);
expect(row?.totalTokens).toBe(1_900_000);
expect(row?.operatorTokens).toBe(0);
expect(row?.messageCount).toBe(380);
// ...and the person waiting still gets their turn.
const chat = await reserveBudget(
userId,
3_000,
dateKey,
200_000,
"operator",
"coach",
);
expect(chat.allowed).toBe(true);
});
it("refunds both counters when a refused reservation is rolled back", async () => {
const { reserveBudget, OPERATOR_COST_CAP } =
await import("@/lib/ai/coach/budget");
@@ -235,6 +333,7 @@ describe("coach_usage.operator_tokens (real Postgres)", () => {
dateKey,
OPERATOR_COST_CAP,
"operator",
"coach",
);
expect(res.allowed).toBe(false);
@@ -37,13 +37,27 @@ describe("reserveBudget (real Postgres)", () => {
// Pre-spend 900 so a single 600-token reservation stays under the cap
// but two concurrent ones cannot both be admitted.
//
// v1.38.19 (Wave E) — the 900 sits on BOTH counters. The ledger grew a cost
// dimension: an operator-owned reservation is measured against
// `operator_tokens`, so a fixture that pre-spends only the mixed total no
// longer states its own premise ("900 already spent under this cap") and
// would prove nothing about concurrency. The contract this test exists for
// — two concurrent reservations cannot both pass one ceiling — is
// unchanged, and so is every assertion below.
await getPrismaClient().coachUsage.create({
data: { userId, dateKey, totalTokens: 900, messageCount: 1 },
data: {
userId,
dateKey,
totalTokens: 900,
operatorTokens: 900,
messageCount: 1,
},
});
const [a, b] = await Promise.all([
reserveBudget(userId, 600, dateKey, cap, "operator"),
reserveBudget(userId, 600, dateKey, cap, "operator"),
reserveBudget(userId, 600, dateKey, cap, "operator", "coach"),
reserveBudget(userId, 600, dateKey, cap, "operator", "coach"),
]);
// Exactly one is admitted: the first to commit sees prior 900 (< cap) and
@@ -58,6 +72,36 @@ describe("reserveBudget (real Postgres)", () => {
where: { userId_dateKey: { userId, dateKey } },
});
expect(row?.totalTokens).toBe(1500);
expect(row?.operatorTokens).toBe(1500);
});
it("lets at most one of two concurrent requests push past the TOTAL ceiling", async () => {
// The same guarantee on the other arm. An operator-funded chain whose key
// keeps failing leaves `operator_tokens` near zero all day, so the abuse
// ceiling on the mixed total is the only ceiling left — and it has to
// serialise exactly like the owner's does.
const { reserveBudget, USER_PLAN_CAP } =
await import("@/lib/ai/coach/budget");
const userId = await seedUser();
const dateKey = "2026-06-23";
await getPrismaClient().coachUsage.create({
data: {
userId,
dateKey,
totalTokens: USER_PLAN_CAP - 100,
operatorTokens: 0,
messageCount: 1,
},
});
const [a, b] = await Promise.all([
reserveBudget(userId, 600, dateKey, 200_000, "operator", "coach"),
reserveBudget(userId, 600, dateKey, 200_000, "operator", "coach"),
]);
expect([a, b].filter((r) => r.allowed).length).toBe(1);
expect([a, b].find((r) => !r.allowed)?.limit).toBe("total-cap");
});
it("admits the first request of a fresh day via the upsert create branch", async () => {
@@ -65,7 +109,14 @@ describe("reserveBudget (real Postgres)", () => {
const userId = await seedUser();
const dateKey = "2026-06-20";
const res = await reserveBudget(userId, 600, dateKey, 25_000, "operator");
const res = await reserveBudget(
userId,
600,
dateKey,
25_000,
"operator",
"coach",
);
expect(res.allowed).toBe(true);
const row = await getPrismaClient().coachUsage.findUnique({
@@ -81,7 +132,14 @@ describe("reserveBudget (real Postgres)", () => {
const userId = await seedUser();
const dateKey = "2026-06-21";
const res = await reserveBudget(userId, 600, dateKey, 25_000, "operator");
const res = await reserveBudget(
userId,
600,
dateKey,
25_000,
"operator",
"coach",
);
await reconcileSpend(userId, res.reserved, 120, dateKey, 0, {
servedBy: "admin-openai",
reservedOwner: res.owner,
@@ -102,7 +160,14 @@ describe("reserveBudget (real Postgres)", () => {
// Provider returned 450 tokens but the reply was empty/sentinel: the
// reconcile must still record the 450 burned, not zero.
const res = await reserveBudget(userId, 600, dateKey, 25_000, "operator");
const res = await reserveBudget(
userId,
600,
dateKey,
25_000,
"operator",
"coach",
);
await reconcileSpend(userId, res.reserved, 450, dateKey, 0, {
servedBy: "admin-openai",
reservedOwner: res.owner,