Guard the operator's cap on fallback hops and reap dead summary promises

- The provider-chain walker now checks the day's recorded spend against
  OPERATOR_COST_CAP before invoking an operator-funded admin-* entry. A
  request whose primary provider runs on the user's own key reserves
  under the generous user-plan ceiling; a fallback hop (or health-ledger
  reorder) onto the operator's key could then spend past the operator's
  cap unchecked. An exhausted cap skips the candidate like a failed hop.
- summaryState PENDING gets a giver-upper: the detail serialiser
  degrades a PENDING older than an hour to UNAVAILABLE (which offers the
  manual generate action), and an hourly reaper persists the same heal
  following the orphan-ImportJob reconcile precedent — a job dead past
  its retry budget no longer leaves the sheet saying "generating"
  forever.
This commit is contained in:
Marc-André Bombeck committed 2026-08-14 05:53:17 +02:00
1 parent f67fcec1c9
commit 6ebd5e8711
8 files changed
+333 -4

No files matched your search

@@ -10,6 +10,19 @@ import {
// provider-health ledger. These pure chain tests do not stand up a DB,
// so swap the default for a no-op; the dedicated ledger-aware cases
// below inject an in-memory ledger explicitly.
// v1.37.19 (A7-2) — the runner consults the day's recorded spend before an
// operator-funded hop. These pure chain tests do not stand up a DB, so the
// ledger read is swapped for a controllable in-memory figure.
const budgetState = vi.hoisted(() => ({ spent: 0 }));
vi.mock("../coach/budget", async () => {
const actual =
await vi.importActual<typeof import("../coach/budget")>("../coach/budget");
return {
...actual,
readDailySpend: vi.fn(async () => budgetState.spent),
};
});
vi.mock("../provider-health-ledger", async () => {
const actual = await vi.importActual<
typeof import("../provider-health-ledger")
@@ -97,6 +110,7 @@ function err(status: number, msg = "boom"): Error & { httpStatus: number } {
beforeEach(() => {
clearLastWorkingProviderCache();
budgetState.spent = 0;
});
afterEach(() => {
@@ -671,3 +685,76 @@ describe("runStreamingRawCompletionWithFallback", () => {
expect(result.fallbackHops.map((h) => h.providerType)).toEqual(["openai"]);
});
});
// v1.37.19 (A7-2) — hop-time operator-cost guard.
//
// Watched red: with the `isOperatorFundedProvider` check removed from
// `runRawChain` (the pre-fix walker), the exhausted-cap case below fails —
// the admin-openai fallback was invoked and spent operator money although
// the request had only been reserved under the user-plan ceiling.
describe("runRawCompletionWithFallback — operator-cost cap at hop time", () => {
const params: CompletionParams = singleUserTurn({ system: "s", user: "u" });
it("refuses the admin-* fallback hop once the operator cap is exhausted", async () => {
budgetState.spent = 200_000; // OPERATOR_COST_CAP
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(500) }],
});
const adminOpenai = new ScriptedProvider({ script: [{ ok: true }] });
await expect(
runRawCompletionWithFallback({
userId: "u-cap",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "admin-openai", instance: adminOpenai },
],
params,
}),
).rejects.toMatchObject({
attempts: [
expect.objectContaining({ providerType: "codex" }),
expect.objectContaining({
providerType: "admin-openai",
failureReason: "operator-cost-cap-exhausted",
}),
],
});
// The operator-funded provider was never invoked.
expect(adminOpenai.callCount).toBe(0);
});
it("lets the admin-* fallback run while the operator cap has headroom", async () => {
budgetState.spent = 100;
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(500) }],
});
const adminOpenai = new ScriptedProvider({ script: [{ ok: true }] });
const outcome = await runRawCompletionWithFallback({
userId: "u-headroom",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "admin-openai", instance: adminOpenai },
],
params,
});
expect(outcome.workingProvider.providerType).toBe("admin-openai");
expect(adminOpenai.callCount).toBe(1);
});
it("never consults the ledger for a user-funded chain", async () => {
budgetState.spent = 999_999_999;
const openai = new ScriptedProvider({ script: [{ ok: true }] });
const outcome = await runRawCompletionWithFallback({
userId: "u-own-key",
providers: [{ providerType: "openai", instance: openai }],
params,
});
// The user's own key is not the operator's money — the cap does not
// apply and the call proceeds regardless of the recorded spend.
expect(outcome.workingProvider.providerType).toBe("openai");
});
});
+33 -3
View File
@@ -64,13 +64,43 @@ export function resolveDailyCap(
chain: ReadonlyArray<{ providerType: ProviderChainType }>,
): number {
const primary = chain[0]?.providerType;
return primary === "admin-openai" ||
primary === "admin-codex" ||
primary === undefined
return primary === undefined || isOperatorFundedProvider(primary)
? OPERATOR_COST_CAP
: USER_PLAN_CAP;
}
/**
* v1.37.19 (A7-2) — does this provider spend the OPERATOR's money?
* `admin-openai` (the server's own API key) and `admin-codex` (the
* server's shared ChatGPT account) do; everything else is the user's own
* egress. One predicate so `resolveDailyCap` and the chain walker's
* hop-time guard cannot drift on the classification.
*/
export function isOperatorFundedProvider(type: ProviderChainType): boolean {
return type === "admin-openai" || type === "admin-codex";
}
/**
* v1.37.19 (A7-2) — the day's recorded spend for one user (0 when no row).
*
* Read by the chain walker's hop-time operator-cap guard: a request whose
* PRIMARY provider runs on the user's own key reserves under the generous
* `USER_PLAN_CAP`, but the chain may still FALL BACK onto an operator-funded
* `admin-*` entry (or the health-ledger reorder may promote one) — and that
* hop must not spend past `OPERATOR_COST_CAP` just because the reservation
* was checked against the wrong owner's ceiling.
*/
export async function readDailySpend(
userId: string,
dateKey: string = buildDateKey(),
): Promise<number> {
const row = await prisma.coachUsage.findUnique({
where: { userId_dateKey: { userId, dateKey } },
select: { totalTokens: true },
});
return row?.totalTokens ?? 0;
}
/**
* Build the UTC day-key for a given clock. Defaults to "now".
*
+35
View File
@@ -6,6 +6,11 @@ import {
type ProviderSkipHint,
} from "./provider-health-ledger";
import { annotate } from "@/lib/logging/context";
import {
isOperatorFundedProvider,
OPERATOR_COST_CAP,
readDailySpend,
} from "@/lib/ai/coach/budget";
/**
* The local model is the guaranteed floor (Epic B Pillar 4): a
@@ -394,6 +399,36 @@ async function runRawChain(
for (let i = 0; i < ordered.length; i += 1) {
const candidate = ordered[i];
// v1.37.19 (A7-2) — hop-time operator-cost guard. A request whose
// primary provider runs on the USER's own key reserves its budget
// under the generous user-plan ceiling; when the chain then falls
// back onto an operator-funded `admin-*` entry (or the health-ledger
// reorder promotes one), that hop would spend the OPERATOR's money
// under the wrong owner's cap. Check the day's recorded spend
// against OPERATOR_COST_CAP before letting an operator-funded
// candidate run; an exhausted cap skips the candidate exactly like a
// failed hop (next entry, or AllProvidersFailedError at the end).
if (isOperatorFundedProvider(candidate.providerType)) {
const spent = await readDailySpend(userId);
if (spent >= OPERATOR_COST_CAP) {
const hop: FallbackHop = {
providerType: candidate.providerType,
attempt: i + 1,
failureReason: "operator-cost-cap-exhausted",
httpStatus: null,
};
hops.push(hop);
annotate({
action: { name: "ai.chain.operator_cap_refused" },
meta: {
[`ai_chain_hop_${i + 1}_provider`]: candidate.providerType,
[`ai_chain_hop_${i + 1}_reason`]: "operator-cost-cap-exhausted",
operator_cap_spent: spent,
},
});
continue;
}
}
try {
const result = await invoke(candidate);
rememberWorkingProvider(userId, candidate.providerType);