mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-15 23:12:04 +00:00
Guard the operator's cap on fallback hops and reap dead summary promises
- The provider-chain walker now checks the day's recorded spend against OPERATOR_COST_CAP before invoking an operator-funded admin-* entry. A request whose primary provider runs on the user's own key reserves under the generous user-plan ceiling; a fallback hop (or health-ledger reorder) onto the operator's key could then spend past the operator's cap unchecked. An exhausted cap skips the candidate like a failed hop. - summaryState PENDING gets a giver-upper: the detail serialiser degrades a PENDING older than an hour to UNAVAILABLE (which offers the manual generate action), and an hourly reaper persists the same heal following the orphan-ImportJob reconcile precedent — a job dead past its retry budget no longer leaves the sheet saying "generating" forever.
This commit is contained in:
1 parent
f67fcec1c9
commit
6ebd5e8711
8 files changed
+333
-4
No files matched your search
@@ -10,6 +10,19 @@ import {
|
||||
// provider-health ledger. These pure chain tests do not stand up a DB,
|
||||
// so swap the default for a no-op; the dedicated ledger-aware cases
|
||||
// below inject an in-memory ledger explicitly.
|
||||
// v1.37.19 (A7-2) — the runner consults the day's recorded spend before an
|
||||
// operator-funded hop. These pure chain tests do not stand up a DB, so the
|
||||
// ledger read is swapped for a controllable in-memory figure.
|
||||
const budgetState = vi.hoisted(() => ({ spent: 0 }));
|
||||
vi.mock("../coach/budget", async () => {
|
||||
const actual =
|
||||
await vi.importActual<typeof import("../coach/budget")>("../coach/budget");
|
||||
return {
|
||||
...actual,
|
||||
readDailySpend: vi.fn(async () => budgetState.spent),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../provider-health-ledger", async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("../provider-health-ledger")
|
||||
@@ -97,6 +110,7 @@ function err(status: number, msg = "boom"): Error & { httpStatus: number } {
|
||||
|
||||
beforeEach(() => {
|
||||
clearLastWorkingProviderCache();
|
||||
budgetState.spent = 0;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -671,3 +685,76 @@ describe("runStreamingRawCompletionWithFallback", () => {
|
||||
expect(result.fallbackHops.map((h) => h.providerType)).toEqual(["openai"]);
|
||||
});
|
||||
});
|
||||
|
||||
// v1.37.19 (A7-2) — hop-time operator-cost guard.
|
||||
//
|
||||
// Watched red: with the `isOperatorFundedProvider` check removed from
|
||||
// `runRawChain` (the pre-fix walker), the exhausted-cap case below fails —
|
||||
// the admin-openai fallback was invoked and spent operator money although
|
||||
// the request had only been reserved under the user-plan ceiling.
|
||||
describe("runRawCompletionWithFallback — operator-cost cap at hop time", () => {
|
||||
const params: CompletionParams = singleUserTurn({ system: "s", user: "u" });
|
||||
|
||||
it("refuses the admin-* fallback hop once the operator cap is exhausted", async () => {
|
||||
budgetState.spent = 200_000; // OPERATOR_COST_CAP
|
||||
const codex = new ScriptedProvider({
|
||||
type: "codex",
|
||||
script: [{ ok: false, error: err(500) }],
|
||||
});
|
||||
const adminOpenai = new ScriptedProvider({ script: [{ ok: true }] });
|
||||
|
||||
await expect(
|
||||
runRawCompletionWithFallback({
|
||||
userId: "u-cap",
|
||||
providers: [
|
||||
{ providerType: "codex", instance: codex },
|
||||
{ providerType: "admin-openai", instance: adminOpenai },
|
||||
],
|
||||
params,
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
attempts: [
|
||||
expect.objectContaining({ providerType: "codex" }),
|
||||
expect.objectContaining({
|
||||
providerType: "admin-openai",
|
||||
failureReason: "operator-cost-cap-exhausted",
|
||||
}),
|
||||
],
|
||||
});
|
||||
// The operator-funded provider was never invoked.
|
||||
expect(adminOpenai.callCount).toBe(0);
|
||||
});
|
||||
|
||||
it("lets the admin-* fallback run while the operator cap has headroom", async () => {
|
||||
budgetState.spent = 100;
|
||||
const codex = new ScriptedProvider({
|
||||
type: "codex",
|
||||
script: [{ ok: false, error: err(500) }],
|
||||
});
|
||||
const adminOpenai = new ScriptedProvider({ script: [{ ok: true }] });
|
||||
|
||||
const outcome = await runRawCompletionWithFallback({
|
||||
userId: "u-headroom",
|
||||
providers: [
|
||||
{ providerType: "codex", instance: codex },
|
||||
{ providerType: "admin-openai", instance: adminOpenai },
|
||||
],
|
||||
params,
|
||||
});
|
||||
expect(outcome.workingProvider.providerType).toBe("admin-openai");
|
||||
expect(adminOpenai.callCount).toBe(1);
|
||||
});
|
||||
|
||||
it("never consults the ledger for a user-funded chain", async () => {
|
||||
budgetState.spent = 999_999_999;
|
||||
const openai = new ScriptedProvider({ script: [{ ok: true }] });
|
||||
const outcome = await runRawCompletionWithFallback({
|
||||
userId: "u-own-key",
|
||||
providers: [{ providerType: "openai", instance: openai }],
|
||||
params,
|
||||
});
|
||||
// The user's own key is not the operator's money — the cap does not
|
||||
// apply and the call proceeds regardless of the recorded spend.
|
||||
expect(outcome.workingProvider.providerType).toBe("openai");
|
||||
});
|
||||
});
|
||||
@@ -64,13 +64,43 @@ export function resolveDailyCap(
|
||||
chain: ReadonlyArray<{ providerType: ProviderChainType }>,
|
||||
): number {
|
||||
const primary = chain[0]?.providerType;
|
||||
return primary === "admin-openai" ||
|
||||
primary === "admin-codex" ||
|
||||
primary === undefined
|
||||
return primary === undefined || isOperatorFundedProvider(primary)
|
||||
? OPERATOR_COST_CAP
|
||||
: USER_PLAN_CAP;
|
||||
}
|
||||
|
||||
/**
|
||||
* v1.37.19 (A7-2) — does this provider spend the OPERATOR's money?
|
||||
* `admin-openai` (the server's own API key) and `admin-codex` (the
|
||||
* server's shared ChatGPT account) do; everything else is the user's own
|
||||
* egress. One predicate so `resolveDailyCap` and the chain walker's
|
||||
* hop-time guard cannot drift on the classification.
|
||||
*/
|
||||
export function isOperatorFundedProvider(type: ProviderChainType): boolean {
|
||||
return type === "admin-openai" || type === "admin-codex";
|
||||
}
|
||||
|
||||
/**
|
||||
* v1.37.19 (A7-2) — the day's recorded spend for one user (0 when no row).
|
||||
*
|
||||
* Read by the chain walker's hop-time operator-cap guard: a request whose
|
||||
* PRIMARY provider runs on the user's own key reserves under the generous
|
||||
* `USER_PLAN_CAP`, but the chain may still FALL BACK onto an operator-funded
|
||||
* `admin-*` entry (or the health-ledger reorder may promote one) — and that
|
||||
* hop must not spend past `OPERATOR_COST_CAP` just because the reservation
|
||||
* was checked against the wrong owner's ceiling.
|
||||
*/
|
||||
export async function readDailySpend(
|
||||
userId: string,
|
||||
dateKey: string = buildDateKey(),
|
||||
): Promise<number> {
|
||||
const row = await prisma.coachUsage.findUnique({
|
||||
where: { userId_dateKey: { userId, dateKey } },
|
||||
select: { totalTokens: true },
|
||||
});
|
||||
return row?.totalTokens ?? 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the UTC day-key for a given clock. Defaults to "now".
|
||||
*
|
||||
|
||||
@@ -6,6 +6,11 @@ import {
|
||||
type ProviderSkipHint,
|
||||
} from "./provider-health-ledger";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
isOperatorFundedProvider,
|
||||
OPERATOR_COST_CAP,
|
||||
readDailySpend,
|
||||
} from "@/lib/ai/coach/budget";
|
||||
|
||||
/**
|
||||
* The local model is the guaranteed floor (Epic B Pillar 4): a
|
||||
@@ -394,6 +399,36 @@ async function runRawChain(
|
||||
|
||||
for (let i = 0; i < ordered.length; i += 1) {
|
||||
const candidate = ordered[i];
|
||||
// v1.37.19 (A7-2) — hop-time operator-cost guard. A request whose
|
||||
// primary provider runs on the USER's own key reserves its budget
|
||||
// under the generous user-plan ceiling; when the chain then falls
|
||||
// back onto an operator-funded `admin-*` entry (or the health-ledger
|
||||
// reorder promotes one), that hop would spend the OPERATOR's money
|
||||
// under the wrong owner's cap. Check the day's recorded spend
|
||||
// against OPERATOR_COST_CAP before letting an operator-funded
|
||||
// candidate run; an exhausted cap skips the candidate exactly like a
|
||||
// failed hop (next entry, or AllProvidersFailedError at the end).
|
||||
if (isOperatorFundedProvider(candidate.providerType)) {
|
||||
const spent = await readDailySpend(userId);
|
||||
if (spent >= OPERATOR_COST_CAP) {
|
||||
const hop: FallbackHop = {
|
||||
providerType: candidate.providerType,
|
||||
attempt: i + 1,
|
||||
failureReason: "operator-cost-cap-exhausted",
|
||||
httpStatus: null,
|
||||
};
|
||||
hops.push(hop);
|
||||
annotate({
|
||||
action: { name: "ai.chain.operator_cap_refused" },
|
||||
meta: {
|
||||
[`ai_chain_hop_${i + 1}_provider`]: candidate.providerType,
|
||||
[`ai_chain_hop_${i + 1}_reason`]: "operator-cost-cap-exhausted",
|
||||
operator_cap_spent: spent,
|
||||
},
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
try {
|
||||
const result = await invoke(candidate);
|
||||
rememberWorkingProvider(userId, candidate.providerType);
|
||||
|
||||
Reference in new issue
Block a user