feat(admin): name the status a failing provider came back with

The provider-health ledger has recorded the HTTP status of every failure
since v1.11.0 and nothing has ever read it. The operator card could say
three of five chains were failing without saying whose problem that was,
which is the one thing the operator needs: a 401 is a credential only they
can replace, a 429 or a 503 is the provider's afternoon and waiting is the
right move.

The fold carries the status of the failure whose instant the row already
shows, not an aggregate over the column. A MAX of two HTTP statuses is
arithmetic on a label, and a status describing a different failure than
the one on screen would send the operator to the wrong place. The column
joins the grouping key instead, and the fold picks the group belonging to
the newest failure.

Blank where there was nothing to record: a connection that never completed
has no status, and a chain that has since recovered had its status cleared
along with the rest of its failure state.
This commit is contained in:
Marc-André Bombeck committed 2026-09-03 06:05:58 +02:00
1 parent 2ea08c6966
commit 7dbec3bc4d
12 files changed
+186 -2

No files matched your search

@@ -5,6 +5,11 @@
* provider type, failing counted from the LAST result only, the worst
* uninterrupted failure run taken across failing users, central types
* sorted first, and no per-user data in the response.
*
* It also pins the HTTP status the ledger has recorded on every failure
* since v1.11.0 and nothing read until now: it must describe the SAME
* failure the row's instant names, or the card tells the operator to go
* looking at the wrong thing.
*/
import { describe, it, expect, vi, beforeEach } from "vitest";
@@ -49,11 +54,13 @@ function group(
consecutiveFailures?: number;
lastFailureAt?: Date | null;
lastOkAt?: Date | null;
lastStatus?: number | null;
} = {},
) {
return {
providerType,
lastResult,
lastStatus: max.lastStatus ?? null,
_count: { _all: count },
_max: {
consecutiveFailures: max.consecutiveFailures ?? 0,
@@ -86,6 +93,7 @@ describe("GET /api/admin/provider-health", () => {
group("admin-openai", "hard_failed", 2, {
consecutiveFailures: 3334,
lastFailureAt: new Date("2026-08-27T06:00:00Z"),
lastStatus: 503,
}),
] as never);
@@ -99,10 +107,64 @@ describe("GET /api/admin/provider-health", () => {
maxConsecutiveFailures: 3334,
lastOkAt: "2026-08-27T07:00:00.000Z",
lastFailureAt: "2026-08-27T06:00:00.000Z",
lastFailureStatus: 503,
},
]);
});
it("reports the status of the newest failure, not the highest number", async () => {
// A 500 an hour ago and a 401 a minute ago: the operator needs the
// 401, because that is the one that will not clear on its own. Taking
// a MAX over the column would hand back the 500.
groupBy.mockResolvedValue([
group("openai", "hard_failed", 4, {
consecutiveFailures: 2,
lastFailureAt: new Date("2026-08-27T05:00:00Z"),
lastStatus: 500,
}),
group("openai", "auth_failed", 1, {
consecutiveFailures: 1,
lastFailureAt: new Date("2026-08-27T06:59:00Z"),
lastStatus: 401,
}),
] as never);
const res = await GET();
const body = await res.json();
expect(body.data.providers[0]).toMatchObject({
lastFailureAt: "2026-08-27T06:59:00.000Z",
lastFailureStatus: 401,
});
});
it("leaves the status out for a network-class failure that never had one", async () => {
groupBy.mockResolvedValue([
group("local", "hard_failed", 1, {
consecutiveFailures: 9,
lastFailureAt: new Date("2026-08-27T06:00:00Z"),
lastStatus: null,
}),
] as never);
const res = await GET();
const body = await res.json();
expect(body.data.providers[0]).toMatchObject({
lastFailureAt: "2026-08-27T06:00:00.000Z",
lastFailureStatus: null,
});
});
it("asks the ledger to split the fold by status rather than aggregate it", async () => {
groupBy.mockResolvedValue([] as never);
await GET();
expect(groupBy).toHaveBeenCalledWith(
expect.objectContaining({
by: ["providerType", "lastResult", "lastStatus"],
}),
);
});
it("counts auth_failed as failing and keeps an all-ok type at zero", async () => {
groupBy.mockResolvedValue([
group("codex", "auth_failed", 1, { consecutiveFailures: 7 }),
+23 -1
View File
@@ -15,6 +15,12 @@ export const dynamic = "force-dynamic";
* folds the ledger into one row per provider type so the console can show
* it. Read-only, admin-cookie-only, and it exposes counts and timestamps
* only — never which user a row belongs to.
*
* The ledger also records the HTTP status each failure came back with, and
* that is the difference between the two questions an operator actually has
* when a provider goes red: a 401 means the key is dead and only the operator
* can fix it, a 429 or a 5xx means the provider is having a day and waiting is
* the right move. The fold carries it beside the instant it belongs to.
*/
interface ProviderHealthSummary {
@@ -27,6 +33,13 @@ interface ProviderHealthSummary {
maxConsecutiveFailures: number;
lastOkAt: string | null;
lastFailureAt: string | null;
/**
* HTTP status the failure at `lastFailureAt` came back with. Null when
* that failure was a network-class one (no response to read a status
* from), and null when the user it belongs to has since recovered — a
* success clears the status along with the rest of the failure state.
*/
lastFailureStatus: number | null;
}
/** The two operator-managed tags sort first — they affect every user on the chain. */
@@ -37,7 +50,11 @@ export const GET = apiHandler(async () => {
annotate({ action: { name: "admin.provider-health.get" } });
const groups = await prisma.providerHealth.groupBy({
by: ["providerType", "lastResult"],
// `lastStatus` joins the grouping key rather than an aggregate: the
// MAX of two HTTP statuses is not a status, it is arithmetic on a
// label. Grouping by it splits each result bucket per status and the
// fold below picks the one belonging to the newest failure.
by: ["providerType", "lastResult", "lastStatus"],
_count: { _all: true },
_max: {
consecutiveFailures: true,
@@ -55,6 +72,7 @@ export const GET = apiHandler(async () => {
maxConsecutiveFailures: 0,
lastOkAt: null,
lastFailureAt: null,
lastFailureStatus: null,
};
row.tracked += g._count._all;
if (g.lastResult !== "ok") {
@@ -69,6 +87,10 @@ export const GET = apiHandler(async () => {
const fail = g._max.lastFailureAt?.toISOString() ?? null;
if (fail && (!row.lastFailureAt || fail > row.lastFailureAt)) {
row.lastFailureAt = fail;
// Tied to the instant, not taken from whichever group happens to
// have one: a status that describes a different failure than the
// one on screen is worse than no status at all.
row.lastFailureStatus = g.lastStatus;
}
byType.set(g.providerType, row);
}