mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-12 14:16:12 +00:00
feat(admin): name the status a failing provider came back with
The provider-health ledger has recorded the HTTP status of every failure since v1.11.0 and nothing has ever read it. The operator card could say three of five chains were failing without saying whose problem that was, which is the one thing the operator needs: a 401 is a credential only they can replace, a 429 or a 503 is the provider's afternoon and waiting is the right move. The fold carries the status of the failure whose instant the row already shows, not an aggregate over the column. A MAX of two HTTP statuses is arithmetic on a label, and a status describing a different failure than the one on screen would send the operator to the wrong place. The column joins the grouping key instead, and the fold picks the group belonging to the newest failure. Blank where there was nothing to record: a connection that never completed has no status, and a chain that has since recovered had its status cleared along with the rest of its failure state.
This commit is contained in:
1 parent
2ea08c6966
commit
7dbec3bc4d
12 files changed
+186
-2
No files matched your search
@@ -5,6 +5,11 @@
|
||||
* provider type, failing counted from the LAST result only, the worst
|
||||
* uninterrupted failure run taken across failing users, central types
|
||||
* sorted first, and no per-user data in the response.
|
||||
*
|
||||
* It also pins the HTTP status the ledger has recorded on every failure
|
||||
* since v1.11.0 and nothing read until now: it must describe the SAME
|
||||
* failure the row's instant names, or the card tells the operator to go
|
||||
* looking at the wrong thing.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
|
||||
@@ -49,11 +54,13 @@ function group(
|
||||
consecutiveFailures?: number;
|
||||
lastFailureAt?: Date | null;
|
||||
lastOkAt?: Date | null;
|
||||
lastStatus?: number | null;
|
||||
} = {},
|
||||
) {
|
||||
return {
|
||||
providerType,
|
||||
lastResult,
|
||||
lastStatus: max.lastStatus ?? null,
|
||||
_count: { _all: count },
|
||||
_max: {
|
||||
consecutiveFailures: max.consecutiveFailures ?? 0,
|
||||
@@ -86,6 +93,7 @@ describe("GET /api/admin/provider-health", () => {
|
||||
group("admin-openai", "hard_failed", 2, {
|
||||
consecutiveFailures: 3334,
|
||||
lastFailureAt: new Date("2026-08-27T06:00:00Z"),
|
||||
lastStatus: 503,
|
||||
}),
|
||||
] as never);
|
||||
|
||||
@@ -99,10 +107,64 @@ describe("GET /api/admin/provider-health", () => {
|
||||
maxConsecutiveFailures: 3334,
|
||||
lastOkAt: "2026-08-27T07:00:00.000Z",
|
||||
lastFailureAt: "2026-08-27T06:00:00.000Z",
|
||||
lastFailureStatus: 503,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports the status of the newest failure, not the highest number", async () => {
|
||||
// A 500 an hour ago and a 401 a minute ago: the operator needs the
|
||||
// 401, because that is the one that will not clear on its own. Taking
|
||||
// a MAX over the column would hand back the 500.
|
||||
groupBy.mockResolvedValue([
|
||||
group("openai", "hard_failed", 4, {
|
||||
consecutiveFailures: 2,
|
||||
lastFailureAt: new Date("2026-08-27T05:00:00Z"),
|
||||
lastStatus: 500,
|
||||
}),
|
||||
group("openai", "auth_failed", 1, {
|
||||
consecutiveFailures: 1,
|
||||
lastFailureAt: new Date("2026-08-27T06:59:00Z"),
|
||||
lastStatus: 401,
|
||||
}),
|
||||
] as never);
|
||||
|
||||
const res = await GET();
|
||||
const body = await res.json();
|
||||
expect(body.data.providers[0]).toMatchObject({
|
||||
lastFailureAt: "2026-08-27T06:59:00.000Z",
|
||||
lastFailureStatus: 401,
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves the status out for a network-class failure that never had one", async () => {
|
||||
groupBy.mockResolvedValue([
|
||||
group("local", "hard_failed", 1, {
|
||||
consecutiveFailures: 9,
|
||||
lastFailureAt: new Date("2026-08-27T06:00:00Z"),
|
||||
lastStatus: null,
|
||||
}),
|
||||
] as never);
|
||||
|
||||
const res = await GET();
|
||||
const body = await res.json();
|
||||
expect(body.data.providers[0]).toMatchObject({
|
||||
lastFailureAt: "2026-08-27T06:00:00.000Z",
|
||||
lastFailureStatus: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("asks the ledger to split the fold by status rather than aggregate it", async () => {
|
||||
groupBy.mockResolvedValue([] as never);
|
||||
await GET();
|
||||
|
||||
expect(groupBy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
by: ["providerType", "lastResult", "lastStatus"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("counts auth_failed as failing and keeps an all-ok type at zero", async () => {
|
||||
groupBy.mockResolvedValue([
|
||||
group("codex", "auth_failed", 1, { consecutiveFailures: 7 }),
|
||||
|
||||
@@ -15,6 +15,12 @@ export const dynamic = "force-dynamic";
|
||||
* folds the ledger into one row per provider type so the console can show
|
||||
* it. Read-only, admin-cookie-only, and it exposes counts and timestamps
|
||||
* only — never which user a row belongs to.
|
||||
*
|
||||
* The ledger also records the HTTP status each failure came back with, and
|
||||
* that is the difference between the two questions an operator actually has
|
||||
* when a provider goes red: a 401 means the key is dead and only the operator
|
||||
* can fix it, a 429 or a 5xx means the provider is having a day and waiting is
|
||||
* the right move. The fold carries it beside the instant it belongs to.
|
||||
*/
|
||||
|
||||
interface ProviderHealthSummary {
|
||||
@@ -27,6 +33,13 @@ interface ProviderHealthSummary {
|
||||
maxConsecutiveFailures: number;
|
||||
lastOkAt: string | null;
|
||||
lastFailureAt: string | null;
|
||||
/**
|
||||
* HTTP status the failure at `lastFailureAt` came back with. Null when
|
||||
* that failure was a network-class one (no response to read a status
|
||||
* from), and null when the user it belongs to has since recovered — a
|
||||
* success clears the status along with the rest of the failure state.
|
||||
*/
|
||||
lastFailureStatus: number | null;
|
||||
}
|
||||
|
||||
/** The two operator-managed tags sort first — they affect every user on the chain. */
|
||||
@@ -37,7 +50,11 @@ export const GET = apiHandler(async () => {
|
||||
annotate({ action: { name: "admin.provider-health.get" } });
|
||||
|
||||
const groups = await prisma.providerHealth.groupBy({
|
||||
by: ["providerType", "lastResult"],
|
||||
// `lastStatus` joins the grouping key rather than an aggregate: the
|
||||
// MAX of two HTTP statuses is not a status, it is arithmetic on a
|
||||
// label. Grouping by it splits each result bucket per status and the
|
||||
// fold below picks the one belonging to the newest failure.
|
||||
by: ["providerType", "lastResult", "lastStatus"],
|
||||
_count: { _all: true },
|
||||
_max: {
|
||||
consecutiveFailures: true,
|
||||
@@ -55,6 +72,7 @@ export const GET = apiHandler(async () => {
|
||||
maxConsecutiveFailures: 0,
|
||||
lastOkAt: null,
|
||||
lastFailureAt: null,
|
||||
lastFailureStatus: null,
|
||||
};
|
||||
row.tracked += g._count._all;
|
||||
if (g.lastResult !== "ok") {
|
||||
@@ -69,6 +87,10 @@ export const GET = apiHandler(async () => {
|
||||
const fail = g._max.lastFailureAt?.toISOString() ?? null;
|
||||
if (fail && (!row.lastFailureAt || fail > row.lastFailureAt)) {
|
||||
row.lastFailureAt = fail;
|
||||
// Tied to the instant, not taken from whichever group happens to
|
||||
// have one: a status that describes a different failure than the
|
||||
// one on screen is worse than no status at all.
|
||||
row.lastFailureStatus = g.lastStatus;
|
||||
}
|
||||
byType.set(g.providerType, row);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user