mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 04:50:59 +00:00
feat(admin): name the status a failing provider came back with
The provider-health ledger has recorded the HTTP status of every failure since v1.11.0 and nothing has ever read it. The operator card could say three of five chains were failing without saying whose problem that was, which is the one thing the operator needs: a 401 is a credential only they can replace, a 429 or a 503 is the provider's afternoon and waiting is the right move. The fold carries the status of the failure whose instant the row already shows, not an aggregate over the column. A MAX of two HTTP statuses is arithmetic on a label, and a status describing a different failure than the one on screen would send the operator to the wrong place. The column joins the grouping key instead, and the fold picks the group belonging to the newest failure. Blank where there was nothing to record: a connection that never completed has no status, and a chain that has since recovered had its status cleared along with the rest of its failure state.
This commit is contained in:
1 parent
2ea08c6966
commit
7dbec3bc4d
12 files changed
+186
-2
No files matched your search
@@ -6501,6 +6501,7 @@
|
||||
"trackedUsers": "{count} Nutzer erfasst",
|
||||
"lastOk": "letzter Erfolg {at}",
|
||||
"lastFailure": "letzter Fehlschlag {at}",
|
||||
"lastFailureStatus": "HTTP {status}",
|
||||
"statusHealthy": "In Ordnung",
|
||||
"statusFailing": "{failing} von {tracked} scheitern, {run} in Folge",
|
||||
"typeAdminOpenai": "Server-Schlüssel (Betreiber)",
|
||||
|
||||
@@ -6501,6 +6501,7 @@
|
||||
"trackedUsers": "{count} users tracked",
|
||||
"lastOk": "last success {at}",
|
||||
"lastFailure": "last failure {at}",
|
||||
"lastFailureStatus": "HTTP {status}",
|
||||
"statusHealthy": "Healthy",
|
||||
"statusFailing": "{failing} of {tracked} failing, {run} in a row",
|
||||
"typeAdminOpenai": "Server key (operator)",
|
||||
|
||||
@@ -6501,6 +6501,7 @@
|
||||
"trackedUsers": "{count} usuarios registrados",
|
||||
"lastOk": "último éxito {at}",
|
||||
"lastFailure": "último fallo {at}",
|
||||
"lastFailureStatus": "HTTP {status}",
|
||||
"statusHealthy": "Correcto",
|
||||
"statusFailing": "{failing} de {tracked} fallan, {run} seguidos",
|
||||
"typeAdminOpenai": "Clave del servidor (operador)",
|
||||
|
||||
@@ -6501,6 +6501,7 @@
|
||||
"trackedUsers": "{count} utilisateurs suivis",
|
||||
"lastOk": "dernier succès {at}",
|
||||
"lastFailure": "dernier échec {at}",
|
||||
"lastFailureStatus": "HTTP {status}",
|
||||
"statusHealthy": "En bon état",
|
||||
"statusFailing": "{failing} sur {tracked} en échec, {run} d'affilée",
|
||||
"typeAdminOpenai": "Clé du serveur (opérateur)",
|
||||
|
||||
@@ -6501,6 +6501,7 @@
|
||||
"trackedUsers": "{count} utenti tracciati",
|
||||
"lastOk": "ultimo successo {at}",
|
||||
"lastFailure": "ultimo errore {at}",
|
||||
"lastFailureStatus": "HTTP {status}",
|
||||
"statusHealthy": "Regolare",
|
||||
"statusFailing": "{failing} su {tracked} in errore, {run} di fila",
|
||||
"typeAdminOpenai": "Chiave del server (operatore)",
|
||||
|
||||
@@ -7083,6 +7083,7 @@
|
||||
"trackedUsers": "{count}명 사용자 추적됨",
|
||||
"lastOk": "마지막 성공 {at}",
|
||||
"lastFailure": "마지막 실패 {at}",
|
||||
"lastFailureStatus": "HTTP {status}",
|
||||
"statusHealthy": "정상",
|
||||
"statusFailing": "{tracked}개 중 {failing}개 실패, {run}회 연속",
|
||||
"typeAdminOpenai": "서버 키 (운영자)",
|
||||
|
||||
@@ -6501,6 +6501,7 @@
|
||||
"trackedUsers": "Śledzeni użytkownicy: {count}",
|
||||
"lastOk": "ostatni sukces {at}",
|
||||
"lastFailure": "ostatnia awaria {at}",
|
||||
"lastFailureStatus": "HTTP {status}",
|
||||
"statusHealthy": "Sprawny",
|
||||
"statusFailing": "{failing} z {tracked} zawodzi, {run} z rzędu",
|
||||
"typeAdminOpenai": "Klucz serwera (operator)",
|
||||
|
||||
@@ -5070,7 +5070,13 @@ model ProviderHealth {
|
||||
userId String @map("user_id")
|
||||
providerType String @map("provider_type") // ProviderChainType tag
|
||||
lastResult String @map("last_result") // "ok" | "hard_failed" | "auth_failed"
|
||||
lastStatus Int? @map("last_status") // HTTP status of the last failure; null on ok/network
|
||||
/// HTTP status the last failure came back with; null on a success and on
|
||||
/// a network-class failure that never had a response to read one from.
|
||||
/// Consumer: the operator readout (`src/app/api/admin/provider-health/
|
||||
/// route.ts`), which carries it beside the failure instant it belongs to —
|
||||
/// it is what separates a dead credential the operator has to replace from
|
||||
/// a provider having a bad afternoon.
|
||||
lastStatus Int? @map("last_status")
|
||||
consecutiveFailures Int @default(0) @map("consecutive_failures")
|
||||
lastOkAt DateTime? @map("last_ok_at")
|
||||
lastFailureAt DateTime? @map("last_failure_at")
|
||||
|
||||
@@ -5,6 +5,11 @@
|
||||
* provider type, failing counted from the LAST result only, the worst
|
||||
* uninterrupted failure run taken across failing users, central types
|
||||
* sorted first, and no per-user data in the response.
|
||||
*
|
||||
* It also pins the HTTP status the ledger has recorded on every failure
|
||||
* since v1.11.0 and nothing read until now: it must describe the SAME
|
||||
* failure the row's instant names, or the card tells the operator to go
|
||||
* looking at the wrong thing.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
|
||||
@@ -49,11 +54,13 @@ function group(
|
||||
consecutiveFailures?: number;
|
||||
lastFailureAt?: Date | null;
|
||||
lastOkAt?: Date | null;
|
||||
lastStatus?: number | null;
|
||||
} = {},
|
||||
) {
|
||||
return {
|
||||
providerType,
|
||||
lastResult,
|
||||
lastStatus: max.lastStatus ?? null,
|
||||
_count: { _all: count },
|
||||
_max: {
|
||||
consecutiveFailures: max.consecutiveFailures ?? 0,
|
||||
@@ -86,6 +93,7 @@ describe("GET /api/admin/provider-health", () => {
|
||||
group("admin-openai", "hard_failed", 2, {
|
||||
consecutiveFailures: 3334,
|
||||
lastFailureAt: new Date("2026-08-27T06:00:00Z"),
|
||||
lastStatus: 503,
|
||||
}),
|
||||
] as never);
|
||||
|
||||
@@ -99,10 +107,64 @@ describe("GET /api/admin/provider-health", () => {
|
||||
maxConsecutiveFailures: 3334,
|
||||
lastOkAt: "2026-08-27T07:00:00.000Z",
|
||||
lastFailureAt: "2026-08-27T06:00:00.000Z",
|
||||
lastFailureStatus: 503,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports the status of the newest failure, not the highest number", async () => {
|
||||
// A 500 an hour ago and a 401 a minute ago: the operator needs the
|
||||
// 401, because that is the one that will not clear on its own. Taking
|
||||
// a MAX over the column would hand back the 500.
|
||||
groupBy.mockResolvedValue([
|
||||
group("openai", "hard_failed", 4, {
|
||||
consecutiveFailures: 2,
|
||||
lastFailureAt: new Date("2026-08-27T05:00:00Z"),
|
||||
lastStatus: 500,
|
||||
}),
|
||||
group("openai", "auth_failed", 1, {
|
||||
consecutiveFailures: 1,
|
||||
lastFailureAt: new Date("2026-08-27T06:59:00Z"),
|
||||
lastStatus: 401,
|
||||
}),
|
||||
] as never);
|
||||
|
||||
const res = await GET();
|
||||
const body = await res.json();
|
||||
expect(body.data.providers[0]).toMatchObject({
|
||||
lastFailureAt: "2026-08-27T06:59:00.000Z",
|
||||
lastFailureStatus: 401,
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves the status out for a network-class failure that never had one", async () => {
|
||||
groupBy.mockResolvedValue([
|
||||
group("local", "hard_failed", 1, {
|
||||
consecutiveFailures: 9,
|
||||
lastFailureAt: new Date("2026-08-27T06:00:00Z"),
|
||||
lastStatus: null,
|
||||
}),
|
||||
] as never);
|
||||
|
||||
const res = await GET();
|
||||
const body = await res.json();
|
||||
expect(body.data.providers[0]).toMatchObject({
|
||||
lastFailureAt: "2026-08-27T06:00:00.000Z",
|
||||
lastFailureStatus: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("asks the ledger to split the fold by status rather than aggregate it", async () => {
|
||||
groupBy.mockResolvedValue([] as never);
|
||||
await GET();
|
||||
|
||||
expect(groupBy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
by: ["providerType", "lastResult", "lastStatus"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("counts auth_failed as failing and keeps an all-ok type at zero", async () => {
|
||||
groupBy.mockResolvedValue([
|
||||
group("codex", "auth_failed", 1, { consecutiveFailures: 7 }),
|
||||
|
||||
@@ -15,6 +15,12 @@ export const dynamic = "force-dynamic";
|
||||
* folds the ledger into one row per provider type so the console can show
|
||||
* it. Read-only, admin-cookie-only, and it exposes counts and timestamps
|
||||
* only — never which user a row belongs to.
|
||||
*
|
||||
* The ledger also records the HTTP status each failure came back with, and
|
||||
* that is the difference between the two questions an operator actually has
|
||||
* when a provider goes red: a 401 means the key is dead and only the operator
|
||||
* can fix it, a 429 or a 5xx means the provider is having a day and waiting is
|
||||
* the right move. The fold carries it beside the instant it belongs to.
|
||||
*/
|
||||
|
||||
interface ProviderHealthSummary {
|
||||
@@ -27,6 +33,13 @@ interface ProviderHealthSummary {
|
||||
maxConsecutiveFailures: number;
|
||||
lastOkAt: string | null;
|
||||
lastFailureAt: string | null;
|
||||
/**
|
||||
* HTTP status the failure at `lastFailureAt` came back with. Null when
|
||||
* that failure was a network-class one (no response to read a status
|
||||
* from), and null when the user it belongs to has since recovered — a
|
||||
* success clears the status along with the rest of the failure state.
|
||||
*/
|
||||
lastFailureStatus: number | null;
|
||||
}
|
||||
|
||||
/** The two operator-managed tags sort first — they affect every user on the chain. */
|
||||
@@ -37,7 +50,11 @@ export const GET = apiHandler(async () => {
|
||||
annotate({ action: { name: "admin.provider-health.get" } });
|
||||
|
||||
const groups = await prisma.providerHealth.groupBy({
|
||||
by: ["providerType", "lastResult"],
|
||||
// `lastStatus` joins the grouping key rather than an aggregate: the
|
||||
// MAX of two HTTP statuses is not a status, it is arithmetic on a
|
||||
// label. Grouping by it splits each result bucket per status and the
|
||||
// fold below picks the one belonging to the newest failure.
|
||||
by: ["providerType", "lastResult", "lastStatus"],
|
||||
_count: { _all: true },
|
||||
_max: {
|
||||
consecutiveFailures: true,
|
||||
@@ -55,6 +72,7 @@ export const GET = apiHandler(async () => {
|
||||
maxConsecutiveFailures: 0,
|
||||
lastOkAt: null,
|
||||
lastFailureAt: null,
|
||||
lastFailureStatus: null,
|
||||
};
|
||||
row.tracked += g._count._all;
|
||||
if (g.lastResult !== "ok") {
|
||||
@@ -69,6 +87,10 @@ export const GET = apiHandler(async () => {
|
||||
const fail = g._max.lastFailureAt?.toISOString() ?? null;
|
||||
if (fail && (!row.lastFailureAt || fail > row.lastFailureAt)) {
|
||||
row.lastFailureAt = fail;
|
||||
// Tied to the instant, not taken from whichever group happens to
|
||||
// have one: a status that describes a different failure than the
|
||||
// one on screen is worse than no status at all.
|
||||
row.lastFailureStatus = g.lastStatus;
|
||||
}
|
||||
byType.set(g.providerType, row);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
|
||||
import { I18nProvider } from "@/lib/i18n/context";
|
||||
|
||||
/**
|
||||
* The provider-health card and the HTTP status of the last failure.
|
||||
*
|
||||
* The ledger has recorded a status on every failure since v1.11.0 and no
|
||||
* surface read it, so the card could say a provider was failing without
|
||||
* saying whose problem it was. A 401 is a credential the operator has to
|
||||
* replace; a 503 is the provider's afternoon. These pin that the status
|
||||
* reaches the row, that it is left out when there was none to record,
|
||||
* and that a healthy provider does not carry a stale one.
|
||||
*/
|
||||
|
||||
const mockQueryState = vi.hoisted(() => ({
|
||||
data: null as null | object,
|
||||
isPending: false,
|
||||
isError: false,
|
||||
}));
|
||||
|
||||
vi.mock("@tanstack/react-query", () => ({
|
||||
useQuery: () => ({
|
||||
data: mockQueryState.data,
|
||||
isPending: mockQueryState.isPending,
|
||||
isError: mockQueryState.isError,
|
||||
}),
|
||||
}));
|
||||
|
||||
import { ProviderHealthSection } from "../provider-health-section";
|
||||
|
||||
function row(over: Record<string, unknown> = {}) {
|
||||
return {
|
||||
providerType: "admin-openai",
|
||||
tracked: 5,
|
||||
failing: 2,
|
||||
maxConsecutiveFailures: 4,
|
||||
lastOkAt: null,
|
||||
lastFailureAt: "2026-08-27T06:00:00.000Z",
|
||||
lastFailureStatus: 401,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
function render(providers: object[]) {
|
||||
mockQueryState.data = { providers };
|
||||
mockQueryState.isPending = false;
|
||||
mockQueryState.isError = false;
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider initialLocale="en">
|
||||
<ProviderHealthSection />
|
||||
</I18nProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
describe("<ProviderHealthSection> — the last failure's HTTP status", () => {
|
||||
it("names the status beside the failure it belongs to", () => {
|
||||
expect(render([row()])).toContain("HTTP 401");
|
||||
});
|
||||
|
||||
it("says nothing where the failure never had a status to record", () => {
|
||||
expect(render([row({ lastFailureStatus: null })])).not.toContain("HTTP");
|
||||
});
|
||||
|
||||
it("carries no status for a provider that is not failing", () => {
|
||||
expect(
|
||||
render([
|
||||
row({ failing: 0, lastFailureAt: null, lastFailureStatus: 500 }),
|
||||
]),
|
||||
).not.toContain("HTTP");
|
||||
});
|
||||
});
|
||||
@@ -18,6 +18,12 @@ import { formatDateTime } from "@/lib/format";
|
||||
* read the database. The card shows one row per provider type: how many
|
||||
* users' chains touch it, how many of them are currently failing, the worst
|
||||
* uninterrupted failure run, and the last success / failure instants.
|
||||
*
|
||||
* The failure line also carries the HTTP status the ledger recorded for that
|
||||
* failure, because "failing" on its own does not say whose problem it is. A
|
||||
* 401 is a dead credential and waiting will not fix it; a 429 or a 503 is the
|
||||
* provider's day and waiting is exactly right. Absent for a network-class
|
||||
* failure, which never had a status to record.
|
||||
*/
|
||||
|
||||
interface ProviderHealthRow {
|
||||
@@ -27,6 +33,7 @@ interface ProviderHealthRow {
|
||||
maxConsecutiveFailures: number;
|
||||
lastOkAt: string | null;
|
||||
lastFailureAt: string | null;
|
||||
lastFailureStatus: number | null;
|
||||
}
|
||||
|
||||
export function ProviderHealthSection() {
|
||||
@@ -106,6 +113,13 @@ export function ProviderHealthSection() {
|
||||
at: formatDateTime(p.lastFailureAt),
|
||||
})}`
|
||||
: ""}
|
||||
{p.failing > 0 &&
|
||||
p.lastFailureAt &&
|
||||
p.lastFailureStatus !== null
|
||||
? ` · ${t("admin.providerHealth.lastFailureStatus", {
|
||||
status: p.lastFailureStatus,
|
||||
})}`
|
||||
: ""}
|
||||
</span>
|
||||
</div>
|
||||
{p.failing === 0 ? (
|
||||
|
||||
Reference in new issue
Block a user