From 7dbec3bc4d9850a364817d78206bf6f7cb79268e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 3 Sep 2026 05:46:46 +0200 Subject: [PATCH] feat(admin): name the status a failing provider came back with The provider-health ledger has recorded the HTTP status of every failure since v1.11.0 and nothing has ever read it. The operator card could say three of five chains were failing without saying whose problem that was, which is the one thing the operator needs: a 401 is a credential only they can replace, a 429 or a 503 is the provider's afternoon and waiting is the right move. The fold carries the status of the failure whose instant the row already shows, not an aggregate over the column. A MAX of two HTTP statuses is arithmetic on a label, and a status describing a different failure than the one on screen would send the operator to the wrong place. The column joins the grouping key instead, and the fold picks the group belonging to the newest failure. Blank where there was nothing to record: a connection that never completed has no status, and a chain that has since recovered had its status cleared along with the rest of its failure state. --- messages/de.json | 1 + messages/en.json | 1 + messages/es.json | 1 + messages/fr.json | 1 + messages/it.json | 1 + messages/ko.json | 1 + messages/pl.json | 1 + prisma/schema.prisma | 8 +- .../provider-health/__tests__/route.test.ts | 62 ++++++++++++++++ src/app/api/admin/provider-health/route.ts | 24 +++++- .../provider-health-failure-status.test.tsx | 73 +++++++++++++++++++ .../admin/provider-health-section.tsx | 14 ++++ 12 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 src/components/admin/__tests__/provider-health-failure-status.test.tsx diff --git a/messages/de.json b/messages/de.json index 229068fe7..b437210a3 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6501,6 +6501,7 @@ "trackedUsers": "{count} Nutzer erfasst", "lastOk": "letzter Erfolg {at}", "lastFailure": "letzter Fehlschlag {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "In Ordnung", "statusFailing": "{failing} von {tracked} scheitern, {run} in Folge", "typeAdminOpenai": "Server-Schlüssel (Betreiber)", diff --git a/messages/en.json b/messages/en.json index a0de65aed..047e8a121 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6501,6 +6501,7 @@ "trackedUsers": "{count} users tracked", "lastOk": "last success {at}", "lastFailure": "last failure {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "Healthy", "statusFailing": "{failing} of {tracked} failing, {run} in a row", "typeAdminOpenai": "Server key (operator)", diff --git a/messages/es.json b/messages/es.json index e40df4240..259f30ae2 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6501,6 +6501,7 @@ "trackedUsers": "{count} usuarios registrados", "lastOk": "último éxito {at}", "lastFailure": "último fallo {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "Correcto", "statusFailing": "{failing} de {tracked} fallan, {run} seguidos", "typeAdminOpenai": "Clave del servidor (operador)", diff --git a/messages/fr.json b/messages/fr.json index da5b9e58c..9615e31bf 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6501,6 +6501,7 @@ "trackedUsers": "{count} utilisateurs suivis", "lastOk": "dernier succès {at}", "lastFailure": "dernier échec {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "En bon état", "statusFailing": "{failing} sur {tracked} en échec, {run} d'affilée", "typeAdminOpenai": "Clé du serveur (opérateur)", diff --git a/messages/it.json b/messages/it.json index 9410399b1..a341ddc18 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6501,6 +6501,7 @@ "trackedUsers": "{count} utenti tracciati", "lastOk": "ultimo successo {at}", "lastFailure": "ultimo errore {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "Regolare", "statusFailing": "{failing} su {tracked} in errore, {run} di fila", "typeAdminOpenai": "Chiave del server (operatore)", diff --git a/messages/ko.json b/messages/ko.json index e128a90ca..918359ecd 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7083,6 +7083,7 @@ "trackedUsers": "{count}명 사용자 추적됨", "lastOk": "마지막 성공 {at}", "lastFailure": "마지막 실패 {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "정상", "statusFailing": "{tracked}개 중 {failing}개 실패, {run}회 연속", "typeAdminOpenai": "서버 키 (운영자)", diff --git a/messages/pl.json b/messages/pl.json index 1e4a83fc1..b1aa8e00e 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6501,6 +6501,7 @@ "trackedUsers": "Śledzeni użytkownicy: {count}", "lastOk": "ostatni sukces {at}", "lastFailure": "ostatnia awaria {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "Sprawny", "statusFailing": "{failing} z {tracked} zawodzi, {run} z rzędu", "typeAdminOpenai": "Klucz serwera (operator)", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 77a61c6cf..11fce841a 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5070,7 +5070,13 @@ model ProviderHealth { userId String @map("user_id") providerType String @map("provider_type") // ProviderChainType tag lastResult String @map("last_result") // "ok" | "hard_failed" | "auth_failed" - lastStatus Int? @map("last_status") // HTTP status of the last failure; null on ok/network + /// HTTP status the last failure came back with; null on a success and on + /// a network-class failure that never had a response to read one from. + /// Consumer: the operator readout (`src/app/api/admin/provider-health/ + /// route.ts`), which carries it beside the failure instant it belongs to — + /// it is what separates a dead credential the operator has to replace from + /// a provider having a bad afternoon. + lastStatus Int? @map("last_status") consecutiveFailures Int @default(0) @map("consecutive_failures") lastOkAt DateTime? @map("last_ok_at") lastFailureAt DateTime? @map("last_failure_at") diff --git a/src/app/api/admin/provider-health/__tests__/route.test.ts b/src/app/api/admin/provider-health/__tests__/route.test.ts index 0ce584510..03e0fb8de 100644 --- a/src/app/api/admin/provider-health/__tests__/route.test.ts +++ b/src/app/api/admin/provider-health/__tests__/route.test.ts @@ -5,6 +5,11 @@ * provider type, failing counted from the LAST result only, the worst * uninterrupted failure run taken across failing users, central types * sorted first, and no per-user data in the response. + * + * It also pins the HTTP status the ledger has recorded on every failure + * since v1.11.0 and nothing read until now: it must describe the SAME + * failure the row's instant names, or the card tells the operator to go + * looking at the wrong thing. */ import { describe, it, expect, vi, beforeEach } from "vitest"; @@ -49,11 +54,13 @@ function group( consecutiveFailures?: number; lastFailureAt?: Date | null; lastOkAt?: Date | null; + lastStatus?: number | null; } = {}, ) { return { providerType, lastResult, + lastStatus: max.lastStatus ?? null, _count: { _all: count }, _max: { consecutiveFailures: max.consecutiveFailures ?? 0, @@ -86,6 +93,7 @@ describe("GET /api/admin/provider-health", () => { group("admin-openai", "hard_failed", 2, { consecutiveFailures: 3334, lastFailureAt: new Date("2026-08-27T06:00:00Z"), + lastStatus: 503, }), ] as never); @@ -99,10 +107,64 @@ describe("GET /api/admin/provider-health", () => { maxConsecutiveFailures: 3334, lastOkAt: "2026-08-27T07:00:00.000Z", lastFailureAt: "2026-08-27T06:00:00.000Z", + lastFailureStatus: 503, }, ]); }); + it("reports the status of the newest failure, not the highest number", async () => { + // A 500 an hour ago and a 401 a minute ago: the operator needs the + // 401, because that is the one that will not clear on its own. Taking + // a MAX over the column would hand back the 500. + groupBy.mockResolvedValue([ + group("openai", "hard_failed", 4, { + consecutiveFailures: 2, + lastFailureAt: new Date("2026-08-27T05:00:00Z"), + lastStatus: 500, + }), + group("openai", "auth_failed", 1, { + consecutiveFailures: 1, + lastFailureAt: new Date("2026-08-27T06:59:00Z"), + lastStatus: 401, + }), + ] as never); + + const res = await GET(); + const body = await res.json(); + expect(body.data.providers[0]).toMatchObject({ + lastFailureAt: "2026-08-27T06:59:00.000Z", + lastFailureStatus: 401, + }); + }); + + it("leaves the status out for a network-class failure that never had one", async () => { + groupBy.mockResolvedValue([ + group("local", "hard_failed", 1, { + consecutiveFailures: 9, + lastFailureAt: new Date("2026-08-27T06:00:00Z"), + lastStatus: null, + }), + ] as never); + + const res = await GET(); + const body = await res.json(); + expect(body.data.providers[0]).toMatchObject({ + lastFailureAt: "2026-08-27T06:00:00.000Z", + lastFailureStatus: null, + }); + }); + + it("asks the ledger to split the fold by status rather than aggregate it", async () => { + groupBy.mockResolvedValue([] as never); + await GET(); + + expect(groupBy).toHaveBeenCalledWith( + expect.objectContaining({ + by: ["providerType", "lastResult", "lastStatus"], + }), + ); + }); + it("counts auth_failed as failing and keeps an all-ok type at zero", async () => { groupBy.mockResolvedValue([ group("codex", "auth_failed", 1, { consecutiveFailures: 7 }), diff --git a/src/app/api/admin/provider-health/route.ts b/src/app/api/admin/provider-health/route.ts index 49b4660c7..c0163dece 100644 --- a/src/app/api/admin/provider-health/route.ts +++ b/src/app/api/admin/provider-health/route.ts @@ -15,6 +15,12 @@ export const dynamic = "force-dynamic"; * folds the ledger into one row per provider type so the console can show * it. Read-only, admin-cookie-only, and it exposes counts and timestamps * only — never which user a row belongs to. + * + * The ledger also records the HTTP status each failure came back with, and + * that is the difference between the two questions an operator actually has + * when a provider goes red: a 401 means the key is dead and only the operator + * can fix it, a 429 or a 5xx means the provider is having a day and waiting is + * the right move. The fold carries it beside the instant it belongs to. */ interface ProviderHealthSummary { @@ -27,6 +33,13 @@ interface ProviderHealthSummary { maxConsecutiveFailures: number; lastOkAt: string | null; lastFailureAt: string | null; + /** + * HTTP status the failure at `lastFailureAt` came back with. Null when + * that failure was a network-class one (no response to read a status + * from), and null when the user it belongs to has since recovered — a + * success clears the status along with the rest of the failure state. + */ + lastFailureStatus: number | null; } /** The two operator-managed tags sort first — they affect every user on the chain. */ @@ -37,7 +50,11 @@ export const GET = apiHandler(async () => { annotate({ action: { name: "admin.provider-health.get" } }); const groups = await prisma.providerHealth.groupBy({ - by: ["providerType", "lastResult"], + // `lastStatus` joins the grouping key rather than an aggregate: the + // MAX of two HTTP statuses is not a status, it is arithmetic on a + // label. Grouping by it splits each result bucket per status and the + // fold below picks the one belonging to the newest failure. + by: ["providerType", "lastResult", "lastStatus"], _count: { _all: true }, _max: { consecutiveFailures: true, @@ -55,6 +72,7 @@ export const GET = apiHandler(async () => { maxConsecutiveFailures: 0, lastOkAt: null, lastFailureAt: null, + lastFailureStatus: null, }; row.tracked += g._count._all; if (g.lastResult !== "ok") { @@ -69,6 +87,10 @@ export const GET = apiHandler(async () => { const fail = g._max.lastFailureAt?.toISOString() ?? null; if (fail && (!row.lastFailureAt || fail > row.lastFailureAt)) { row.lastFailureAt = fail; + // Tied to the instant, not taken from whichever group happens to + // have one: a status that describes a different failure than the + // one on screen is worse than no status at all. + row.lastFailureStatus = g.lastStatus; } byType.set(g.providerType, row); } diff --git a/src/components/admin/__tests__/provider-health-failure-status.test.tsx b/src/components/admin/__tests__/provider-health-failure-status.test.tsx new file mode 100644 index 000000000..018ecac80 --- /dev/null +++ b/src/components/admin/__tests__/provider-health-failure-status.test.tsx @@ -0,0 +1,73 @@ +import { describe, it, expect, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; + +import { I18nProvider } from "@/lib/i18n/context"; + +/** + * The provider-health card and the HTTP status of the last failure. + * + * The ledger has recorded a status on every failure since v1.11.0 and no + * surface read it, so the card could say a provider was failing without + * saying whose problem it was. A 401 is a credential the operator has to + * replace; a 503 is the provider's afternoon. These pin that the status + * reaches the row, that it is left out when there was none to record, + * and that a healthy provider does not carry a stale one. + */ + +const mockQueryState = vi.hoisted(() => ({ + data: null as null | object, + isPending: false, + isError: false, +})); + +vi.mock("@tanstack/react-query", () => ({ + useQuery: () => ({ + data: mockQueryState.data, + isPending: mockQueryState.isPending, + isError: mockQueryState.isError, + }), +})); + +import { ProviderHealthSection } from "../provider-health-section"; + +function row(over: Record = {}) { + return { + providerType: "admin-openai", + tracked: 5, + failing: 2, + maxConsecutiveFailures: 4, + lastOkAt: null, + lastFailureAt: "2026-08-27T06:00:00.000Z", + lastFailureStatus: 401, + ...over, + }; +} + +function render(providers: object[]) { + mockQueryState.data = { providers }; + mockQueryState.isPending = false; + mockQueryState.isError = false; + return renderToStaticMarkup( + + + , + ); +} + +describe(" — the last failure's HTTP status", () => { + it("names the status beside the failure it belongs to", () => { + expect(render([row()])).toContain("HTTP 401"); + }); + + it("says nothing where the failure never had a status to record", () => { + expect(render([row({ lastFailureStatus: null })])).not.toContain("HTTP"); + }); + + it("carries no status for a provider that is not failing", () => { + expect( + render([ + row({ failing: 0, lastFailureAt: null, lastFailureStatus: 500 }), + ]), + ).not.toContain("HTTP"); + }); +}); diff --git a/src/components/admin/provider-health-section.tsx b/src/components/admin/provider-health-section.tsx index 70e27328c..d4568c20f 100644 --- a/src/components/admin/provider-health-section.tsx +++ b/src/components/admin/provider-health-section.tsx @@ -18,6 +18,12 @@ import { formatDateTime } from "@/lib/format"; * read the database. The card shows one row per provider type: how many * users' chains touch it, how many of them are currently failing, the worst * uninterrupted failure run, and the last success / failure instants. + * + * The failure line also carries the HTTP status the ledger recorded for that + * failure, because "failing" on its own does not say whose problem it is. A + * 401 is a dead credential and waiting will not fix it; a 429 or a 503 is the + * provider's day and waiting is exactly right. Absent for a network-class + * failure, which never had a status to record. */ interface ProviderHealthRow { @@ -27,6 +33,7 @@ interface ProviderHealthRow { maxConsecutiveFailures: number; lastOkAt: string | null; lastFailureAt: string | null; + lastFailureStatus: number | null; } export function ProviderHealthSection() { @@ -106,6 +113,13 @@ export function ProviderHealthSection() { at: formatDateTime(p.lastFailureAt), })}` : ""} + {p.failing > 0 && + p.lastFailureAt && + p.lastFailureStatus !== null + ? ` · ${t("admin.providerHealth.lastFailureStatus", { + status: p.lastFailureStatus, + })}` + : ""} {p.failing === 0 ? (