diff --git a/messages/de.json b/messages/de.json index 229068fe7..b437210a3 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6501,6 +6501,7 @@ "trackedUsers": "{count} Nutzer erfasst", "lastOk": "letzter Erfolg {at}", "lastFailure": "letzter Fehlschlag {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "In Ordnung", "statusFailing": "{failing} von {tracked} scheitern, {run} in Folge", "typeAdminOpenai": "Server-Schlüssel (Betreiber)", diff --git a/messages/en.json b/messages/en.json index a0de65aed..047e8a121 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6501,6 +6501,7 @@ "trackedUsers": "{count} users tracked", "lastOk": "last success {at}", "lastFailure": "last failure {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "Healthy", "statusFailing": "{failing} of {tracked} failing, {run} in a row", "typeAdminOpenai": "Server key (operator)", diff --git a/messages/es.json b/messages/es.json index e40df4240..259f30ae2 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6501,6 +6501,7 @@ "trackedUsers": "{count} usuarios registrados", "lastOk": "último éxito {at}", "lastFailure": "último fallo {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "Correcto", "statusFailing": "{failing} de {tracked} fallan, {run} seguidos", "typeAdminOpenai": "Clave del servidor (operador)", diff --git a/messages/fr.json b/messages/fr.json index da5b9e58c..9615e31bf 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6501,6 +6501,7 @@ "trackedUsers": "{count} utilisateurs suivis", "lastOk": "dernier succès {at}", "lastFailure": "dernier échec {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "En bon état", "statusFailing": "{failing} sur {tracked} en échec, {run} d'affilée", "typeAdminOpenai": "Clé du serveur (opérateur)", diff --git a/messages/it.json b/messages/it.json index 9410399b1..a341ddc18 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6501,6 +6501,7 @@ "trackedUsers": "{count} utenti tracciati", "lastOk": "ultimo successo {at}", "lastFailure": "ultimo errore {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "Regolare", "statusFailing": "{failing} su {tracked} in errore, {run} di fila", "typeAdminOpenai": "Chiave del server (operatore)", diff --git a/messages/ko.json b/messages/ko.json index e128a90ca..918359ecd 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7083,6 +7083,7 @@ "trackedUsers": "{count}명 사용자 추적됨", "lastOk": "마지막 성공 {at}", "lastFailure": "마지막 실패 {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "정상", "statusFailing": "{tracked}개 중 {failing}개 실패, {run}회 연속", "typeAdminOpenai": "서버 키 (운영자)", diff --git a/messages/pl.json b/messages/pl.json index 1e4a83fc1..b1aa8e00e 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6501,6 +6501,7 @@ "trackedUsers": "Śledzeni użytkownicy: {count}", "lastOk": "ostatni sukces {at}", "lastFailure": "ostatnia awaria {at}", + "lastFailureStatus": "HTTP {status}", "statusHealthy": "Sprawny", "statusFailing": "{failing} z {tracked} zawodzi, {run} z rzędu", "typeAdminOpenai": "Klucz serwera (operator)", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 77a61c6cf..11fce841a 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5070,7 +5070,13 @@ model ProviderHealth { userId String @map("user_id") providerType String @map("provider_type") // ProviderChainType tag lastResult String @map("last_result") // "ok" | "hard_failed" | "auth_failed" - lastStatus Int? @map("last_status") // HTTP status of the last failure; null on ok/network + /// HTTP status the last failure came back with; null on a success and on + /// a network-class failure that never had a response to read one from. + /// Consumer: the operator readout (`src/app/api/admin/provider-health/ + /// route.ts`), which carries it beside the failure instant it belongs to — + /// it is what separates a dead credential the operator has to replace from + /// a provider having a bad afternoon. + lastStatus Int? @map("last_status") consecutiveFailures Int @default(0) @map("consecutive_failures") lastOkAt DateTime? @map("last_ok_at") lastFailureAt DateTime? @map("last_failure_at") diff --git a/src/app/api/admin/provider-health/__tests__/route.test.ts b/src/app/api/admin/provider-health/__tests__/route.test.ts index 0ce584510..03e0fb8de 100644 --- a/src/app/api/admin/provider-health/__tests__/route.test.ts +++ b/src/app/api/admin/provider-health/__tests__/route.test.ts @@ -5,6 +5,11 @@ * provider type, failing counted from the LAST result only, the worst * uninterrupted failure run taken across failing users, central types * sorted first, and no per-user data in the response. + * + * It also pins the HTTP status the ledger has recorded on every failure + * since v1.11.0 and nothing read until now: it must describe the SAME + * failure the row's instant names, or the card tells the operator to go + * looking at the wrong thing. */ import { describe, it, expect, vi, beforeEach } from "vitest"; @@ -49,11 +54,13 @@ function group( consecutiveFailures?: number; lastFailureAt?: Date | null; lastOkAt?: Date | null; + lastStatus?: number | null; } = {}, ) { return { providerType, lastResult, + lastStatus: max.lastStatus ?? null, _count: { _all: count }, _max: { consecutiveFailures: max.consecutiveFailures ?? 0, @@ -86,6 +93,7 @@ describe("GET /api/admin/provider-health", () => { group("admin-openai", "hard_failed", 2, { consecutiveFailures: 3334, lastFailureAt: new Date("2026-08-27T06:00:00Z"), + lastStatus: 503, }), ] as never); @@ -99,10 +107,64 @@ describe("GET /api/admin/provider-health", () => { maxConsecutiveFailures: 3334, lastOkAt: "2026-08-27T07:00:00.000Z", lastFailureAt: "2026-08-27T06:00:00.000Z", + lastFailureStatus: 503, }, ]); }); + it("reports the status of the newest failure, not the highest number", async () => { + // A 500 an hour ago and a 401 a minute ago: the operator needs the + // 401, because that is the one that will not clear on its own. Taking + // a MAX over the column would hand back the 500. + groupBy.mockResolvedValue([ + group("openai", "hard_failed", 4, { + consecutiveFailures: 2, + lastFailureAt: new Date("2026-08-27T05:00:00Z"), + lastStatus: 500, + }), + group("openai", "auth_failed", 1, { + consecutiveFailures: 1, + lastFailureAt: new Date("2026-08-27T06:59:00Z"), + lastStatus: 401, + }), + ] as never); + + const res = await GET(); + const body = await res.json(); + expect(body.data.providers[0]).toMatchObject({ + lastFailureAt: "2026-08-27T06:59:00.000Z", + lastFailureStatus: 401, + }); + }); + + it("leaves the status out for a network-class failure that never had one", async () => { + groupBy.mockResolvedValue([ + group("local", "hard_failed", 1, { + consecutiveFailures: 9, + lastFailureAt: new Date("2026-08-27T06:00:00Z"), + lastStatus: null, + }), + ] as never); + + const res = await GET(); + const body = await res.json(); + expect(body.data.providers[0]).toMatchObject({ + lastFailureAt: "2026-08-27T06:00:00.000Z", + lastFailureStatus: null, + }); + }); + + it("asks the ledger to split the fold by status rather than aggregate it", async () => { + groupBy.mockResolvedValue([] as never); + await GET(); + + expect(groupBy).toHaveBeenCalledWith( + expect.objectContaining({ + by: ["providerType", "lastResult", "lastStatus"], + }), + ); + }); + it("counts auth_failed as failing and keeps an all-ok type at zero", async () => { groupBy.mockResolvedValue([ group("codex", "auth_failed", 1, { consecutiveFailures: 7 }), diff --git a/src/app/api/admin/provider-health/route.ts b/src/app/api/admin/provider-health/route.ts index 49b4660c7..c0163dece 100644 --- a/src/app/api/admin/provider-health/route.ts +++ b/src/app/api/admin/provider-health/route.ts @@ -15,6 +15,12 @@ export const dynamic = "force-dynamic"; * folds the ledger into one row per provider type so the console can show * it. Read-only, admin-cookie-only, and it exposes counts and timestamps * only — never which user a row belongs to. + * + * The ledger also records the HTTP status each failure came back with, and + * that is the difference between the two questions an operator actually has + * when a provider goes red: a 401 means the key is dead and only the operator + * can fix it, a 429 or a 5xx means the provider is having a day and waiting is + * the right move. The fold carries it beside the instant it belongs to. */ interface ProviderHealthSummary { @@ -27,6 +33,13 @@ interface ProviderHealthSummary { maxConsecutiveFailures: number; lastOkAt: string | null; lastFailureAt: string | null; + /** + * HTTP status the failure at `lastFailureAt` came back with. Null when + * that failure was a network-class one (no response to read a status + * from), and null when the user it belongs to has since recovered — a + * success clears the status along with the rest of the failure state. + */ + lastFailureStatus: number | null; } /** The two operator-managed tags sort first — they affect every user on the chain. */ @@ -37,7 +50,11 @@ export const GET = apiHandler(async () => { annotate({ action: { name: "admin.provider-health.get" } }); const groups = await prisma.providerHealth.groupBy({ - by: ["providerType", "lastResult"], + // `lastStatus` joins the grouping key rather than an aggregate: the + // MAX of two HTTP statuses is not a status, it is arithmetic on a + // label. Grouping by it splits each result bucket per status and the + // fold below picks the one belonging to the newest failure. + by: ["providerType", "lastResult", "lastStatus"], _count: { _all: true }, _max: { consecutiveFailures: true, @@ -55,6 +72,7 @@ export const GET = apiHandler(async () => { maxConsecutiveFailures: 0, lastOkAt: null, lastFailureAt: null, + lastFailureStatus: null, }; row.tracked += g._count._all; if (g.lastResult !== "ok") { @@ -69,6 +87,10 @@ export const GET = apiHandler(async () => { const fail = g._max.lastFailureAt?.toISOString() ?? null; if (fail && (!row.lastFailureAt || fail > row.lastFailureAt)) { row.lastFailureAt = fail; + // Tied to the instant, not taken from whichever group happens to + // have one: a status that describes a different failure than the + // one on screen is worse than no status at all. + row.lastFailureStatus = g.lastStatus; } byType.set(g.providerType, row); } diff --git a/src/components/admin/__tests__/provider-health-failure-status.test.tsx b/src/components/admin/__tests__/provider-health-failure-status.test.tsx new file mode 100644 index 000000000..018ecac80 --- /dev/null +++ b/src/components/admin/__tests__/provider-health-failure-status.test.tsx @@ -0,0 +1,73 @@ +import { describe, it, expect, vi } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; + +import { I18nProvider } from "@/lib/i18n/context"; + +/** + * The provider-health card and the HTTP status of the last failure. + * + * The ledger has recorded a status on every failure since v1.11.0 and no + * surface read it, so the card could say a provider was failing without + * saying whose problem it was. A 401 is a credential the operator has to + * replace; a 503 is the provider's afternoon. These pin that the status + * reaches the row, that it is left out when there was none to record, + * and that a healthy provider does not carry a stale one. + */ + +const mockQueryState = vi.hoisted(() => ({ + data: null as null | object, + isPending: false, + isError: false, +})); + +vi.mock("@tanstack/react-query", () => ({ + useQuery: () => ({ + data: mockQueryState.data, + isPending: mockQueryState.isPending, + isError: mockQueryState.isError, + }), +})); + +import { ProviderHealthSection } from "../provider-health-section"; + +function row(over: Record = {}) { + return { + providerType: "admin-openai", + tracked: 5, + failing: 2, + maxConsecutiveFailures: 4, + lastOkAt: null, + lastFailureAt: "2026-08-27T06:00:00.000Z", + lastFailureStatus: 401, + ...over, + }; +} + +function render(providers: object[]) { + mockQueryState.data = { providers }; + mockQueryState.isPending = false; + mockQueryState.isError = false; + return renderToStaticMarkup( + + + , + ); +} + +describe(" — the last failure's HTTP status", () => { + it("names the status beside the failure it belongs to", () => { + expect(render([row()])).toContain("HTTP 401"); + }); + + it("says nothing where the failure never had a status to record", () => { + expect(render([row({ lastFailureStatus: null })])).not.toContain("HTTP"); + }); + + it("carries no status for a provider that is not failing", () => { + expect( + render([ + row({ failing: 0, lastFailureAt: null, lastFailureStatus: 500 }), + ]), + ).not.toContain("HTTP"); + }); +}); diff --git a/src/components/admin/provider-health-section.tsx b/src/components/admin/provider-health-section.tsx index 70e27328c..d4568c20f 100644 --- a/src/components/admin/provider-health-section.tsx +++ b/src/components/admin/provider-health-section.tsx @@ -18,6 +18,12 @@ import { formatDateTime } from "@/lib/format"; * read the database. The card shows one row per provider type: how many * users' chains touch it, how many of them are currently failing, the worst * uninterrupted failure run, and the last success / failure instants. + * + * The failure line also carries the HTTP status the ledger recorded for that + * failure, because "failing" on its own does not say whose problem it is. A + * 401 is a dead credential and waiting will not fix it; a 429 or a 503 is the + * provider's day and waiting is exactly right. Absent for a network-class + * failure, which never had a status to record. */ interface ProviderHealthRow { @@ -27,6 +33,7 @@ interface ProviderHealthRow { maxConsecutiveFailures: number; lastOkAt: string | null; lastFailureAt: string | null; + lastFailureStatus: number | null; } export function ProviderHealthSection() { @@ -106,6 +113,13 @@ export function ProviderHealthSection() { at: formatDateTime(p.lastFailureAt), })}` : ""} + {p.failing > 0 && + p.lastFailureAt && + p.lastFailureStatus !== null + ? ` · ${t("admin.providerHealth.lastFailureStatus", { + status: p.lastFailureStatus, + })}` + : ""} {p.failing === 0 ? (