Give the operator a provider-health readout

The per-user retry ledger has recorded every AI delivery outcome for a
long time, but no surface ever read it for the operator: a central
provider could fail on every call for weeks and the admin console said
nothing. The AI settings section now carries a health card, backed by a
new admin-only endpoint that folds the ledger into one row per provider
type: users tracked, users currently failing, the worst uninterrupted
failure run, and the last success and failure instants. Operator-managed
types sort first because their failure takes the shared fallback down
for everyone whose chain relies on it. Counts and timestamps only; the
response never names a user.

src/__tests__/openapi-route-coverage-guard.test.ts

@@ -258,6 +258,7 @@ const UNPUBLISHED: Readonly<Record<string, Exemption>> = {
     kind: "adminConsole",
     methods: ["POST"],
   },
+  "/api/admin/provider-health": { kind: "adminConsole", methods: ["GET"] },
   "/api/admin/notifications/health": { kind: "adminConsole", methods: ["GET"] },
   "/api/admin/notifications/reminder-check": {
     kind: "adminConsole",

src/app/admin/[section]/renderer.tsx

@@ -18,6 +18,7 @@ import { useAuth } from "@/hooks/use-auth";
 import { AboutSection } from "@/components/settings/about-section";
 import { AiQualitySection } from "@/components/admin/ai-quality-section";
 import { AiServerKeySection } from "@/components/admin/ai-server-key-section";
+import { ProviderHealthSection } from "@/components/admin/provider-health-section";
 import { CentralCodexSection } from "@/components/admin/central-codex-section";
 import { AssistantSection } from "@/components/admin/assistant-section";
 import { CoachFeedbackSection } from "@/components/admin/coach-feedback-section";
@@ -89,6 +90,7 @@ export function AdminSectionRenderer({
           <AssistantSection />
           <AiServerKeySection />
           <CentralCodexSection />
+          <ProviderHealthSection />
           <CoachFeedbackSection />
           <AiQualitySection />
         </SectionFrame>

src/app/api/admin/provider-health/__tests__/route.test.ts

@@ -0,0 +1,147 @@
+/**
+ * v1.37.31 — contract for the operator provider-health readout
+ * (`/api/admin/provider-health`). The per-user retry ledger existed for a
+ * long time with no operator surface; this pins the fold: one row per
+ * provider type, failing counted from the LAST result only, the worst
+ * uninterrupted failure run taken across failing users, central types
+ * sorted first, and no per-user data in the response.
+ */
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("@/lib/db", () => ({
+  prisma: {
+    providerHealth: {
+      groupBy: vi.fn(),
+    },
+  },
+}));
+
+vi.mock("@/lib/api-handler", async () => {
+  const actual =
+    await vi.importActual<typeof import("@/lib/api-handler")>(
+      "@/lib/api-handler",
+    );
+  return {
+    ...actual,
+    apiHandler: <T extends (...args: unknown[]) => Promise<Response>>(
+      h: T,
+    ): T => h,
+    requireAdmin: vi.fn(),
+  };
+});
+
+vi.mock("@/lib/logging/context", () => ({
+  annotate: vi.fn(),
+  getEvent: vi.fn(() => null),
+}));
+
+import { GET } from "../route";
+import { prisma } from "@/lib/db";
+import { requireAdmin } from "@/lib/api-handler";
+
+const groupBy = vi.mocked(prisma.providerHealth.groupBy);
+
+function group(
+  providerType: string,
+  lastResult: string,
+  count: number,
+  max: {
+    consecutiveFailures?: number;
+    lastFailureAt?: Date | null;
+    lastOkAt?: Date | null;
+  } = {},
+) {
+  return {
+    providerType,
+    lastResult,
+    _count: { _all: count },
+    _max: {
+      consecutiveFailures: max.consecutiveFailures ?? 0,
+      lastFailureAt: max.lastFailureAt ?? null,
+      lastOkAt: max.lastOkAt ?? null,
+    },
+  };
+}
+
+describe("GET /api/admin/provider-health", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(requireAdmin).mockResolvedValue({
+      user: { id: "admin1" },
+    } as never);
+  });
+
+  it("is admin-gated before it touches the ledger", async () => {
+    vi.mocked(requireAdmin).mockRejectedValue(new Error("forbidden"));
+
+    await expect(GET()).rejects.toThrow("forbidden");
+    expect(groupBy).not.toHaveBeenCalled();
+  });
+
+  it("folds ok and failing groups of one type into a single row", async () => {
+    groupBy.mockResolvedValue([
+      group("admin-openai", "ok", 3, {
+        lastOkAt: new Date("2026-08-27T07:00:00Z"),
+      }),
+      group("admin-openai", "hard_failed", 2, {
+        consecutiveFailures: 3334,
+        lastFailureAt: new Date("2026-08-27T06:00:00Z"),
+      }),
+    ] as never);
+
+    const res = await GET();
+    const body = await res.json();
+    expect(body.data.providers).toEqual([
+      {
+        providerType: "admin-openai",
+        tracked: 5,
+        failing: 2,
+        maxConsecutiveFailures: 3334,
+        lastOkAt: "2026-08-27T07:00:00.000Z",
+        lastFailureAt: "2026-08-27T06:00:00.000Z",
+      },
+    ]);
+  });
+
+  it("counts auth_failed as failing and keeps an all-ok type at zero", async () => {
+    groupBy.mockResolvedValue([
+      group("codex", "auth_failed", 1, { consecutiveFailures: 7 }),
+      group("local", "ok", 4),
+    ] as never);
+
+    const res = await GET();
+    const body = await res.json();
+    const byType = Object.fromEntries(
+      body.data.providers.map(
+        (p: { providerType: string; failing: number }) => [
+          p.providerType,
+          p.failing,
+        ],
+      ),
+    );
+    expect(byType).toEqual({ codex: 1, local: 0 });
+  });
+
+  it("sorts the operator-managed types first, the rest alphabetically", async () => {
+    groupBy.mockResolvedValue([
+      group("local", "ok", 1),
+      group("anthropic", "ok", 1),
+      group("admin-codex", "ok", 1),
+      group("admin-openai", "ok", 1),
+    ] as never);
+
+    const res = await GET();
+    const body = await res.json();
+    expect(
+      body.data.providers.map((p: { providerType: string }) => p.providerType),
+    ).toEqual(["admin-openai", "admin-codex", "anthropic", "local"]);
+  });
+
+  it("answers an empty ledger with an empty list, not an error", async () => {
+    groupBy.mockResolvedValue([] as never);
+
+    const res = await GET();
+    const body = await res.json();
+    expect(body.data.providers).toEqual([]);
+  });
+});

src/app/api/admin/provider-health/route.ts

@@ -0,0 +1,89 @@
+import { apiHandler, requireAdmin } from "@/lib/api-handler";
+import { prisma } from "@/lib/db";
+import { apiSuccess } from "@/lib/api-response";
+import { annotate } from "@/lib/logging/context";
+
+export const dynamic = "force-dynamic";
+
+/**
+ * v1.37.31 — per-provider delivery health for the operator.
+ *
+ * The per-user retry ledger (`provider_health`) has recorded every AI
+ * delivery outcome for a long time, but no surface ever read it for the
+ * operator: a central provider (`admin-openai` / `admin-codex`) could fail
+ * on every call for weeks and the admin console said nothing. This endpoint
+ * folds the ledger into one row per provider type so the console can show
+ * it. Read-only, admin-cookie-only, and it exposes counts and timestamps
+ * only — never which user a row belongs to.
+ */
+
+interface ProviderHealthSummary {
+  providerType: string;
+  /** Users whose chain has recorded at least one outcome for this type. */
+  tracked: number;
+  /** Users whose LAST outcome for this type was a failure. */
+  failing: number;
+  /** Highest uninterrupted failure run across those users. */
+  maxConsecutiveFailures: number;
+  lastOkAt: string | null;
+  lastFailureAt: string | null;
+}
+
+/** The two operator-managed tags sort first — they affect every user on the chain. */
+const CENTRAL_TYPES = ["admin-openai", "admin-codex"];
+
+export const GET = apiHandler(async () => {
+  await requireAdmin();
+  annotate({ action: { name: "admin.provider-health.get" } });
+
+  const groups = await prisma.providerHealth.groupBy({
+    by: ["providerType", "lastResult"],
+    _count: { _all: true },
+    _max: {
+      consecutiveFailures: true,
+      lastFailureAt: true,
+      lastOkAt: true,
+    },
+  });
+
+  const byType = new Map<string, ProviderHealthSummary>();
+  for (const g of groups) {
+    const row = byType.get(g.providerType) ?? {
+      providerType: g.providerType,
+      tracked: 0,
+      failing: 0,
+      maxConsecutiveFailures: 0,
+      lastOkAt: null,
+      lastFailureAt: null,
+    };
+    row.tracked += g._count._all;
+    if (g.lastResult !== "ok") {
+      row.failing += g._count._all;
+      row.maxConsecutiveFailures = Math.max(
+        row.maxConsecutiveFailures,
+        g._max.consecutiveFailures ?? 0,
+      );
+    }
+    const ok = g._max.lastOkAt?.toISOString() ?? null;
+    if (ok && (!row.lastOkAt || ok > row.lastOkAt)) row.lastOkAt = ok;
+    const fail = g._max.lastFailureAt?.toISOString() ?? null;
+    if (fail && (!row.lastFailureAt || fail > row.lastFailureAt)) {
+      row.lastFailureAt = fail;
+    }
+    byType.set(g.providerType, row);
+  }
+
+  const providers = [...byType.values()].sort((a, b) => {
+    const ca = CENTRAL_TYPES.indexOf(a.providerType);
+    const cb = CENTRAL_TYPES.indexOf(b.providerType);
+    if (ca !== -1 || cb !== -1) {
+      return (
+        (ca === -1 ? CENTRAL_TYPES.length : ca) -
+        (cb === -1 ? CENTRAL_TYPES.length : cb)
+      );
+    }
+    return a.providerType.localeCompare(b.providerType);
+  });
+
+  return apiSuccess({ providers });
+});

src/components/admin/provider-health-section.tsx

@@ -0,0 +1,130 @@
+"use client";
+
+import { useQuery } from "@tanstack/react-query";
+import { Activity } from "lucide-react";
+
+import { Badge } from "@/components/ui/badge";
+import { SettingsCard } from "@/components/settings/settings-card";
+import { SettingsCardHeader } from "@/components/settings/_card-header";
+import { useTranslations } from "@/lib/i18n/context";
+import { queryKeys } from "@/lib/query-keys";
+import { apiGet } from "@/lib/api/api-fetch";
+import { formatDateTime } from "@/lib/format";
+
+/**
+ * v1.37.31 — the delivery-health readout the `provider_health` ledger never
+ * had. The ledger records every AI call outcome per user and provider tag;
+ * before this card, a failing central provider was invisible until someone
+ * read the database. The card shows one row per provider type: how many
+ * users' chains touch it, how many of them are currently failing, the worst
+ * uninterrupted failure run, and the last success / failure instants.
+ */
+
+interface ProviderHealthRow {
+  providerType: string;
+  tracked: number;
+  failing: number;
+  maxConsecutiveFailures: number;
+  lastOkAt: string | null;
+  lastFailureAt: string | null;
+}
+
+export function ProviderHealthSection() {
+  const { t } = useTranslations();
+
+  const { data, isPending, isError } = useQuery({
+    queryKey: queryKeys.adminProviderHealth(),
+    queryFn: () =>
+      apiGet<{ providers: ProviderHealthRow[] }>("/api/admin/provider-health"),
+  });
+
+  const typeLabel = (type: string): string => {
+    switch (type) {
+      case "admin-openai":
+        return t("admin.providerHealth.typeAdminOpenai");
+      case "admin-codex":
+        return t("admin.providerHealth.typeAdminCodex");
+      case "openai":
+        return t("admin.providerHealth.typeOpenai");
+      case "anthropic":
+        return t("admin.providerHealth.typeAnthropic");
+      case "codex":
+        return t("admin.providerHealth.typeCodex");
+      case "local":
+        return t("admin.providerHealth.typeLocal");
+      case "openai-compatible":
+        return t("admin.providerHealth.typeGateway");
+      default:
+        return type;
+    }
+  };
+
+  const providers = data?.providers ?? [];
+
+  return (
+    <SettingsCard>
+      <SettingsCardHeader
+        icon={Activity}
+        title={t("admin.providerHealth.title")}
+        description={t("admin.providerHealth.description")}
+      />
+      {isPending ? (
+        <p className="text-muted-foreground text-sm">
+          {t("admin.providerHealth.loading")}
+        </p>
+      ) : isError ? (
+        <p className="text-muted-foreground text-sm">
+          {t("admin.providerHealth.loadError")}
+        </p>
+      ) : providers.length === 0 ? (
+        <p className="text-muted-foreground text-sm">
+          {t("admin.providerHealth.empty")}
+        </p>
+      ) : (
+        <ul className="flex flex-col gap-3" data-testid="provider-health-list">
+          {providers.map((p) => (
+            <li
+              key={p.providerType}
+              className="flex flex-wrap items-center justify-between gap-2 rounded-md border p-3"
+              data-provider-type={p.providerType}
+            >
+              <div className="flex min-w-0 flex-col gap-1">
+                <span className="text-sm font-medium">
+                  {typeLabel(p.providerType)}
+                </span>
+                <span className="text-muted-foreground text-xs">
+                  {t("admin.providerHealth.trackedUsers", {
+                    count: p.tracked,
+                  })}
+                  {p.lastOkAt
+                    ? ` · ${t("admin.providerHealth.lastOk", {
+                        at: formatDateTime(p.lastOkAt),
+                      })}`
+                    : ""}
+                  {p.failing > 0 && p.lastFailureAt
+                    ? ` · ${t("admin.providerHealth.lastFailure", {
+                        at: formatDateTime(p.lastFailureAt),
+                      })}`
+                    : ""}
+                </span>
+              </div>
+              {p.failing === 0 ? (
+                <Badge variant="secondary">
+                  {t("admin.providerHealth.statusHealthy")}
+                </Badge>
+              ) : (
+                <Badge variant="destructive">
+                  {t("admin.providerHealth.statusFailing", {
+                    failing: p.failing,
+                    tracked: p.tracked,
+                    run: p.maxConsecutiveFailures,
+                  })}
+                </Badge>
+              )}
+            </li>
+          ))}
+        </ul>
+      )}
+    </SettingsCard>
+  );
+}

src/lib/query-keys/admin.ts

@@ -16,6 +16,11 @@ export const adminKeys = {
    * `insightsSettings.hasAdminKey` flag mirrors the configured state.
    */
   adminAiServerKey: () => ["admin", "ai-server-key"] as const,
+  /**
+   * v1.37.31 — per-provider delivery health folded from the per-user
+   * retry ledger. Read via `/api/admin/provider-health`.
+   */
+  adminProviderHealth: () => ["admin", "provider-health"] as const,
   /**
    * Operator-shared central Codex (ChatGPT subscription) connection status.
    * Read/write via `/api/admin/central-codex`; the user-facing