Give the operator a provider-health readout

The per-user retry ledger has recorded every AI delivery outcome for a
long time, but no surface ever read it for the operator: a central
provider could fail on every call for weeks and the admin console said
nothing. The AI settings section now carries a health card, backed by a
new admin-only endpoint that folds the ledger into one row per provider
type: users tracked, users currently failing, the worst uninterrupted
failure run, and the last success and failure instants. Operator-managed
types sort first because their failure takes the shared fallback down
for everyone whose chain relies on it. Counts and timestamps only; the
response never names a user.

src/app/api/admin/provider-health/__tests__/route.test.ts

@@ -0,0 +1,147 @@
+/**
+ * v1.37.31 — contract for the operator provider-health readout
+ * (`/api/admin/provider-health`). The per-user retry ledger existed for a
+ * long time with no operator surface; this pins the fold: one row per
+ * provider type, failing counted from the LAST result only, the worst
+ * uninterrupted failure run taken across failing users, central types
+ * sorted first, and no per-user data in the response.
+ */
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("@/lib/db", () => ({
+  prisma: {
+    providerHealth: {
+      groupBy: vi.fn(),
+    },
+  },
+}));
+
+vi.mock("@/lib/api-handler", async () => {
+  const actual =
+    await vi.importActual<typeof import("@/lib/api-handler")>(
+      "@/lib/api-handler",
+    );
+  return {
+    ...actual,
+    apiHandler: <T extends (...args: unknown[]) => Promise<Response>>(
+      h: T,
+    ): T => h,
+    requireAdmin: vi.fn(),
+  };
+});
+
+vi.mock("@/lib/logging/context", () => ({
+  annotate: vi.fn(),
+  getEvent: vi.fn(() => null),
+}));
+
+import { GET } from "../route";
+import { prisma } from "@/lib/db";
+import { requireAdmin } from "@/lib/api-handler";
+
+const groupBy = vi.mocked(prisma.providerHealth.groupBy);
+
+function group(
+  providerType: string,
+  lastResult: string,
+  count: number,
+  max: {
+    consecutiveFailures?: number;
+    lastFailureAt?: Date | null;
+    lastOkAt?: Date | null;
+  } = {},
+) {
+  return {
+    providerType,
+    lastResult,
+    _count: { _all: count },
+    _max: {
+      consecutiveFailures: max.consecutiveFailures ?? 0,
+      lastFailureAt: max.lastFailureAt ?? null,
+      lastOkAt: max.lastOkAt ?? null,
+    },
+  };
+}
+
+describe("GET /api/admin/provider-health", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(requireAdmin).mockResolvedValue({
+      user: { id: "admin1" },
+    } as never);
+  });
+
+  it("is admin-gated before it touches the ledger", async () => {
+    vi.mocked(requireAdmin).mockRejectedValue(new Error("forbidden"));
+
+    await expect(GET()).rejects.toThrow("forbidden");
+    expect(groupBy).not.toHaveBeenCalled();
+  });
+
+  it("folds ok and failing groups of one type into a single row", async () => {
+    groupBy.mockResolvedValue([
+      group("admin-openai", "ok", 3, {
+        lastOkAt: new Date("2026-08-27T07:00:00Z"),
+      }),
+      group("admin-openai", "hard_failed", 2, {
+        consecutiveFailures: 3334,
+        lastFailureAt: new Date("2026-08-27T06:00:00Z"),
+      }),
+    ] as never);
+
+    const res = await GET();
+    const body = await res.json();
+    expect(body.data.providers).toEqual([
+      {
+        providerType: "admin-openai",
+        tracked: 5,
+        failing: 2,
+        maxConsecutiveFailures: 3334,
+        lastOkAt: "2026-08-27T07:00:00.000Z",
+        lastFailureAt: "2026-08-27T06:00:00.000Z",
+      },
+    ]);
+  });
+
+  it("counts auth_failed as failing and keeps an all-ok type at zero", async () => {
+    groupBy.mockResolvedValue([
+      group("codex", "auth_failed", 1, { consecutiveFailures: 7 }),
+      group("local", "ok", 4),
+    ] as never);
+
+    const res = await GET();
+    const body = await res.json();
+    const byType = Object.fromEntries(
+      body.data.providers.map(
+        (p: { providerType: string; failing: number }) => [
+          p.providerType,
+          p.failing,
+        ],
+      ),
+    );
+    expect(byType).toEqual({ codex: 1, local: 0 });
+  });
+
+  it("sorts the operator-managed types first, the rest alphabetically", async () => {
+    groupBy.mockResolvedValue([
+      group("local", "ok", 1),
+      group("anthropic", "ok", 1),
+      group("admin-codex", "ok", 1),
+      group("admin-openai", "ok", 1),
+    ] as never);
+
+    const res = await GET();
+    const body = await res.json();
+    expect(
+      body.data.providers.map((p: { providerType: string }) => p.providerType),
+    ).toEqual(["admin-openai", "admin-codex", "anthropic", "local"]);
+  });
+
+  it("answers an empty ledger with an empty list, not an error", async () => {
+    groupBy.mockResolvedValue([] as never);
+
+    const res = await GET();
+    const body = await res.json();
+    expect(body.data.providers).toEqual([]);
+  });
+});

src/app/api/admin/provider-health/route.ts

@@ -0,0 +1,89 @@
+import { apiHandler, requireAdmin } from "@/lib/api-handler";
+import { prisma } from "@/lib/db";
+import { apiSuccess } from "@/lib/api-response";
+import { annotate } from "@/lib/logging/context";
+
+export const dynamic = "force-dynamic";
+
+/**
+ * v1.37.31 — per-provider delivery health for the operator.
+ *
+ * The per-user retry ledger (`provider_health`) has recorded every AI
+ * delivery outcome for a long time, but no surface ever read it for the
+ * operator: a central provider (`admin-openai` / `admin-codex`) could fail
+ * on every call for weeks and the admin console said nothing. This endpoint
+ * folds the ledger into one row per provider type so the console can show
+ * it. Read-only, admin-cookie-only, and it exposes counts and timestamps
+ * only — never which user a row belongs to.
+ */
+
+interface ProviderHealthSummary {
+  providerType: string;
+  /** Users whose chain has recorded at least one outcome for this type. */
+  tracked: number;
+  /** Users whose LAST outcome for this type was a failure. */
+  failing: number;
+  /** Highest uninterrupted failure run across those users. */
+  maxConsecutiveFailures: number;
+  lastOkAt: string | null;
+  lastFailureAt: string | null;
+}
+
+/** The two operator-managed tags sort first — they affect every user on the chain. */
+const CENTRAL_TYPES = ["admin-openai", "admin-codex"];
+
+export const GET = apiHandler(async () => {
+  await requireAdmin();
+  annotate({ action: { name: "admin.provider-health.get" } });
+
+  const groups = await prisma.providerHealth.groupBy({
+    by: ["providerType", "lastResult"],
+    _count: { _all: true },
+    _max: {
+      consecutiveFailures: true,
+      lastFailureAt: true,
+      lastOkAt: true,
+    },
+  });
+
+  const byType = new Map<string, ProviderHealthSummary>();
+  for (const g of groups) {
+    const row = byType.get(g.providerType) ?? {
+      providerType: g.providerType,
+      tracked: 0,
+      failing: 0,
+      maxConsecutiveFailures: 0,
+      lastOkAt: null,
+      lastFailureAt: null,
+    };
+    row.tracked += g._count._all;
+    if (g.lastResult !== "ok") {
+      row.failing += g._count._all;
+      row.maxConsecutiveFailures = Math.max(
+        row.maxConsecutiveFailures,
+        g._max.consecutiveFailures ?? 0,
+      );
+    }
+    const ok = g._max.lastOkAt?.toISOString() ?? null;
+    if (ok && (!row.lastOkAt || ok > row.lastOkAt)) row.lastOkAt = ok;
+    const fail = g._max.lastFailureAt?.toISOString() ?? null;
+    if (fail && (!row.lastFailureAt || fail > row.lastFailureAt)) {
+      row.lastFailureAt = fail;
+    }
+    byType.set(g.providerType, row);
+  }
+
+  const providers = [...byType.values()].sort((a, b) => {
+    const ca = CENTRAL_TYPES.indexOf(a.providerType);
+    const cb = CENTRAL_TYPES.indexOf(b.providerType);
+    if (ca !== -1 || cb !== -1) {
+      return (
+        (ca === -1 ? CENTRAL_TYPES.length : ca) -
+        (cb === -1 ? CENTRAL_TYPES.length : cb)
+      );
+    }
+    return a.providerType.localeCompare(b.providerType);
+  });
+
+  return apiSuccess({ providers });
+});