Merge AI operations and error-log discipline into release/v1.42.0

src/app/api/admin/provider-health/__tests__/route.test.ts

@@ -17,13 +17,24 @@ vi.mock("@/lib/db", () => ({
   prisma: {
     providerHealth: {
       groupBy: vi.fn(),
+      // v1.42 — the paused rows.
+      findMany: vi.fn(async () => []),
     },
     coachUsage: {
       aggregate: vi.fn(),
     },
+    // v1.42 — the operator key's model check reads the admin settings.
+    appSettings: { findUnique: vi.fn(async () => null) },
   },
 }));
 
+// v1.42 — the model check probes the operator endpoint's listing.
+const probeModelListing = vi.fn();
+vi.mock("@/lib/ai/model-availability", () => ({
+  probeModelListing: (...a: unknown[]) => probeModelListing(...a),
+}));
+vi.mock("@/lib/crypto", () => ({ decrypt: (v: string) => v }));
+
 vi.mock("@/lib/api-handler", async () => {
   const actual =
     await vi.importActual<typeof import("@/lib/api-handler")>(
@@ -115,6 +126,9 @@ describe("GET /api/admin/provider-health", () => {
         lastOkAt: "2026-08-27T07:00:00.000Z",
         lastFailureAt: "2026-08-27T06:00:00.000Z",
         lastFailureStatus: 503,
+        pausedUsers: 0,
+        pausedUntil: null,
+        modelListing: null,
       },
     ]);
   });
@@ -257,3 +271,89 @@ describe("GET /api/admin/provider-health", () => {
     expect(body.data.providers).toEqual([]);
   });
 });
+
+describe("GET /api/admin/provider-health — pause and model check (v1.42)", () => {
+  const findMany = vi.mocked(prisma.providerHealth.findMany);
+  const settings = vi.mocked(prisma.appSettings.findUnique);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(requireAdmin).mockResolvedValue({
+      user: { id: "admin" },
+    } as never);
+    aggregate.mockResolvedValue({
+      _sum: { totalTokens: 0, operatorTokens: 0 },
+    } as never);
+    findMany.mockResolvedValue([] as never);
+    settings.mockResolvedValue(null as never);
+  });
+
+  it("counts the users a provider is paused for and when the last pause lifts", async () => {
+    groupBy.mockResolvedValue([
+      group("admin-openai", "hard_failed", 2, {
+        consecutiveFailures: 1766,
+        lastFailureAt: new Date("2026-10-08T04:31:00Z"),
+        lastStatus: 500,
+      }),
+    ] as never);
+    findMany.mockResolvedValue([
+      {
+        providerType: "admin-openai",
+        nextRetryAt: new Date("2026-10-08T05:01:00Z"),
+      },
+      {
+        providerType: "admin-openai",
+        nextRetryAt: new Date("2026-10-09T04:31:00Z"),
+      },
+    ] as never);
+
+    const body = await (await GET()).json();
+    expect(body.data.providers[0]).toMatchObject({
+      pausedUsers: 2,
+      pausedUntil: "2026-10-09T04:31:00.000Z",
+    });
+    // Only rows past the pause threshold, inside their window, are read.
+    expect(findMany.mock.calls[0][0]).toMatchObject({
+      where: {
+        lastResult: "hard_failed",
+        consecutiveFailures: { gte: 5 },
+      },
+    });
+  });
+
+  it("names a model the operator's endpoint no longer lists", async () => {
+    groupBy.mockResolvedValue([
+      group("admin-openai", "hard_failed", 1, {
+        consecutiveFailures: 9,
+        lastFailureAt: new Date("2026-10-08T04:31:00Z"),
+        lastStatus: 500,
+      }),
+    ] as never);
+    settings.mockResolvedValue({
+      adminAiModel: "gpt-5.4-mini",
+      adminAiBaseUrl: "http://openai-oauth-proxy:10531/v1",
+      adminAiKeyEncrypted: "key",
+    } as never);
+    probeModelListing.mockResolvedValue("not_listed");
+
+    const body = await (await GET()).json();
+    expect(body.data.providers[0].modelListing).toEqual({
+      model: "gpt-5.4-mini",
+      listing: "not_listed",
+    });
+    expect(probeModelListing).toHaveBeenCalledWith(
+      expect.objectContaining({ operatorTrusted: true, model: "gpt-5.4-mini" }),
+    );
+  });
+
+  it("does not probe while the operator's key is healthy", async () => {
+    groupBy.mockResolvedValue([
+      group("admin-openai", "ok", 4, {
+        lastOkAt: new Date("2026-10-08T04:31:00Z"),
+      }),
+    ] as never);
+    const body = await (await GET()).json();
+    expect(body.data.providers[0].modelListing).toBeNull();
+    expect(probeModelListing).not.toHaveBeenCalled();
+  });
+});

src/app/api/admin/provider-health/route.ts

@@ -4,6 +4,14 @@ import { prisma } from "@/lib/db";
 import { apiSuccess } from "@/lib/api-response";
 import { annotate } from "@/lib/logging/context";
 import { OPERATOR_HELD_PROVIDER_TYPES } from "@/lib/ai/provider-egress";
+import { PROVIDER_PAUSE_THRESHOLD } from "@/lib/ai/provider-health-ledger";
+import {
+  probeModelListing,
+  type ModelListing,
+} from "@/lib/ai/model-availability";
+import { decrypt } from "@/lib/crypto";
+import { isAnthropicBaseUrl } from "@/lib/ai/provider";
+import { isCanonicalOpenAIEndpoint } from "@/lib/ai/openai-capabilities";
 
 export const dynamic = "force-dynamic";
 
@@ -42,6 +50,20 @@ interface ProviderHealthSummary {
    * success clears the status along with the rest of the failure state.
    */
   lastFailureStatus: number | null;
+  /**
+   * v1.42 — users whose chain has this provider PAUSED right now (a run of
+   * hard failures past the pause threshold, inside its window). The chain
+   * walks a paused provider last and tries it again once the window lifts.
+   */
+  pausedUsers: number;
+  /** When the latest of those pauses lifts. Null when none is paused. */
+  pausedUntil: string | null;
+  /**
+   * v1.42 — for the operator's own key while it is failing: whether its
+   * endpoint still lists the configured model. `not_listed` is the fix the
+   * card names; null when there is nothing to check.
+   */
+  modelListing: { model: string; listing: ModelListing } | null;
 }
 
 /**
@@ -71,6 +93,19 @@ export const GET = apiHandler(async () => {
     },
   });
 
+  // v1.42 — paused providers: a hard-failure run at or past the threshold,
+  // inside its window. Read as rows (one per user and type, two columns) so
+  // the count and the latest lift time are exact, not folded from a group.
+  const now = new Date();
+  const pausedRows = await prisma.providerHealth.findMany({
+    where: {
+      lastResult: "hard_failed",
+      consecutiveFailures: { gte: PROVIDER_PAUSE_THRESHOLD },
+      nextRetryAt: { gt: now },
+    },
+    select: { providerType: true, nextRetryAt: true },
+  });
+
   const byType = new Map<string, ProviderHealthSummary>();
   for (const g of groups) {
     const row = byType.get(g.providerType) ?? {
@@ -81,6 +116,9 @@ export const GET = apiHandler(async () => {
       lastOkAt: null,
       lastFailureAt: null,
       lastFailureStatus: null,
+      pausedUsers: 0,
+      pausedUntil: null,
+      modelListing: null,
     };
     row.tracked += g._count._all;
     if (g.lastResult !== "ok") {
@@ -103,6 +141,21 @@ export const GET = apiHandler(async () => {
     byType.set(g.providerType, row);
   }
 
+  for (const paused of pausedRows) {
+    const row = byType.get(paused.providerType);
+    if (!row || paused.nextRetryAt === null) continue;
+    row.pausedUsers += 1;
+    const until = paused.nextRetryAt.toISOString();
+    if (!row.pausedUntil || until > row.pausedUntil) row.pausedUntil = until;
+  }
+
+  // The operator's own key, while it is failing: does its endpoint still list
+  // the configured model? A free GET, cached; only on this admin read.
+  const adminRow = byType.get("admin-openai");
+  if (adminRow && adminRow.failing > 0) {
+    adminRow.modelListing = await adminModelListing();
+  }
+
   const providers = [...byType.values()].sort((a, b) => {
     const ca = CENTRAL_TYPES.indexOf(a.providerType);
     const cb = CENTRAL_TYPES.indexOf(b.providerType);
@@ -136,3 +189,42 @@ export const GET = apiHandler(async () => {
     },
   });
 });
+
+/**
+ * The model check for the operator's key. Never fails the readout: a missing
+ * settings row, a key that does not decrypt, or a probe error is simply no
+ * answer.
+ */
+async function adminModelListing(): Promise<{
+  model: string;
+  listing: ModelListing;
+} | null> {
+  try {
+    const settings = await prisma.appSettings.findUnique({
+      where: { id: "singleton" },
+      select: {
+        adminAiModel: true,
+        adminAiBaseUrl: true,
+        adminAiKeyEncrypted: true,
+      },
+    });
+    if (!settings?.adminAiKeyEncrypted) return null;
+    const baseUrl = settings.adminAiBaseUrl ?? "https://api.openai.com/v1";
+    // The same scope as the client's own check: an OpenAI-compatible
+    // endpoint the operator typed. `api.openai.com` names a missing model in
+    // its own error; Anthropic lists models on a different contract.
+    if (isCanonicalOpenAIEndpoint(baseUrl) || isAnthropicBaseUrl(baseUrl)) {
+      return null;
+    }
+    const model = settings.adminAiModel ?? "gpt-4o";
+    const listing = await probeModelListing({
+      baseUrl,
+      apiKey: decrypt(settings.adminAiKeyEncrypted),
+      model,
+      operatorTrusted: true,
+    });
+    return { model, listing };
+  } catch {
+    return null;
+  }
+}