Merge AI operations and error-log discipline into release/v1.42.0

src/app/api/admin/provider-health/__tests__/route.test.ts

@@ -17,13 +17,24 @@ vi.mock("@/lib/db", () => ({
   prisma: {
     providerHealth: {
       groupBy: vi.fn(),
+      // v1.42 — the paused rows.
+      findMany: vi.fn(async () => []),
     },
     coachUsage: {
       aggregate: vi.fn(),
     },
+    // v1.42 — the operator key's model check reads the admin settings.
+    appSettings: { findUnique: vi.fn(async () => null) },
   },
 }));
 
+// v1.42 — the model check probes the operator endpoint's listing.
+const probeModelListing = vi.fn();
+vi.mock("@/lib/ai/model-availability", () => ({
+  probeModelListing: (...a: unknown[]) => probeModelListing(...a),
+}));
+vi.mock("@/lib/crypto", () => ({ decrypt: (v: string) => v }));
+
 vi.mock("@/lib/api-handler", async () => {
   const actual =
     await vi.importActual<typeof import("@/lib/api-handler")>(
@@ -115,6 +126,9 @@ describe("GET /api/admin/provider-health", () => {
         lastOkAt: "2026-08-27T07:00:00.000Z",
         lastFailureAt: "2026-08-27T06:00:00.000Z",
         lastFailureStatus: 503,
+        pausedUsers: 0,
+        pausedUntil: null,
+        modelListing: null,
       },
     ]);
   });
@@ -257,3 +271,89 @@ describe("GET /api/admin/provider-health", () => {
     expect(body.data.providers).toEqual([]);
   });
 });
+
+describe("GET /api/admin/provider-health — pause and model check (v1.42)", () => {
+  const findMany = vi.mocked(prisma.providerHealth.findMany);
+  const settings = vi.mocked(prisma.appSettings.findUnique);
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(requireAdmin).mockResolvedValue({
+      user: { id: "admin" },
+    } as never);
+    aggregate.mockResolvedValue({
+      _sum: { totalTokens: 0, operatorTokens: 0 },
+    } as never);
+    findMany.mockResolvedValue([] as never);
+    settings.mockResolvedValue(null as never);
+  });
+
+  it("counts the users a provider is paused for and when the last pause lifts", async () => {
+    groupBy.mockResolvedValue([
+      group("admin-openai", "hard_failed", 2, {
+        consecutiveFailures: 1766,
+        lastFailureAt: new Date("2026-10-08T04:31:00Z"),
+        lastStatus: 500,
+      }),
+    ] as never);
+    findMany.mockResolvedValue([
+      {
+        providerType: "admin-openai",
+        nextRetryAt: new Date("2026-10-08T05:01:00Z"),
+      },
+      {
+        providerType: "admin-openai",
+        nextRetryAt: new Date("2026-10-09T04:31:00Z"),
+      },
+    ] as never);
+
+    const body = await (await GET()).json();
+    expect(body.data.providers[0]).toMatchObject({
+      pausedUsers: 2,
+      pausedUntil: "2026-10-09T04:31:00.000Z",
+    });
+    // Only rows past the pause threshold, inside their window, are read.
+    expect(findMany.mock.calls[0][0]).toMatchObject({
+      where: {
+        lastResult: "hard_failed",
+        consecutiveFailures: { gte: 5 },
+      },
+    });
+  });
+
+  it("names a model the operator's endpoint no longer lists", async () => {
+    groupBy.mockResolvedValue([
+      group("admin-openai", "hard_failed", 1, {
+        consecutiveFailures: 9,
+        lastFailureAt: new Date("2026-10-08T04:31:00Z"),
+        lastStatus: 500,
+      }),
+    ] as never);
+    settings.mockResolvedValue({
+      adminAiModel: "gpt-5.4-mini",
+      adminAiBaseUrl: "http://openai-oauth-proxy:10531/v1",
+      adminAiKeyEncrypted: "key",
+    } as never);
+    probeModelListing.mockResolvedValue("not_listed");
+
+    const body = await (await GET()).json();
+    expect(body.data.providers[0].modelListing).toEqual({
+      model: "gpt-5.4-mini",
+      listing: "not_listed",
+    });
+    expect(probeModelListing).toHaveBeenCalledWith(
+      expect.objectContaining({ operatorTrusted: true, model: "gpt-5.4-mini" }),
+    );
+  });
+
+  it("does not probe while the operator's key is healthy", async () => {
+    groupBy.mockResolvedValue([
+      group("admin-openai", "ok", 4, {
+        lastOkAt: new Date("2026-10-08T04:31:00Z"),
+      }),
+    ] as never);
+    const body = await (await GET()).json();
+    expect(body.data.providers[0].modelListing).toBeNull();
+    expect(probeModelListing).not.toHaveBeenCalled();
+  });
+});

src/app/api/admin/provider-health/route.ts

@@ -4,6 +4,14 @@ import { prisma } from "@/lib/db";
 import { apiSuccess } from "@/lib/api-response";
 import { annotate } from "@/lib/logging/context";
 import { OPERATOR_HELD_PROVIDER_TYPES } from "@/lib/ai/provider-egress";
+import { PROVIDER_PAUSE_THRESHOLD } from "@/lib/ai/provider-health-ledger";
+import {
+  probeModelListing,
+  type ModelListing,
+} from "@/lib/ai/model-availability";
+import { decrypt } from "@/lib/crypto";
+import { isAnthropicBaseUrl } from "@/lib/ai/provider";
+import { isCanonicalOpenAIEndpoint } from "@/lib/ai/openai-capabilities";
 
 export const dynamic = "force-dynamic";
 
@@ -42,6 +50,20 @@ interface ProviderHealthSummary {
    * success clears the status along with the rest of the failure state.
    */
   lastFailureStatus: number | null;
+  /**
+   * v1.42 — users whose chain has this provider PAUSED right now (a run of
+   * hard failures past the pause threshold, inside its window). The chain
+   * walks a paused provider last and tries it again once the window lifts.
+   */
+  pausedUsers: number;
+  /** When the latest of those pauses lifts. Null when none is paused. */
+  pausedUntil: string | null;
+  /**
+   * v1.42 — for the operator's own key while it is failing: whether its
+   * endpoint still lists the configured model. `not_listed` is the fix the
+   * card names; null when there is nothing to check.
+   */
+  modelListing: { model: string; listing: ModelListing } | null;
 }
 
 /**
@@ -71,6 +93,19 @@ export const GET = apiHandler(async () => {
     },
   });
 
+  // v1.42 — paused providers: a hard-failure run at or past the threshold,
+  // inside its window. Read as rows (one per user and type, two columns) so
+  // the count and the latest lift time are exact, not folded from a group.
+  const now = new Date();
+  const pausedRows = await prisma.providerHealth.findMany({
+    where: {
+      lastResult: "hard_failed",
+      consecutiveFailures: { gte: PROVIDER_PAUSE_THRESHOLD },
+      nextRetryAt: { gt: now },
+    },
+    select: { providerType: true, nextRetryAt: true },
+  });
+
   const byType = new Map<string, ProviderHealthSummary>();
   for (const g of groups) {
     const row = byType.get(g.providerType) ?? {
@@ -81,6 +116,9 @@ export const GET = apiHandler(async () => {
       lastOkAt: null,
       lastFailureAt: null,
       lastFailureStatus: null,
+      pausedUsers: 0,
+      pausedUntil: null,
+      modelListing: null,
     };
     row.tracked += g._count._all;
     if (g.lastResult !== "ok") {
@@ -103,6 +141,21 @@ export const GET = apiHandler(async () => {
     byType.set(g.providerType, row);
   }
 
+  for (const paused of pausedRows) {
+    const row = byType.get(paused.providerType);
+    if (!row || paused.nextRetryAt === null) continue;
+    row.pausedUsers += 1;
+    const until = paused.nextRetryAt.toISOString();
+    if (!row.pausedUntil || until > row.pausedUntil) row.pausedUntil = until;
+  }
+
+  // The operator's own key, while it is failing: does its endpoint still list
+  // the configured model? A free GET, cached; only on this admin read.
+  const adminRow = byType.get("admin-openai");
+  if (adminRow && adminRow.failing > 0) {
+    adminRow.modelListing = await adminModelListing();
+  }
+
   const providers = [...byType.values()].sort((a, b) => {
     const ca = CENTRAL_TYPES.indexOf(a.providerType);
     const cb = CENTRAL_TYPES.indexOf(b.providerType);
@@ -136,3 +189,42 @@ export const GET = apiHandler(async () => {
     },
   });
 });
+
+/**
+ * The model check for the operator's key. Never fails the readout: a missing
+ * settings row, a key that does not decrypt, or a probe error is simply no
+ * answer.
+ */
+async function adminModelListing(): Promise<{
+  model: string;
+  listing: ModelListing;
+} | null> {
+  try {
+    const settings = await prisma.appSettings.findUnique({
+      where: { id: "singleton" },
+      select: {
+        adminAiModel: true,
+        adminAiBaseUrl: true,
+        adminAiKeyEncrypted: true,
+      },
+    });
+    if (!settings?.adminAiKeyEncrypted) return null;
+    const baseUrl = settings.adminAiBaseUrl ?? "https://api.openai.com/v1";
+    // The same scope as the client's own check: an OpenAI-compatible
+    // endpoint the operator typed. `api.openai.com` names a missing model in
+    // its own error; Anthropic lists models on a different contract.
+    if (isCanonicalOpenAIEndpoint(baseUrl) || isAnthropicBaseUrl(baseUrl)) {
+      return null;
+    }
+    const model = settings.adminAiModel ?? "gpt-4o";
+    const listing = await probeModelListing({
+      baseUrl,
+      apiKey: decrypt(settings.adminAiKeyEncrypted),
+      model,
+      operatorTrusted: true,
+    });
+    return { model, listing };
+  } catch {
+    return null;
+  }
+}

src/app/api/insights/derived/__tests__/route.test.ts

@@ -90,6 +90,7 @@ import { getSession } from "@/lib/auth/session";
 import { prisma } from "@/lib/db";
 import { computeDerivedMetric } from "@/lib/insights/derived";
 import { DERIVED_MAX_WINDOW_DAYS } from "@/lib/insights/derived/types";
+import { __resetAllCachesForTests } from "@/lib/cache/server-cache";
 
 const SESSION_OK = {
   session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) },
@@ -119,6 +120,8 @@ function makeReqWith(params: Record<string, string>): NextRequest {
 
 beforeEach(() => {
   vi.clearAllMocks();
+  // v1.42 — the route reads through the derived cache now.
+  __resetAllCachesForTests();
   vi.mocked(prisma.appSettings.findUnique).mockResolvedValue(null as never);
   vi.mocked(prisma.user.findUnique).mockResolvedValue({
     dateOfBirth: new Date("1986-01-01"),
@@ -270,3 +273,34 @@ describe("GET /api/insights/derived — windowDays", () => {
     }
   });
 });
+
+// v1.42 — the single-score read was the one derived surface with no cache:
+// every open of a score's detail recomputed it over the rollup tier.
+describe("GET /api/insights/derived — read-through cache (v1.42)", () => {
+  it("serves a repeat read from the cache, per metric, type and window", async () => {
+    vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
+    await callGet(makeReq("VITALS_BASELINE", "RESTING_HEART_RATE"));
+    await callGet(makeReq("VITALS_BASELINE", "RESTING_HEART_RATE"));
+    expect(computeDerivedMetric).toHaveBeenCalledTimes(1);
+
+    await callGet(makeReq("VITALS_BASELINE", "WEIGHT"));
+    await callGet(
+      makeReqWith({
+        metric: "VITALS_BASELINE",
+        type: "RESTING_HEART_RATE",
+        windowDays: "7",
+      }),
+    );
+    expect(computeDerivedMetric).toHaveBeenCalledTimes(3);
+  });
+
+  it("recomputes after the account's measurements change", async () => {
+    vi.mocked(getSession).mockResolvedValue(SESSION_OK as never);
+    const { invalidateUserMeasurements } =
+      await import("@/lib/cache/invalidate");
+    await callGet(makeReq("VITALS_BASELINE", "RESTING_HEART_RATE"));
+    invalidateUserMeasurements("user-1");
+    await callGet(makeReq("VITALS_BASELINE", "RESTING_HEART_RATE"));
+    expect(computeDerivedMetric).toHaveBeenCalledTimes(2);
+  });
+});

src/app/api/insights/derived/route.ts

@@ -12,7 +12,8 @@
  * The `data` payload is the flat `Derived<T>` union so iOS can decode +
  * combine values across metrics: `{ metric, status, value?, coverage,
  * confidence?, provenance, reason? }`. Numbers are pure compute over the
- * rollup tier (Tier-1, no narrative) — no provider call, no cache table.
+ * rollup tier (Tier-1, no narrative) — no provider call, no cache table;
+ * since v1.42 the in-process derived cache, as the batch route uses.
  *
  * Wave 1 implements `VITALS_BASELINE` end-to-end; other registered ids
  * return a `not_implemented` insufficient until their compute lands in a
@@ -36,6 +37,7 @@ import { DERIVED_MAX_WINDOW_DAYS } from "@/lib/insights/derived/types";
 import { resolveDerivedAssessment } from "@/lib/insights/derived/derived-assessment-ai";
 import { resolveServerLocale } from "@/lib/i18n/server-locale";
 import { aiCapabilityToServe } from "@/lib/ai/capabilities/gate";
+import { cachedSwr, caches, type ServerCache } from "@/lib/cache/server-cache";
 
 export const dynamic = "force-dynamic";
 
@@ -122,14 +124,31 @@ export const GET = apiHandler(async (request: NextRequest) => {
     if (!gate.enabled) return gate.response;
   }
 
-  const profile = await loadBaselineProfile(prisma, user.id);
-  const derived = await computeDerivedMetric({
-    metric,
-    userId: user.id,
-    profile,
-    type: parsed.data.type ?? null,
-    windowDays: parsed.data.windowDays,
-  });
+  // v1.42 — read-through the per-user derived cache with
+  // stale-while-revalidate, like the batch grid beside it. This single-score
+  // read was the one derived surface that recomputed on every request: the
+  // score detail sheet asks for it on each open, and a composite walks the
+  // rollup tier once per input. Same bucket, so the invalidation the batch
+  // relies on covers it unchanged (measurement writes evict or mark the
+  // `${userId}|` prefix stale, mood writes mark it stale). The module gate
+  // above and the AI assessment below stay per request.
+  const type = parsed.data.type ?? null;
+  const windowDays = parsed.data.windowDays;
+  const derived = await cachedSwr(
+    caches.insightsDerived as ServerCache<
+      Awaited<ReturnType<typeof computeDerivedMetric>>
+    >,
+    `${user.id}|single|${metric}|${type ?? ""}|${windowDays ?? ""}`,
+    async () =>
+      computeDerivedMetric({
+        metric,
+        userId: user.id,
+        profile: await loadBaselineProfile(prisma, user.id),
+        type,
+        windowDays,
+      }),
+    annotate,
+  );
 
   annotate({
     action: { name: "insights.derived" },