diff --git a/src/app/api/admin/central-codex/device-poll/route.ts b/src/app/api/admin/central-codex/device-poll/route.ts index b5e77028f..86aab4eed 100644 --- a/src/app/api/admin/central-codex/device-poll/route.ts +++ b/src/app/api/admin/central-codex/device-poll/route.ts @@ -8,6 +8,7 @@ import { prisma } from "@/lib/db"; import { auditLog } from "@/lib/auth/audit"; import { getClientIp, apiSuccess, apiError } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; +import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger"; /** * Operator-shared central Codex — device-code poll (admin only). @@ -91,6 +92,9 @@ export const POST = apiHandler(async (request: NextRequest) => { }); cookieStore.delete("central_codex_device"); + // A new central sign-in: every account's ledger rows describe the refused + // one, and the central connection is one credential shared by all of them. + await forgetProviderHealth({ providerType: "admin-codex" }); await auditLog("admin.central-codex.connected", { userId: user.id, diff --git a/src/app/api/auth/codex/device-poll/route.ts b/src/app/api/auth/codex/device-poll/route.ts index 1853de300..47f9987b0 100644 --- a/src/app/api/auth/codex/device-poll/route.ts +++ b/src/app/api/auth/codex/device-poll/route.ts @@ -9,6 +9,7 @@ import { auditLog } from "@/lib/auth/audit"; import { getClientIp, apiSuccess, apiError } from "@/lib/api-response"; import { annotate } from "@/lib/logging/context"; import { INSIGHTS_CACHE_CLEARED } from "@/lib/insights/cached-text-cipher"; +import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger"; /** * v1.4.7.1: Device-code flow polling endpoint. @@ -88,6 +89,9 @@ export const POST = apiHandler(async (request: NextRequest) => { }); cookieStore.delete("codex_device"); + // A new sign-in: whatever the ledger held against the old one no longer + // describes this credential. + await forgetProviderHealth({ userId: user.id, providerType: "codex" }); await auditLog("codex.oauth.connected", { userId: user.id, diff --git a/src/lib/ai/provider-health-ledger.ts b/src/lib/ai/provider-health-ledger.ts index 1ffbb461c..b6e25a4b6 100644 --- a/src/lib/ai/provider-health-ledger.ts +++ b/src/lib/ai/provider-health-ledger.ts @@ -266,6 +266,33 @@ export const postgresProviderHealthLedger: ProviderHealthLedger = { }, }; +/** + * Drop the recorded outcomes of a credential that has just been replaced. + * + * The ledger describes a credential, not a provider slot: once the person + * (or, for `admin-codex`, the operator) signs in again, the `auth_failed` + * row of the refused credential would otherwise keep the fresh one at the + * tail of the chain and reported as expired for the rest of its cooldown. + * Without `userId` the rows of every account are dropped — the operator's + * central connection is one credential shared by all of them. Best effort, + * like every ledger write. + */ +export async function forgetProviderHealth(target: { + providerType: ProviderChainType; + userId?: string; +}): Promise { + try { + await prisma.providerHealth.deleteMany({ + where: { + providerType: target.providerType, + ...(target.userId ? { userId: target.userId } : {}), + }, + }); + } catch { + // A stale row only deprioritises; it never blocks. + } +} + /** * Providers whose credential is currently known-bad (auth-class failure * inside the cooldown window) for a user. Drives the proactive diff --git a/src/lib/ai/provider.ts b/src/lib/ai/provider.ts index 51f99b5c6..e05758bed 100644 --- a/src/lib/ai/provider.ts +++ b/src/lib/ai/provider.ts @@ -30,6 +30,7 @@ import { type ProviderWorkAuthority, } from "@/lib/sharing/provider-work-authority"; import { memoizePerRequest } from "@/lib/request-cache"; +import { emitSignal } from "@/lib/logging/signal"; import type { AiModality } from "@/lib/ai/capabilities/types"; import type { ProviderEntryPresence, @@ -42,6 +43,93 @@ import { const TOKEN_REFRESH_BUFFER_MS = 5 * 60 * 1000; // 5 minutes +/** + * Whether a failed Codex refresh proves the sign-in dead: the issuer refused + * the grant itself (`CodexRefreshError.credentialExpired`), as opposed to a + * 429, a 5xx or a network error, which say nothing about the credential. + * Read by shape rather than `instanceof` so a mocked `codex-oauth` module + * classifies the same way. + */ +function isDeadCodexSignIn(error: unknown): boolean { + return ( + (error as { credentialExpired?: unknown } | null)?.credentialExpired === + true + ); +} + +function oauthErrorOf(error: unknown): string | null { + const code = (error as { oauthError?: unknown } | null)?.oauthError; + return typeof code === "string" ? code : null; +} + +/** + * Park a Codex connection whose sign-in the issuer has refused for good. + * + * Retrying a dead refresh token cannot succeed; until this existed every + * generation of every card of every run asked the issuer again, failed with + * the same 401, and logged it. Parking flips the connection to the existing + * `expired` state — the one Settings already renders as "reconnect ChatGPT" — + * so the chain, the presence probe and with it every AI capability stop + * counting it, and nothing is sent until the person signs in again (the + * device-code flow writes `connected`). + * + * Compare-and-set on the refresh token that was refused: a reconnect that + * landed in between wrote a different one and is left alone, and of several + * workers failing at once exactly one flips the row, so the transition is + * signalled once and never again per run. + */ +async function parkUserCodex( + userId: string, + refusedRefreshEncrypted: string, + error: unknown, +): Promise { + try { + const parked = await prisma.user.updateMany({ + where: { + id: userId, + codexConnectionStatus: "connected", + codexRefreshTokenEncrypted: refusedRefreshEncrypted, + }, + data: { codexConnectionStatus: "expired" }, + }); + if (parked.count === 1) { + emitSignal({ + action: "ai.credential.expired", + level: "warn", + meta: { provider: "codex", oauth_error: oauthErrorOf(error) }, + }); + } + } catch { + // Best effort: an unparked connection is retried exactly as before. + } +} + +/** The operator's central Codex, parked the same way. */ +async function parkCentralCodex( + refusedRefreshEncrypted: string, + error: unknown, +): Promise { + try { + const parked = await prisma.appSettings.updateMany({ + where: { + id: "singleton", + adminCodexConnectionStatus: "connected", + adminCodexRefreshTokenEncrypted: refusedRefreshEncrypted, + }, + data: { adminCodexConnectionStatus: "expired" }, + }); + if (parked.count === 1) { + emitSignal({ + action: "ai.credential.expired", + level: "warn", + meta: { provider: "admin-codex", oauth_error: oauthErrorOf(error) }, + }); + } + } catch { + // Best effort, as above. + } +} + class NoProvider implements AIProvider { readonly type = "none" as const; @@ -308,9 +396,15 @@ async function resolveCodexProvider( codexTokenExpiresAt: fresh.expiresAt, }, }); - } catch { - // Fall through — CodexClient will trigger an on-401 refresh - // and persist there. + } catch (error) { + // A refused grant parks the connection and drops it from this chain: + // calling Codex with the expired access token could only 401 into a + // second refusal. Anything else falls through — CodexClient will + // trigger an on-401 refresh and persist there. + if (isDeadCodexSignIn(error)) { + await parkUserCodex(userId, user.codexRefreshTokenEncrypted, error); + return null; + } } } @@ -336,9 +430,27 @@ async function resolveCodexProvider( refreshEncrypted: freshUser.codexRefreshTokenEncrypted, }); if (!decoded) { - throw codexReconnectRequired("token storage corrupt"); + const error = codexReconnectRequired("token storage corrupt"); + await parkUserCodex( + userId, + freshUser.codexRefreshTokenEncrypted, + error, + ); + throw error; + } + let fresh; + try { + fresh = await refreshDeviceTokens(decoded.refreshToken); + } catch (error) { + if (isDeadCodexSignIn(error)) { + await parkUserCodex( + userId, + freshUser.codexRefreshTokenEncrypted, + error, + ); + } + throw error; } - const fresh = await refreshDeviceTokens(decoded.refreshToken); const enc = encryptCodexCreds(fresh); await prisma.user.update({ where: { id: userId }, @@ -421,9 +533,14 @@ async function resolveAdminCodexProvider(): Promise { adminCodexTokenExpiresAt: enc.expiresAt, }, }); - } catch { - // Fall through — CodexClient triggers an on-401 refresh and persists via - // the callback below. + } catch (error) { + // A refused grant parks the central connection (see `parkUserCodex`); + // anything else falls through — CodexClient triggers an on-401 refresh + // and persists via the callback below. + if (isDeadCodexSignIn(error)) { + await parkCentralCodex(settings.adminCodexRefreshTokenEncrypted, error); + return null; + } } } @@ -454,9 +571,19 @@ async function resolveAdminCodexProvider(): Promise { expiresAt: fresh.adminCodexTokenExpiresAt, }); if (!decoded) { - throw codexReconnectRequired("central token storage corrupt"); + const error = codexReconnectRequired("central token storage corrupt"); + await parkCentralCodex(fresh.adminCodexRefreshTokenEncrypted, error); + throw error; + } + let refreshed; + try { + refreshed = await refreshDeviceTokens(decoded.refreshToken); + } catch (error) { + if (isDeadCodexSignIn(error)) { + await parkCentralCodex(fresh.adminCodexRefreshTokenEncrypted, error); + } + throw error; } - const refreshed = await refreshDeviceTokens(decoded.refreshToken); const enc = encryptAdminCodexCreds(refreshed); await prisma.appSettings.update({ where: { id: "singleton" }, diff --git a/src/lib/jobs/insight-pregenerate.ts b/src/lib/jobs/insight-pregenerate.ts index c67b1c929..509551b51 100644 --- a/src/lib/jobs/insight-pregenerate.ts +++ b/src/lib/jobs/insight-pregenerate.ts @@ -888,7 +888,8 @@ export async function runInsightPregenerate( } else if (generated.credentialExpired) { // v1.42 — a dead sign-in (a Codex refresh token the issuer // revoked) does not heal in 45 minutes either; the person has - // to reconnect, and the ledger already tells them so. + // to reconnect. The refusal parked the connection as expired, + // which Settings shows, and the next run no longer counts it. settle("auth_failed", generated.reason); } else { settle("failed", generated.reason); diff --git a/tests/integration/codex-credential-park.test.ts b/tests/integration/codex-credential-park.test.ts new file mode 100644 index 000000000..95ea57391 --- /dev/null +++ b/tests/integration/codex-credential-park.test.ts @@ -0,0 +1,263 @@ +/** + * A Codex sign-in the issuer refused is parked until the person signs in + * again, against the real rows. + * + * Before the park, a dead refresh token was asked again on every hop of + * every run: the chain resolved Codex, the proactive refresh failed and was + * swallowed, the request 401'd, the on-401 refresh failed again, and the + * nightly warm repeated that for the briefing and every card. Parking flips + * the connection to the `expired` state Settings already renders, so the + * chain and the presence probe stop counting it and nothing leaves until a + * new sign-in writes `connected`. + */ +import { NextRequest } from "next/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +process.env.ENCRYPTION_KEY ??= + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +import { cookieJar, headerJar } from "./mock-next-headers"; +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("next/headers", async () => { + const { cookieJar, headerJar } = await import("./mock-next-headers"); + return { + headers: vi.fn(async () => ({ + get: (name: string) => headerJar.get(name.toLowerCase()) ?? null, + })), + cookies: vi.fn(async () => ({ + get: (name: string) => { + const value = cookieJar.get(name); + return value ? { name, value } : undefined; + }, + set: (name: string, value: string) => { + cookieJar.set(name, value); + }, + delete: (name: string) => { + cookieJar.delete(name); + }, + })), + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/logging/signal", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, emitSignal: vi.fn(actual.emitSignal) }; +}); + +// Only the two calls that would leave the machine are replaced; the +// credential codec stays real so the rows hold genuine ciphertext. +vi.mock("@/lib/ai/codex-oauth", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + refreshDeviceTokens: vi.fn(), + pollDeviceCode: vi.fn(), + }; +}); + +import { + CodexRefreshError, + encryptAdminCodexCreds, + encryptCodexCreds, + pollDeviceCode, + refreshDeviceTokens, +} from "@/lib/ai/codex-oauth"; +import { encrypt } from "@/lib/crypto"; +import { emitSignal } from "@/lib/logging/signal"; +import { probeProviderChain, resolveProviderChain } from "@/lib/ai/provider"; +import { postgresProviderHealthLedger } from "@/lib/ai/provider-health-ledger"; + +const prisma = getPrismaClient(); + +const HOUR_MS = 60 * 60 * 1000; + +function creds(tag: string, expiresInMs: number) { + return { + accessToken: `access-${tag}`, + refreshToken: `refresh-${tag}`, + accountId: `account-${tag}`, + expiresAt: new Date(Date.now() + expiresInMs), + }; +} + +function parkSignals(provider: string) { + return vi + .mocked(emitSignal) + .mock.calls.filter( + ([input]) => + input.action === "ai.credential.expired" && + input.meta?.provider === provider, + ); +} + +async function seedCodexUser() { + // An access token that expired long ago: the state a dead sign-in sits in. + const enc = encryptCodexCreds(creds("old", -HOUR_MS)); + const user = await prisma.user.create({ + data: { + username: "codex-park", + email: "codex-park@example.test", + aiProviderChain: [{ providerType: "codex", priority: 1, enabled: true }], + codexAccessTokenEncrypted: enc.accessEncrypted, + codexRefreshTokenEncrypted: enc.refreshEncrypted, + codexConnectionStatus: "connected", + codexConnectedAt: new Date(), + }, + }); + const session = await prisma.session.create({ + data: { userId: user.id, expiresAt: new Date(Date.now() + 60_000) }, + }); + cookieJar.set("healthlog_session", session.id); + return user; +} + +beforeEach(async () => { + await truncateAllTables(prisma); + cookieJar.clear(); + headerJar.clear(); + vi.mocked(emitSignal).mockClear(); + vi.mocked(refreshDeviceTokens).mockReset(); + vi.mocked(pollDeviceCode).mockReset(); +}); + +afterEach(() => { + vi.mocked(refreshDeviceTokens).mockReset(); +}); + +describe("a refused Codex sign-in", () => { + it("is parked once, stops being asked, and resumes after a new sign-in", async () => { + const user = await seedCodexUser(); + vi.mocked(refreshDeviceTokens).mockRejectedValue( + new CodexRefreshError(400, "invalid_grant"), + ); + + // Two workers meet the refusal at once: one transition, one signal. + const [first, second] = await Promise.all([ + resolveProviderChain(user.id), + resolveProviderChain(user.id), + ]); + expect(first.map((p) => p.providerType)).not.toContain("codex"); + expect(second.map((p) => p.providerType)).not.toContain("codex"); + expect( + ( + await prisma.user.findUniqueOrThrow({ + where: { id: user.id }, + select: { codexConnectionStatus: true }, + }) + ).codexConnectionStatus, + ).toBe("expired"); + expect(parkSignals("codex")).toHaveLength(1); + expect(parkSignals("codex")[0][0].meta?.oauth_error).toBe("invalid_grant"); + + // Parked: later runs neither ask the issuer nor count Codex as present. + const asked = vi.mocked(refreshDeviceTokens).mock.calls.length; + for (let i = 0; i < 3; i += 1) { + const chain = await resolveProviderChain(user.id); + expect(chain.map((p) => p.providerType)).not.toContain("codex"); + } + expect(vi.mocked(refreshDeviceTokens).mock.calls.length).toBe(asked); + expect(parkSignals("codex")).toHaveLength(1); + const presence = await probeProviderChain(user.id); + expect(presence.availableTypes).not.toContain("codex"); + + // The hop that met the refusal left an auth_failed row behind. + await postgresProviderHealthLedger.recordFailure(user.id, "codex", 401); + expect( + (await postgresProviderHealthLedger.getSkipHints(user.id)).get("codex") + ?.reason, + ).toBe("credential_expired"); + + // The person signs in again through the device-code flow. + vi.mocked(pollDeviceCode).mockResolvedValue({ + status: "connected", + creds: creds("new", 10 * HOUR_MS), + } as Awaited>); + cookieJar.set( + "codex_device", + encrypt(JSON.stringify({ deviceAuthId: "device", userCode: "CODE" })), + ); + const { POST } = await import("@/app/api/auth/codex/device-poll/route"); + const res = await POST( + new NextRequest("http://localhost/api/auth/codex/device-poll", { + method: "POST", + }), + ); + expect(res.status).toBe(200); + + // Resumed in place: connected, first in the chain, not benched by the + // refused credential's ledger row, and the fresh token needs no refresh. + const resumed = await resolveProviderChain(user.id); + expect(resumed[0]?.providerType).toBe("codex"); + expect( + (await postgresProviderHealthLedger.getSkipHints(user.id)).has("codex"), + ).toBe(false); + expect(vi.mocked(refreshDeviceTokens).mock.calls.length).toBe(asked); + }); + + it("is not parked by a refusal that says nothing about the credential", async () => { + const user = await seedCodexUser(); + vi.mocked(refreshDeviceTokens).mockRejectedValue( + new CodexRefreshError(503, null), + ); + + const chain = await resolveProviderChain(user.id); + + // A brown-out at the issuer: the connection stays, the client's own + // on-401 path gets its chance, and nothing is signalled. + expect(chain.map((p) => p.providerType)).toContain("codex"); + expect( + ( + await prisma.user.findUniqueOrThrow({ + where: { id: user.id }, + select: { codexConnectionStatus: true }, + }) + ).codexConnectionStatus, + ).toBe("connected"); + expect(parkSignals("codex")).toHaveLength(0); + }); + + it("parks the operator's central connection the same way", async () => { + const enc = encryptAdminCodexCreds(creds("central", -HOUR_MS)); + await prisma.appSettings.create({ + data: { + id: "singleton", + adminCodexAccessTokenEncrypted: enc.accessEncrypted, + adminCodexRefreshTokenEncrypted: enc.refreshEncrypted, + adminCodexAccountIdEncrypted: enc.accountIdEncrypted, + adminCodexTokenExpiresAt: enc.expiresAt, + adminCodexConnectionStatus: "connected", + }, + }); + const user = await prisma.user.create({ + data: { + username: "central-park", + email: "central-park@example.test", + aiProviderChain: [], + useCentralCodex: true, + }, + }); + vi.mocked(refreshDeviceTokens).mockRejectedValue( + new CodexRefreshError(401, null), + ); + + const chain = await resolveProviderChain(user.id); + expect(chain.map((p) => p.providerType)).not.toContain("admin-codex"); + expect( + ( + await prisma.appSettings.findUniqueOrThrow({ + where: { id: "singleton" }, + select: { adminCodexConnectionStatus: true }, + }) + ).adminCodexConnectionStatus, + ).toBe("expired"); + expect(parkSignals("admin-codex")).toHaveLength(1); + + await resolveProviderChain(user.id); + expect(vi.mocked(refreshDeviceTokens)).toHaveBeenCalledTimes(1); + }); +});