fix(ai): park a Codex sign-in the issuer refused until the next sign-in

A dead refresh token was asked again on every hop of every run: the
proactive refresh failed and was swallowed, the request 401'd, the on-401
refresh failed again, and the nightly warm repeated that for the briefing
and every card. A refusal that proves the grant dead now flips the
connection to the existing expired state (compare-and-set on the refused
token, one signal per transition), so the chain, the presence probe and
the AI capabilities stop counting it. A new device-code sign-in clears the
refused credential's ledger rows so the fresh one resumes in place.
This commit is contained in:
Marc-André Bombeck committed 2026-10-20 20:04:21 +02:00
1 parent 5502342ec5
commit 07d70cd275
6 files changed
+437 -11

No files matched your search

@@ -8,6 +8,7 @@ import { prisma } from "@/lib/db";
import { auditLog } from "@/lib/auth/audit";
import { getClientIp, apiSuccess, apiError } from "@/lib/api-response";
import { annotate } from "@/lib/logging/context";
import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger";
/**
* Operator-shared central Codex — device-code poll (admin only).
@@ -91,6 +92,9 @@ export const POST = apiHandler(async (request: NextRequest) => {
});
cookieStore.delete("central_codex_device");
// A new central sign-in: every account's ledger rows describe the refused
// one, and the central connection is one credential shared by all of them.
await forgetProviderHealth({ providerType: "admin-codex" });
await auditLog("admin.central-codex.connected", {
userId: user.id,
@@ -9,6 +9,7 @@ import { auditLog } from "@/lib/auth/audit";
import { getClientIp, apiSuccess, apiError } from "@/lib/api-response";
import { annotate } from "@/lib/logging/context";
import { INSIGHTS_CACHE_CLEARED } from "@/lib/insights/cached-text-cipher";
import { forgetProviderHealth } from "@/lib/ai/provider-health-ledger";
/**
* v1.4.7.1: Device-code flow polling endpoint.
@@ -88,6 +89,9 @@ export const POST = apiHandler(async (request: NextRequest) => {
});
cookieStore.delete("codex_device");
// A new sign-in: whatever the ledger held against the old one no longer
// describes this credential.
await forgetProviderHealth({ userId: user.id, providerType: "codex" });
await auditLog("codex.oauth.connected", {
userId: user.id,
+27
View File
@@ -266,6 +266,33 @@ export const postgresProviderHealthLedger: ProviderHealthLedger = {
},
};
/**
* Drop the recorded outcomes of a credential that has just been replaced.
*
* The ledger describes a credential, not a provider slot: once the person
* (or, for `admin-codex`, the operator) signs in again, the `auth_failed`
* row of the refused credential would otherwise keep the fresh one at the
* tail of the chain and reported as expired for the rest of its cooldown.
* Without `userId` the rows of every account are dropped — the operator's
* central connection is one credential shared by all of them. Best effort,
* like every ledger write.
*/
export async function forgetProviderHealth(target: {
providerType: ProviderChainType;
userId?: string;
}): Promise<void> {
try {
await prisma.providerHealth.deleteMany({
where: {
providerType: target.providerType,
...(target.userId ? { userId: target.userId } : {}),
},
});
} catch {
// A stale row only deprioritises; it never blocks.
}
}
/**
* Providers whose credential is currently known-bad (auth-class failure
* inside the cooldown window) for a user. Drives the proactive
+137 -10
View File
@@ -30,6 +30,7 @@ import {
type ProviderWorkAuthority,
} from "@/lib/sharing/provider-work-authority";
import { memoizePerRequest } from "@/lib/request-cache";
import { emitSignal } from "@/lib/logging/signal";
import type { AiModality } from "@/lib/ai/capabilities/types";
import type {
ProviderEntryPresence,
@@ -42,6 +43,93 @@ import {
const TOKEN_REFRESH_BUFFER_MS = 5 * 60 * 1000; // 5 minutes
/**
* Whether a failed Codex refresh proves the sign-in dead: the issuer refused
* the grant itself (`CodexRefreshError.credentialExpired`), as opposed to a
* 429, a 5xx or a network error, which say nothing about the credential.
* Read by shape rather than `instanceof` so a mocked `codex-oauth` module
* classifies the same way.
*/
function isDeadCodexSignIn(error: unknown): boolean {
return (
(error as { credentialExpired?: unknown } | null)?.credentialExpired ===
true
);
}
function oauthErrorOf(error: unknown): string | null {
const code = (error as { oauthError?: unknown } | null)?.oauthError;
return typeof code === "string" ? code : null;
}
/**
* Park a Codex connection whose sign-in the issuer has refused for good.
*
* Retrying a dead refresh token cannot succeed; until this existed every
* generation of every card of every run asked the issuer again, failed with
* the same 401, and logged it. Parking flips the connection to the existing
* `expired` state — the one Settings already renders as "reconnect ChatGPT" —
* so the chain, the presence probe and with it every AI capability stop
* counting it, and nothing is sent until the person signs in again (the
* device-code flow writes `connected`).
*
* Compare-and-set on the refresh token that was refused: a reconnect that
* landed in between wrote a different one and is left alone, and of several
* workers failing at once exactly one flips the row, so the transition is
* signalled once and never again per run.
*/
async function parkUserCodex(
userId: string,
refusedRefreshEncrypted: string,
error: unknown,
): Promise<void> {
try {
const parked = await prisma.user.updateMany({
where: {
id: userId,
codexConnectionStatus: "connected",
codexRefreshTokenEncrypted: refusedRefreshEncrypted,
},
data: { codexConnectionStatus: "expired" },
});
if (parked.count === 1) {
emitSignal({
action: "ai.credential.expired",
level: "warn",
meta: { provider: "codex", oauth_error: oauthErrorOf(error) },
});
}
} catch {
// Best effort: an unparked connection is retried exactly as before.
}
}
/** The operator's central Codex, parked the same way. */
async function parkCentralCodex(
refusedRefreshEncrypted: string,
error: unknown,
): Promise<void> {
try {
const parked = await prisma.appSettings.updateMany({
where: {
id: "singleton",
adminCodexConnectionStatus: "connected",
adminCodexRefreshTokenEncrypted: refusedRefreshEncrypted,
},
data: { adminCodexConnectionStatus: "expired" },
});
if (parked.count === 1) {
emitSignal({
action: "ai.credential.expired",
level: "warn",
meta: { provider: "admin-codex", oauth_error: oauthErrorOf(error) },
});
}
} catch {
// Best effort, as above.
}
}
class NoProvider implements AIProvider {
readonly type = "none" as const;
@@ -308,9 +396,15 @@ async function resolveCodexProvider(
codexTokenExpiresAt: fresh.expiresAt,
},
});
} catch {
// Fall through — CodexClient will trigger an on-401 refresh
// and persist there.
} catch (error) {
// A refused grant parks the connection and drops it from this chain:
// calling Codex with the expired access token could only 401 into a
// second refusal. Anything else falls through — CodexClient will
// trigger an on-401 refresh and persist there.
if (isDeadCodexSignIn(error)) {
await parkUserCodex(userId, user.codexRefreshTokenEncrypted, error);
return null;
}
}
}
@@ -336,9 +430,27 @@ async function resolveCodexProvider(
refreshEncrypted: freshUser.codexRefreshTokenEncrypted,
});
if (!decoded) {
throw codexReconnectRequired("token storage corrupt");
const error = codexReconnectRequired("token storage corrupt");
await parkUserCodex(
userId,
freshUser.codexRefreshTokenEncrypted,
error,
);
throw error;
}
let fresh;
try {
fresh = await refreshDeviceTokens(decoded.refreshToken);
} catch (error) {
if (isDeadCodexSignIn(error)) {
await parkUserCodex(
userId,
freshUser.codexRefreshTokenEncrypted,
error,
);
}
throw error;
}
const fresh = await refreshDeviceTokens(decoded.refreshToken);
const enc = encryptCodexCreds(fresh);
await prisma.user.update({
where: { id: userId },
@@ -421,9 +533,14 @@ async function resolveAdminCodexProvider(): Promise<AIProvider | null> {
adminCodexTokenExpiresAt: enc.expiresAt,
},
});
} catch {
// Fall through — CodexClient triggers an on-401 refresh and persists via
// the callback below.
} catch (error) {
// A refused grant parks the central connection (see `parkUserCodex`);
// anything else falls through — CodexClient triggers an on-401 refresh
// and persists via the callback below.
if (isDeadCodexSignIn(error)) {
await parkCentralCodex(settings.adminCodexRefreshTokenEncrypted, error);
return null;
}
}
}
@@ -454,9 +571,19 @@ async function resolveAdminCodexProvider(): Promise<AIProvider | null> {
expiresAt: fresh.adminCodexTokenExpiresAt,
});
if (!decoded) {
throw codexReconnectRequired("central token storage corrupt");
const error = codexReconnectRequired("central token storage corrupt");
await parkCentralCodex(fresh.adminCodexRefreshTokenEncrypted, error);
throw error;
}
let refreshed;
try {
refreshed = await refreshDeviceTokens(decoded.refreshToken);
} catch (error) {
if (isDeadCodexSignIn(error)) {
await parkCentralCodex(fresh.adminCodexRefreshTokenEncrypted, error);
}
throw error;
}
const refreshed = await refreshDeviceTokens(decoded.refreshToken);
const enc = encryptAdminCodexCreds(refreshed);
await prisma.appSettings.update({
where: { id: "singleton" },
+2 -1
View File
@@ -888,7 +888,8 @@ export async function runInsightPregenerate(
} else if (generated.credentialExpired) {
// v1.42 — a dead sign-in (a Codex refresh token the issuer
// revoked) does not heal in 45 minutes either; the person has
// to reconnect, and the ledger already tells them so.
// to reconnect. The refusal parked the connection as expired,
// which Settings shows, and the next run no longer counts it.
settle("auth_failed", generated.reason);
} else {
settle("failed", generated.reason);
@@ -0,0 +1,263 @@
/**
* A Codex sign-in the issuer refused is parked until the person signs in
* again, against the real rows.
*
* Before the park, a dead refresh token was asked again on every hop of
* every run: the chain resolved Codex, the proactive refresh failed and was
* swallowed, the request 401'd, the on-401 refresh failed again, and the
* nightly warm repeated that for the briefing and every card. Parking flips
* the connection to the `expired` state Settings already renders, so the
* chain and the presence probe stop counting it and nothing leaves until a
* new sign-in writes `connected`.
*/
import { NextRequest } from "next/server";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
process.env.ENCRYPTION_KEY ??=
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
import { cookieJar, headerJar } from "./mock-next-headers";
import { getPrismaClient, truncateAllTables } from "./setup";
vi.mock("next/headers", async () => {
const { cookieJar, headerJar } = await import("./mock-next-headers");
return {
headers: vi.fn(async () => ({
get: (name: string) => headerJar.get(name.toLowerCase()) ?? null,
})),
cookies: vi.fn(async () => ({
get: (name: string) => {
const value = cookieJar.get(name);
return value ? { name, value } : undefined;
},
set: (name: string, value: string) => {
cookieJar.set(name, value);
},
delete: (name: string) => {
cookieJar.delete(name);
},
})),
};
});
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/logging/signal", async (importOriginal) => {
const actual = await importOriginal<typeof import("@/lib/logging/signal")>();
return { ...actual, emitSignal: vi.fn(actual.emitSignal) };
});
// Only the two calls that would leave the machine are replaced; the
// credential codec stays real so the rows hold genuine ciphertext.
vi.mock("@/lib/ai/codex-oauth", async (importOriginal) => {
const actual = await importOriginal<typeof import("@/lib/ai/codex-oauth")>();
return {
...actual,
refreshDeviceTokens: vi.fn(),
pollDeviceCode: vi.fn(),
};
});
import {
CodexRefreshError,
encryptAdminCodexCreds,
encryptCodexCreds,
pollDeviceCode,
refreshDeviceTokens,
} from "@/lib/ai/codex-oauth";
import { encrypt } from "@/lib/crypto";
import { emitSignal } from "@/lib/logging/signal";
import { probeProviderChain, resolveProviderChain } from "@/lib/ai/provider";
import { postgresProviderHealthLedger } from "@/lib/ai/provider-health-ledger";
const prisma = getPrismaClient();
const HOUR_MS = 60 * 60 * 1000;
function creds(tag: string, expiresInMs: number) {
return {
accessToken: `access-${tag}`,
refreshToken: `refresh-${tag}`,
accountId: `account-${tag}`,
expiresAt: new Date(Date.now() + expiresInMs),
};
}
function parkSignals(provider: string) {
return vi
.mocked(emitSignal)
.mock.calls.filter(
([input]) =>
input.action === "ai.credential.expired" &&
input.meta?.provider === provider,
);
}
async function seedCodexUser() {
// An access token that expired long ago: the state a dead sign-in sits in.
const enc = encryptCodexCreds(creds("old", -HOUR_MS));
const user = await prisma.user.create({
data: {
username: "codex-park",
email: "codex-park@example.test",
aiProviderChain: [{ providerType: "codex", priority: 1, enabled: true }],
codexAccessTokenEncrypted: enc.accessEncrypted,
codexRefreshTokenEncrypted: enc.refreshEncrypted,
codexConnectionStatus: "connected",
codexConnectedAt: new Date(),
},
});
const session = await prisma.session.create({
data: { userId: user.id, expiresAt: new Date(Date.now() + 60_000) },
});
cookieJar.set("healthlog_session", session.id);
return user;
}
beforeEach(async () => {
await truncateAllTables(prisma);
cookieJar.clear();
headerJar.clear();
vi.mocked(emitSignal).mockClear();
vi.mocked(refreshDeviceTokens).mockReset();
vi.mocked(pollDeviceCode).mockReset();
});
afterEach(() => {
vi.mocked(refreshDeviceTokens).mockReset();
});
describe("a refused Codex sign-in", () => {
it("is parked once, stops being asked, and resumes after a new sign-in", async () => {
const user = await seedCodexUser();
vi.mocked(refreshDeviceTokens).mockRejectedValue(
new CodexRefreshError(400, "invalid_grant"),
);
// Two workers meet the refusal at once: one transition, one signal.
const [first, second] = await Promise.all([
resolveProviderChain(user.id),
resolveProviderChain(user.id),
]);
expect(first.map((p) => p.providerType)).not.toContain("codex");
expect(second.map((p) => p.providerType)).not.toContain("codex");
expect(
(
await prisma.user.findUniqueOrThrow({
where: { id: user.id },
select: { codexConnectionStatus: true },
})
).codexConnectionStatus,
).toBe("expired");
expect(parkSignals("codex")).toHaveLength(1);
expect(parkSignals("codex")[0][0].meta?.oauth_error).toBe("invalid_grant");
// Parked: later runs neither ask the issuer nor count Codex as present.
const asked = vi.mocked(refreshDeviceTokens).mock.calls.length;
for (let i = 0; i < 3; i += 1) {
const chain = await resolveProviderChain(user.id);
expect(chain.map((p) => p.providerType)).not.toContain("codex");
}
expect(vi.mocked(refreshDeviceTokens).mock.calls.length).toBe(asked);
expect(parkSignals("codex")).toHaveLength(1);
const presence = await probeProviderChain(user.id);
expect(presence.availableTypes).not.toContain("codex");
// The hop that met the refusal left an auth_failed row behind.
await postgresProviderHealthLedger.recordFailure(user.id, "codex", 401);
expect(
(await postgresProviderHealthLedger.getSkipHints(user.id)).get("codex")
?.reason,
).toBe("credential_expired");
// The person signs in again through the device-code flow.
vi.mocked(pollDeviceCode).mockResolvedValue({
status: "connected",
creds: creds("new", 10 * HOUR_MS),
} as Awaited<ReturnType<typeof pollDeviceCode>>);
cookieJar.set(
"codex_device",
encrypt(JSON.stringify({ deviceAuthId: "device", userCode: "CODE" })),
);
const { POST } = await import("@/app/api/auth/codex/device-poll/route");
const res = await POST(
new NextRequest("http://localhost/api/auth/codex/device-poll", {
method: "POST",
}),
);
expect(res.status).toBe(200);
// Resumed in place: connected, first in the chain, not benched by the
// refused credential's ledger row, and the fresh token needs no refresh.
const resumed = await resolveProviderChain(user.id);
expect(resumed[0]?.providerType).toBe("codex");
expect(
(await postgresProviderHealthLedger.getSkipHints(user.id)).has("codex"),
).toBe(false);
expect(vi.mocked(refreshDeviceTokens).mock.calls.length).toBe(asked);
});
it("is not parked by a refusal that says nothing about the credential", async () => {
const user = await seedCodexUser();
vi.mocked(refreshDeviceTokens).mockRejectedValue(
new CodexRefreshError(503, null),
);
const chain = await resolveProviderChain(user.id);
// A brown-out at the issuer: the connection stays, the client's own
// on-401 path gets its chance, and nothing is signalled.
expect(chain.map((p) => p.providerType)).toContain("codex");
expect(
(
await prisma.user.findUniqueOrThrow({
where: { id: user.id },
select: { codexConnectionStatus: true },
})
).codexConnectionStatus,
).toBe("connected");
expect(parkSignals("codex")).toHaveLength(0);
});
it("parks the operator's central connection the same way", async () => {
const enc = encryptAdminCodexCreds(creds("central", -HOUR_MS));
await prisma.appSettings.create({
data: {
id: "singleton",
adminCodexAccessTokenEncrypted: enc.accessEncrypted,
adminCodexRefreshTokenEncrypted: enc.refreshEncrypted,
adminCodexAccountIdEncrypted: enc.accountIdEncrypted,
adminCodexTokenExpiresAt: enc.expiresAt,
adminCodexConnectionStatus: "connected",
},
});
const user = await prisma.user.create({
data: {
username: "central-park",
email: "central-park@example.test",
aiProviderChain: [],
useCentralCodex: true,
},
});
vi.mocked(refreshDeviceTokens).mockRejectedValue(
new CodexRefreshError(401, null),
);
const chain = await resolveProviderChain(user.id);
expect(chain.map((p) => p.providerType)).not.toContain("admin-codex");
expect(
(
await prisma.appSettings.findUniqueOrThrow({
where: { id: "singleton" },
select: { adminCodexConnectionStatus: true },
})
).adminCodexConnectionStatus,
).toBe("expired");
expect(parkSignals("admin-codex")).toHaveLength(1);
await resolveProviderChain(user.id);
expect(vi.mocked(refreshDeviceTokens)).toHaveBeenCalledTimes(1);
});
});