mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-11 05:42:19 +00:00
Marc-André Bombeck
dfe34386be
Every write to the per-sign-in Codex memory checked the account digest, but the reads did not. A credential reconnected to another ChatGPT account therefore tried the old account's working model first, sorted by its refusals and trusted its model list, until the first write rebound the entry, and the vision gate kept answering from the old account in the meantime. The client reads (working model, refusals, model list) now take the credential and treat an entry learned under another account as empty. The vision gate holds no account id, so the connect routes rebind the entry and the disconnect routes drop it, and the gate reads the entry as its process last bound it.
353 lines
12 KiB
TypeScript
353 lines
12 KiB
TypeScript
/**
|
|
* The Codex client end to end against a stubbed backend: the account's model
|
|
* list read the way the Codex CLI reads it, the walk built from it, and the
|
|
* working model kept per sign-in (#1287).
|
|
*
|
|
* Every request goes through the real `safeFetch` into a stubbed `fetch`
|
|
* that answers `/codex/models` and `/codex/responses` per account, so the
|
|
* list read, the cache around it and the walk are all the production code.
|
|
*/
|
|
import { readFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
|
|
import { CodexClient } from "../codex-client";
|
|
import { singleUserTurn } from "../types";
|
|
import {
|
|
CODEX_SLUG_CACHE_TTL_MS,
|
|
clearCodexSlugCache,
|
|
codexCredentialKey,
|
|
getCachedCodexSlug,
|
|
} from "../codex-slug-cache";
|
|
|
|
const FIXTURE = JSON.parse(
|
|
readFileSync(join(__dirname, "fixtures/codex/models-response.json"), "utf8"),
|
|
) as { models: Array<{ slug: string }> };
|
|
|
|
/** The fixture cut to `slugs`, as a plan that offers only those would see it. */
|
|
function planList(...slugs: string[]): string {
|
|
return JSON.stringify({
|
|
models: FIXTURE.models.filter((m) => slugs.includes(m.slug)),
|
|
});
|
|
}
|
|
|
|
function sse(text = '{"ok":true}'): Response {
|
|
const body = [
|
|
`data: ${JSON.stringify({
|
|
type: "response.output_item.done",
|
|
item: {
|
|
type: "message",
|
|
role: "assistant",
|
|
content: [{ type: "output_text", text }],
|
|
},
|
|
})}\n\n`,
|
|
`data: ${JSON.stringify({
|
|
type: "response.completed",
|
|
response: { usage: { total_tokens: 3 } },
|
|
})}\n\n`,
|
|
].join("");
|
|
return new Response(body, {
|
|
status: 200,
|
|
headers: { "content-type": "text/event-stream" },
|
|
});
|
|
}
|
|
|
|
function retired(slug: string): Response {
|
|
return new Response(
|
|
JSON.stringify({
|
|
detail: `The '${slug}' model is not supported when using Codex with a ChatGPT account.`,
|
|
error: { type: "invalid_request_error", code: "model_not_found" },
|
|
}),
|
|
{ status: 400, headers: { "content-type": "application/json" } },
|
|
);
|
|
}
|
|
|
|
interface Account {
|
|
/** The `/models` answer, or a function standing in for a failure. */
|
|
models: string | (() => Response | Promise<Response>);
|
|
/** The models the responses endpoint accepts for this account. */
|
|
accepts: string[];
|
|
}
|
|
|
|
/**
|
|
* A backend with one entry per ChatGPT account id. Records every request as
|
|
* `<account> models` or `<account> responses:<slug>`.
|
|
*/
|
|
function stubBackend(accounts: Record<string, Account>) {
|
|
const log: string[] = [];
|
|
const fetchMock = vi.fn(async (url: string, init: RequestInit) => {
|
|
const headers = init.headers as Record<string, string>;
|
|
const account = accounts[headers["ChatGPT-Account-ID"]];
|
|
if (!account) return new Response("unknown account", { status: 401 });
|
|
const name = headers["ChatGPT-Account-ID"];
|
|
if (url.includes("/codex/models")) {
|
|
log.push(`${name} models`);
|
|
return typeof account.models === "string"
|
|
? new Response(account.models, { status: 200 })
|
|
: account.models();
|
|
}
|
|
const slug = JSON.parse(String(init.body)).model as string;
|
|
log.push(`${name} responses:${slug}`);
|
|
return account.accepts.includes(slug) ? sse() : retired(slug);
|
|
});
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
return log;
|
|
}
|
|
|
|
function client(userKey: string, accountId: string) {
|
|
return new CodexClient({
|
|
accessToken: `token-${accountId}`,
|
|
accountId,
|
|
credentialKey: userKey,
|
|
onTokenRefresh: vi.fn(),
|
|
});
|
|
}
|
|
|
|
const ask = (c: CodexClient) =>
|
|
c.generateCompletion(singleUserTurn({ system: "s", user: "u" }));
|
|
|
|
const ENV_KEYS = ["CODEX_MODEL", "CODEX_MODEL_FALLBACK_CHAIN"] as const;
|
|
let savedEnv: Record<string, string | undefined> = {};
|
|
|
|
beforeEach(() => {
|
|
savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
|
|
for (const k of ENV_KEYS) delete process.env[k];
|
|
clearCodexSlugCache();
|
|
});
|
|
|
|
afterEach(() => {
|
|
for (const k of ENV_KEYS) {
|
|
if (savedEnv[k] === undefined) delete process.env[k];
|
|
else process.env[k] = savedEnv[k];
|
|
}
|
|
vi.unstubAllGlobals();
|
|
vi.useRealTimers();
|
|
});
|
|
|
|
describe("Codex reads the account's live model list", () => {
|
|
it("goes straight to an offered model after a retirement the built-in order does not know about", async () => {
|
|
// The 2026-10-14 shape: nothing on the built-in order's head is offered.
|
|
const log = stubBackend({
|
|
plus: {
|
|
models: planList("gpt-6.1-sol", "gpt-6-luna"),
|
|
accepts: ["gpt-6.1-sol", "gpt-6-luna"],
|
|
},
|
|
});
|
|
const c = client(codexCredentialKey.user("u1"), "plus");
|
|
await ask(c);
|
|
expect(log).toEqual(["plus models", "plus responses:gpt-6.1-sol"]);
|
|
expect(c.getLastDiagnostics()?.attempted).toEqual(["gpt-6.1-sol"]);
|
|
});
|
|
|
|
it("does not read the list again while the working model holds", async () => {
|
|
const log = stubBackend({
|
|
plus: { models: planList("gpt-6-sol"), accepts: ["gpt-6-sol"] },
|
|
});
|
|
const c = client(codexCredentialKey.user("u1"), "plus");
|
|
await ask(c);
|
|
await ask(c);
|
|
await ask(c);
|
|
expect(log.filter((l) => l.endsWith("models"))).toHaveLength(1);
|
|
expect(c.getLastDiagnostics()?.cacheState).toBe("hit");
|
|
});
|
|
|
|
it("a models call that fails still reaches a working model through the static walk", async () => {
|
|
for (const failure of [
|
|
() => new Response("bad gateway", { status: 502 }),
|
|
() => new Response("<html>", { status: 200 }),
|
|
() => new Response('{"models":[]}', { status: 200 }),
|
|
() => {
|
|
throw new TypeError("fetch failed");
|
|
},
|
|
]) {
|
|
clearCodexSlugCache();
|
|
const log = stubBackend({
|
|
plus: { models: failure, accepts: ["gpt-6-sol"] },
|
|
});
|
|
const c = client(codexCredentialKey.user("u1"), "plus");
|
|
await ask(c);
|
|
// Today's walk: terra refused, gpt-6-sol answers.
|
|
expect(log).toEqual([
|
|
"plus models",
|
|
"plus responses:gpt-5.6-terra",
|
|
"plus responses:gpt-6-sol",
|
|
]);
|
|
}
|
|
});
|
|
|
|
it("a models call that never answers is cut off and the walk goes on", async () => {
|
|
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
|
const log = stubBackend({
|
|
plus: {
|
|
// Resolves only when the 5 s bound aborts it.
|
|
models: () =>
|
|
new Promise<Response>((_, reject) =>
|
|
setTimeout(
|
|
() =>
|
|
reject(
|
|
new DOMException("The operation timed out.", "TimeoutError"),
|
|
),
|
|
5_000,
|
|
),
|
|
),
|
|
accepts: ["gpt-5.6-terra"],
|
|
},
|
|
});
|
|
const c = client(codexCredentialKey.user("u1"), "plus");
|
|
const answer = ask(c);
|
|
await vi.advanceTimersByTimeAsync(5_000);
|
|
await answer;
|
|
expect(log).toEqual(["plus models", "plus responses:gpt-5.6-terra"]);
|
|
});
|
|
|
|
it("an explicit CODEX_MODEL still goes first, ahead of the list", async () => {
|
|
process.env.CODEX_MODEL = "gpt-6-astra";
|
|
const log = stubBackend({
|
|
plus: {
|
|
models: planList("gpt-6-astra", "gpt-6-sol"),
|
|
accepts: ["gpt-6-astra", "gpt-6-sol"],
|
|
},
|
|
});
|
|
await ask(client(codexCredentialKey.user("u1"), "plus"));
|
|
expect(log).toEqual(["plus models", "plus responses:gpt-6-astra"]);
|
|
});
|
|
|
|
it("CODEX_MODEL_FALLBACK_CHAIN replaces the order and the list is not read", async () => {
|
|
process.env.CODEX_MODEL_FALLBACK_CHAIN = "gpt-6-luna";
|
|
const log = stubBackend({
|
|
plus: { models: planList("gpt-6-sol"), accepts: ["gpt-6-luna"] },
|
|
});
|
|
await ask(client(codexCredentialKey.user("u1"), "plus"));
|
|
expect(log).toEqual(["plus responses:gpt-6-luna"]);
|
|
});
|
|
});
|
|
|
|
describe("Codex keeps the working model per sign-in", () => {
|
|
it("two sign-ins on different plans do not overwrite each other", async () => {
|
|
const log = stubBackend({
|
|
// One plan still has terra; the other only the gpt-6 line.
|
|
pro: {
|
|
models: planList("gpt-5.6-terra", "gpt-6-sol"),
|
|
accepts: ["gpt-5.6-terra", "gpt-6-sol"],
|
|
},
|
|
edu: {
|
|
models: planList("gpt-6.1-sol", "gpt-6-luna"),
|
|
accepts: ["gpt-6.1-sol", "gpt-6-luna"],
|
|
},
|
|
});
|
|
const a = client(codexCredentialKey.user("a"), "pro");
|
|
const b = client(codexCredentialKey.user("b"), "edu");
|
|
const central = client(codexCredentialKey.central, "edu");
|
|
|
|
await ask(a);
|
|
await ask(b);
|
|
await ask(central);
|
|
await ask(a);
|
|
await ask(b);
|
|
|
|
expect(
|
|
getCachedCodexSlug({
|
|
key: codexCredentialKey.user("a"),
|
|
accountId: "pro",
|
|
}),
|
|
).toBe("gpt-5.6-terra");
|
|
expect(
|
|
getCachedCodexSlug({
|
|
key: codexCredentialKey.user("b"),
|
|
accountId: "edu",
|
|
}),
|
|
).toBe("gpt-6.1-sol");
|
|
expect(
|
|
getCachedCodexSlug({ key: codexCredentialKey.central, accountId: "edu" }),
|
|
).toBe("gpt-6.1-sol");
|
|
// Not one refused request: each sign-in only ever asked what it is offered.
|
|
expect(log.filter((l) => l.includes("responses"))).toEqual([
|
|
"pro responses:gpt-5.6-terra",
|
|
"edu responses:gpt-6.1-sol",
|
|
"edu responses:gpt-6.1-sol",
|
|
"pro responses:gpt-5.6-terra",
|
|
"edu responses:gpt-6.1-sol",
|
|
]);
|
|
});
|
|
|
|
it("a listed model that answers model_not_found is skipped for that sign-in only", async () => {
|
|
vi.useFakeTimers({ toFake: ["Date"] });
|
|
const both = planList("gpt-5.6-terra", "gpt-6-sol");
|
|
const log = stubBackend({
|
|
// The list still offers terra, but the backend already retired it.
|
|
stale: { models: both, accepts: ["gpt-6-sol"] },
|
|
fresh: { models: both, accepts: ["gpt-5.6-terra", "gpt-6-sol"] },
|
|
});
|
|
const stale = client(codexCredentialKey.user("s"), "stale");
|
|
const fresh = client(codexCredentialKey.user("f"), "fresh");
|
|
|
|
await ask(stale);
|
|
await ask(fresh);
|
|
expect(log).toEqual([
|
|
"stale models",
|
|
"stale responses:gpt-5.6-terra",
|
|
"stale responses:gpt-6-sol",
|
|
"fresh models",
|
|
"fresh responses:gpt-5.6-terra",
|
|
]);
|
|
|
|
// Past the working model's TTL the stale sign-in walks again, and terra
|
|
// is at the back of its line even though the list still offers it.
|
|
log.length = 0;
|
|
vi.setSystemTime(Date.now() + CODEX_SLUG_CACHE_TTL_MS + 1);
|
|
await ask(stale);
|
|
expect(log).toEqual(["stale models", "stale responses:gpt-6-sol"]);
|
|
});
|
|
|
|
it("a credential reconnected to another account tries nothing the old account learned", async () => {
|
|
const log = stubBackend({
|
|
pro: {
|
|
models: planList("gpt-5.6-terra", "gpt-6-sol"),
|
|
accepts: ["gpt-5.6-terra", "gpt-6-sol"],
|
|
},
|
|
// Another plan: terra is neither offered nor accepted.
|
|
edu: { models: planList("gpt-6.1-sol"), accepts: ["gpt-6.1-sol"] },
|
|
});
|
|
const key = codexCredentialKey.user("a");
|
|
await ask(client(key, "pro"));
|
|
expect(log).toEqual(["pro models", "pro responses:gpt-5.6-terra"]);
|
|
|
|
// The same stored credential, signed in again on the other account, in
|
|
// a process that has not seen the connect (no write has rebound it).
|
|
log.length = 0;
|
|
const again = client(key, "edu");
|
|
await ask(again);
|
|
expect(log).toEqual(["edu models", "edu responses:gpt-6.1-sol"]);
|
|
expect(again.getLastDiagnostics()).toMatchObject({ cacheState: "miss" });
|
|
});
|
|
|
|
it("the working model that stops answering is dropped and the walk goes on", async () => {
|
|
const accepts = ["gpt-5.6-terra", "gpt-6-sol"];
|
|
const log = stubBackend({
|
|
plus: { models: planList("gpt-5.6-terra", "gpt-6-sol"), accepts },
|
|
});
|
|
const c = client(codexCredentialKey.user("u1"), "plus");
|
|
await ask(c);
|
|
expect(
|
|
getCachedCodexSlug({
|
|
key: codexCredentialKey.user("u1"),
|
|
accountId: "plus",
|
|
}),
|
|
).toBe("gpt-5.6-terra");
|
|
|
|
// terra retires between two requests.
|
|
accepts.splice(0, 1);
|
|
log.length = 0;
|
|
await ask(c);
|
|
expect(log).toEqual([
|
|
"plus responses:gpt-5.6-terra",
|
|
"plus responses:gpt-6-sol",
|
|
]);
|
|
expect(c.getLastDiagnostics()).toMatchObject({
|
|
cacheState: "hit",
|
|
attempted: ["gpt-5.6-terra", "gpt-6-sol"],
|
|
workingSlug: "gpt-6-sol",
|
|
});
|
|
});
|
|
});
|