Files
HealthLog/src/lib/ai/__tests__/provider-runner.test.ts
T
Marc-André Bombeck fbd45f3044 feat(ai): durable provider-health ledger and credential-expired surfacing
Promote the volatile per-worker last-working cache to a Postgres
provider-health ledger so the fallback chain shares one health signal
across every instance, modelled on the atomic-upsert rate limiter.

The ledger holds a durable negative cache: an auth-class failure
(401/403) benches that provider for a cooldown instead of re-burning the
dead round-trip on every generation; a sustained hard failure backs off
briefly. The local model is never benched — it stays the guaranteed
floor of the chain. Every ledger interaction fails open, so a DB error
falls back to today's chain walk exactly.

When the user's primary provider fails auth-class, the Coach emits a
distinct credential_expired frame that deep-links to reconnect rather
than a generic "try again later".

Adds the provider_health table (migration 0112) and the
errorCredentialExpired copy across all locales.
2026-06-04 00:50:13 +02:00

664 lines
22 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { AIProvider, CompletionParams, CompletionResult } from "../types";
// v1.11.0 W1 — the runner now defaults to the Postgres-backed
// provider-health ledger. These pure chain tests do not stand up a DB,
// so swap the default for a no-op; the dedicated ledger-aware cases
// below inject an in-memory ledger explicitly.
vi.mock("../provider-health-ledger", async () => {
const actual = await vi.importActual<
typeof import("../provider-health-ledger")
>("../provider-health-ledger");
return {
...actual,
postgresProviderHealthLedger: {
async getSkipHints() {
return new Map();
},
async recordSuccess() {},
async recordFailure() {},
},
};
});
import {
AllProvidersFailedError,
clearLastWorkingProviderCache,
getLastWorkingProvider,
isHardProviderFailure,
runRawCompletionWithFallback,
runWithFallback,
} from "../provider-runner";
import {
AUTH_FAILURE_COOLDOWN_MS,
createInMemoryProviderHealthLedger,
} from "../provider-health-ledger";
const VALID_RESPONSE = JSON.stringify({
summary: "ok",
recommendations: [],
citations: [],
warnings: [],
});
interface ScriptedCall {
ok: boolean;
content?: string;
error?: Error & { httpStatus?: number };
}
class ScriptedProvider implements AIProvider {
readonly type: AIProvider["type"];
readonly model: string;
readonly calls: CompletionParams[] = [];
private readonly script: ScriptedCall[];
private cursor = 0;
constructor(opts: {
type?: AIProvider["type"];
model?: string;
script: ScriptedCall[];
}) {
this.type = opts.type ?? "local";
this.model = opts.model ?? "scripted-model";
this.script = opts.script;
}
async generateCompletion(
params: CompletionParams,
): Promise<CompletionResult> {
this.calls.push(params);
const step = this.script[Math.min(this.cursor, this.script.length - 1)];
this.cursor += 1;
if (!step.ok) {
throw step.error ?? new Error("scripted-fail");
}
return {
content: step.content ?? VALID_RESPONSE,
tokensUsed: 1,
model: this.model,
providerType: this.type,
};
}
get callCount(): number {
return this.calls.length;
}
}
function err(status: number, msg = "boom"): Error & { httpStatus: number } {
return Object.assign(new Error(msg), { httpStatus: status });
}
beforeEach(() => {
clearLastWorkingProviderCache();
});
afterEach(() => {
clearLastWorkingProviderCache();
vi.useRealTimers();
});
describe("isHardProviderFailure", () => {
it("flags 401 (auth-class) as hard", () => {
expect(isHardProviderFailure(err(401))).toBe(true);
});
it("flags 403 (auth-class) as hard", () => {
expect(isHardProviderFailure(err(403))).toBe(true);
});
it("flags 5xx (server-class) as hard after retry exhaustion", () => {
expect(isHardProviderFailure(err(500))).toBe(true);
expect(isHardProviderFailure(err(503))).toBe(true);
});
it("does NOT flag 4xx validation errors (e.g. 422) as hard — those bubble", () => {
expect(isHardProviderFailure(err(422))).toBe(false);
});
it("flags network errors (no httpStatus) as hard", () => {
expect(isHardProviderFailure(new Error("ECONNRESET"))).toBe(true);
});
it("flags zero / sentinel httpStatus as hard (transport-level)", () => {
expect(isHardProviderFailure(err(0, "ECONNRESET"))).toBe(true);
});
it("flags 429 rate-limit as hard so we walk to the next provider rather than starve out", () => {
expect(isHardProviderFailure(err(429))).toBe(true);
});
});
describe("runWithFallback — happy path", () => {
it("returns the parsed response from the first working provider", async () => {
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: true }],
});
const result = await runWithFallback({
userId: "u1",
providers: [{ providerType: "codex", instance: codex }],
params: { systemPrompt: "s", userPrompt: "u" },
});
expect(result.parsed.summary).toBe("ok");
expect(result.raw.providerType).toBe("codex");
expect(result.fallbackHops).toEqual([]);
expect(codex.callCount).toBe(1);
});
it("attaches the provider type to the outcome for B5e feedback attribution", async () => {
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true }],
});
const result = await runWithFallback({
userId: "u1",
providers: [{ providerType: "openai", instance: openai }],
params: { systemPrompt: "s", userPrompt: "u" },
});
expect(result.workingProvider.providerType).toBe("openai");
expect(result.raw.providerType).toBe("admin-key");
});
});
describe("runWithFallback — primary fails 401, secondary succeeds", () => {
it("falls through to the secondary and surfaces the failure reason in the hop log", async () => {
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(401, "OAuth expired") }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true }],
});
const result = await runWithFallback({
userId: "u1",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
});
expect(result.parsed.summary).toBe("ok");
expect(result.workingProvider.providerType).toBe("openai");
expect(result.fallbackHops).toHaveLength(1);
expect(result.fallbackHops[0]).toMatchObject({
providerType: "codex",
attempt: 1,
});
expect(result.fallbackHops[0].failureReason).toMatch(/401/);
});
});
describe("runWithFallback — all-fail cascade", () => {
it("throws AllProvidersFailedError when every chain entry fails hard", async () => {
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(0, "ECONNRESET") }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: false, error: err(503, "upstream down") }],
});
let caught: unknown;
try {
await runWithFallback({
userId: "u1",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
});
} catch (e) {
caught = e;
}
expect(caught).toBeInstanceOf(AllProvidersFailedError);
const e = caught as AllProvidersFailedError;
expect(e.httpStatus).toBe(503);
expect(e.attempts).toHaveLength(2);
expect(e.attempts.map((a) => a.providerType)).toEqual(["codex", "openai"]);
});
});
describe("runWithFallback — non-hard errors bubble unchanged", () => {
it("re-throws a 422 schema error from the first provider rather than walking", async () => {
// schema-mismatch is the wrapper's job, not the runner's. The runner
// only walks on transport / auth / 5xx — bad JSON from the first
// provider must still propagate as the wrapper's
// InsightSchemaError so the user gets the existing 422 surface.
const breaking = new ScriptedProvider({
type: "codex",
// Both wrapper attempts return invalid JSON → wrapper raises
// InsightSchemaError(422). The runner does not walk to provider
// 2 because that error is not a "hard provider failure".
script: [
{ ok: true, content: "not-json" },
{ ok: true, content: "still-not-json" },
],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true }],
});
let caught: unknown;
try {
await runWithFallback({
userId: "u1",
providers: [
{ providerType: "codex", instance: breaking },
{ providerType: "openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
});
} catch (e) {
caught = e;
}
// Wrapper threw — fallback runner should NOT have escalated to
// openai because schema-mismatch is not a hard provider failure.
expect(caught).toBeDefined();
expect((caught as Error).name).toBe("InsightSchemaError");
expect(openai.callCount).toBe(0);
});
});
describe("runWithFallback — last-working cache", () => {
it("caches the working provider so the next call starts there", async () => {
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(401) }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true }, { ok: true }],
});
// First call: codex fails, openai succeeds.
await runWithFallback({
userId: "u-cache",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
});
expect(getLastWorkingProvider("u-cache")).toBe("openai");
// Second call: cache reorders so openai is tried first; codex is
// never re-invoked even though it appears first in the input chain.
await runWithFallback({
userId: "u-cache",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
});
// codex was called exactly once (first call's failure); the cache
// skipped it on the second call.
expect(codex.callCount).toBe(1);
expect(openai.callCount).toBe(2);
});
it("expires the cache after 1 hour", async () => {
vi.useFakeTimers();
const t0 = new Date("2026-05-09T12:00:00Z");
vi.setSystemTime(t0);
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(401) }, { ok: true }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true }],
});
// First call: codex fails, openai succeeds, cache = openai.
await runWithFallback({
userId: "u-ttl",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
});
expect(getLastWorkingProvider("u-ttl")).toBe("openai");
// 61 minutes later — cache should have expired.
vi.setSystemTime(new Date(t0.getTime() + 61 * 60_000));
expect(getLastWorkingProvider("u-ttl")).toBeNull();
// Next call walks the original priority order again (codex first,
// and this time codex's second scripted entry succeeds).
await runWithFallback({
userId: "u-ttl",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
});
expect(getLastWorkingProvider("u-ttl")).toBe("codex");
});
it("clearLastWorkingProviderCache wipes the cache", async () => {
const ok = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true }],
});
await runWithFallback({
userId: "u-clear",
providers: [{ providerType: "openai", instance: ok }],
params: { systemPrompt: "s", userPrompt: "u" },
});
expect(getLastWorkingProvider("u-clear")).toBe("openai");
clearLastWorkingProviderCache();
expect(getLastWorkingProvider("u-clear")).toBeNull();
});
});
describe("runRawCompletionWithFallback — legacy route shim", () => {
it("returns the first non-hard-failure response", async () => {
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(503) }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true, content: '{ "legacy": true }' }],
});
const result = await runRawCompletionWithFallback({
userId: "u-raw",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "admin-openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
});
expect(result.result.content).toBe('{ "legacy": true }');
expect(result.workingProvider.providerType).toBe("admin-openai");
expect(result.fallbackHops).toHaveLength(1);
expect(result.fallbackHops[0].providerType).toBe("codex");
});
it("advances past an auth-class (401) failure to the next provider and counts the fallback", async () => {
// Generation routes consume this raw shim. An invalidated credential
// on the primary (401/403, or a 5xx whose body signals an
// invalidated session) must not abort the run — it advances to the
// next enabled provider and bumps the fallback count.
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(401, "token invalidated") }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true, content: '{ "ok": true }' }],
});
const result = await runRawCompletionWithFallback({
userId: "u-raw-auth-advance",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
});
expect(result.result.content).toBe('{ "ok": true }');
expect(result.workingProvider.providerType).toBe("openai");
expect(result.fallbackHops).toHaveLength(1);
expect(result.fallbackHops[0].providerType).toBe("codex");
expect(result.fallbackHops[0].httpStatus).toBe(401);
expect(codex.callCount).toBe(1);
expect(openai.callCount).toBe(1);
});
it("throws AllProvidersFailedError when every entry fails hard", async () => {
const a = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(401) }],
});
const b = new ScriptedProvider({
type: "admin-key",
script: [{ ok: false, error: err(503) }],
});
let caught: unknown;
try {
await runRawCompletionWithFallback({
userId: "u-raw-allfail",
providers: [
{ providerType: "codex", instance: a },
{ providerType: "admin-openai", instance: b },
],
params: { systemPrompt: "s", userPrompt: "u" },
});
} catch (e) {
caught = e;
}
expect(caught).toBeInstanceOf(AllProvidersFailedError);
expect((caught as AllProvidersFailedError).attempts).toHaveLength(2);
});
});
describe("runWithFallback — empty input", () => {
it("throws AllProvidersFailedError with httpStatus 422 when zero providers configured", async () => {
let caught: unknown;
try {
await runWithFallback({
userId: "u-empty",
providers: [],
params: { systemPrompt: "s", userPrompt: "u" },
});
} catch (e) {
caught = e;
}
expect(caught).toBeInstanceOf(AllProvidersFailedError);
expect((caught as AllProvidersFailedError).httpStatus).toBe(422);
});
});
// ── v1.11.0 W1 — durable provider-health ledger integration ──────────
describe("provider-health ledger — auth-failure negative cache", () => {
it("records the auth failure and deprioritises the dead provider on the next call", async () => {
const ledger = createInMemoryProviderHealthLedger();
const codex = new ScriptedProvider({
type: "codex",
// First call 401, then (if ever re-tried) it would succeed — but
// the negative cache must keep it benched.
script: [{ ok: false, error: err(401, "OAuth expired") }, { ok: true }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: true, content: '{ "ok": true }' }, { ok: true }],
});
const providers = [
{ providerType: "codex" as const, instance: codex },
{ providerType: "openai" as const, instance: openai },
];
// First call: codex 401 → openai succeeds. Ledger now benches codex.
await runRawCompletionWithFallback({
userId: "u-neg",
providers,
params: { systemPrompt: "s", userPrompt: "u" },
ledger,
});
expect((await ledger.getSkipHints("u-neg")).get("codex")?.reason).toBe(
"credential_expired",
);
// Reset the per-worker last-working cache so the ONLY thing steering
// order on the second call is the durable ledger.
clearLastWorkingProviderCache();
// Second call: codex is benched to the tail by the negative cache,
// so openai is tried first and codex is never re-invoked.
const codexCallsBefore = codex.callCount;
await runRawCompletionWithFallback({
userId: "u-neg",
providers,
params: { systemPrompt: "s", userPrompt: "u" },
ledger,
});
expect(codex.callCount).toBe(codexCallsBefore); // not re-burned
});
it("clears the negative cache after a provider succeeds again", async () => {
const ledger = createInMemoryProviderHealthLedger();
await ledger.recordFailure("u-clear-neg", "codex", 401);
expect((await ledger.getSkipHints("u-clear-neg")).size).toBe(1);
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: true }],
});
// A direct success on codex (e.g. user re-linked) clears the row.
await runRawCompletionWithFallback({
userId: "u-clear-neg",
providers: [{ providerType: "codex", instance: codex }],
params: { systemPrompt: "s", userPrompt: "u" },
ledger,
});
expect((await ledger.getSkipHints("u-clear-neg")).size).toBe(0);
});
it("lets the benched provider back to the front once the cooldown lapses", async () => {
vi.useFakeTimers();
const ledger = createInMemoryProviderHealthLedger();
await ledger.recordFailure("u-lapse", "codex", 401);
expect((await ledger.getSkipHints("u-lapse")).has("codex")).toBe(true);
vi.advanceTimersByTime(AUTH_FAILURE_COOLDOWN_MS + 1);
expect((await ledger.getSkipHints("u-lapse")).has("codex")).toBe(false);
});
});
describe("provider-health ledger — local model as guaranteed floor", () => {
it("still reaches the local provider even when every keyed provider is benched", async () => {
const ledger = createInMemoryProviderHealthLedger();
// Pre-bench every remote provider with an auth failure.
await ledger.recordFailure("u-floor", "codex", 401);
await ledger.recordFailure("u-floor", "openai", 401);
await ledger.recordFailure("u-floor", "anthropic", 401);
const dead = (type: AIProvider["type"]) =>
new ScriptedProvider({ type, script: [{ ok: false, error: err(401) }] });
const local = new ScriptedProvider({
type: "local",
script: [{ ok: true, content: '{ "floor": true }' }],
});
const result = await runRawCompletionWithFallback({
userId: "u-floor",
providers: [
{ providerType: "codex", instance: dead("codex") },
{ providerType: "openai", instance: dead("admin-key") },
{ providerType: "anthropic", instance: dead("anthropic") },
{ providerType: "local", instance: local },
],
params: { systemPrompt: "s", userPrompt: "u" },
ledger,
});
// The local floor is never benched by the negative cache and is
// reached even though every other provider is in an auth cooldown.
expect(result.result.content).toBe('{ "floor": true }');
expect(result.workingProvider.providerType).toBe("local");
expect(local.callCount).toBe(1);
});
it("never benches the local provider in the ledger even on a hard failure", async () => {
const ledger = createInMemoryProviderHealthLedger();
const local = new ScriptedProvider({
type: "local",
script: [{ ok: false, error: err(503) }, { ok: true }],
});
// A 503 on local records a hard failure but the runner must keep it
// in the chain — there is no remote credential to "expire".
let caught: unknown;
try {
await runRawCompletionWithFallback({
userId: "u-local-hard",
providers: [{ providerType: "local", instance: local }],
params: { systemPrompt: "s", userPrompt: "u" },
ledger,
});
} catch (e) {
caught = e;
}
expect(caught).toBeInstanceOf(AllProvidersFailedError);
// Even after a recorded hard failure, the next call still tries
// local (NEVER_SKIP) and this time it succeeds.
clearLastWorkingProviderCache();
const result = await runRawCompletionWithFallback({
userId: "u-local-hard",
providers: [{ providerType: "local", instance: local }],
params: { systemPrompt: "s", userPrompt: "u" },
ledger,
});
expect(result.workingProvider.providerType).toBe("local");
});
});
describe("AllProvidersFailedError — primaryCredentialExpired", () => {
it("flags primaryCredentialExpired when the first hop is auth-class", async () => {
const ledger = createInMemoryProviderHealthLedger();
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(401) }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: false, error: err(503) }],
});
let caught: unknown;
try {
await runRawCompletionWithFallback({
userId: "u-primary-auth",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "admin-openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
ledger,
});
} catch (e) {
caught = e;
}
const e = caught as AllProvidersFailedError;
expect(e.primaryCredentialExpired).toBe(true);
});
it("does NOT flag primaryCredentialExpired when the first hop is a 5xx", async () => {
const ledger = createInMemoryProviderHealthLedger();
const codex = new ScriptedProvider({
type: "codex",
script: [{ ok: false, error: err(503) }],
});
const openai = new ScriptedProvider({
type: "admin-key",
script: [{ ok: false, error: err(401) }],
});
let caught: unknown;
try {
await runRawCompletionWithFallback({
userId: "u-primary-5xx",
providers: [
{ providerType: "codex", instance: codex },
{ providerType: "admin-openai", instance: openai },
],
params: { systemPrompt: "s", userPrompt: "u" },
ledger,
});
} catch (e) {
caught = e;
}
const e = caught as AllProvidersFailedError;
expect(e.primaryCredentialExpired).toBe(false);
});
});