mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 12:32:43 +00:00
feat(ai): a tri-state health of the instance's shared provider, fail closed
The done screen promised working insights on the strength of a presence read. A fresh account's own provider-health ledger is empty by construction (the table is unique on user + provider), so the evidence has to be an instance-wide fold. It folds across accounts and returns three words: no counts, no timestamps, no account can be inferred from it. Fails closed against the grain of the ledger it reads. There the fail-open protects a generation already running; here the output is a promise made to a person, so an empty table, a timeout or a read error is 'unknown', and 'unknown' buys nothing.
This commit is contained in:
1 parent
274923a060
commit
82b7c11b05
2 files changed
+249
No files matched your search
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* v1.38.19 (wave D) — the instance-wide health of the operator's shared
|
||||
* provider, as a tri-state a FRESH account is allowed to read.
|
||||
*
|
||||
* The ledger itself is per-user and fails OPEN (an optimisation, never a
|
||||
* gate). This projection is the opposite on both counts: it folds rows
|
||||
* across every account into one instance-level verdict, and it fails
|
||||
* CLOSED — anything but a demonstrated recent success is not an offer.
|
||||
*
|
||||
* What it must never leak: how many accounts are tracked, when any of them
|
||||
* last succeeded, or which account a row belongs to. The return type is the
|
||||
* whole contract.
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
providerHealth: { findMany: vi.fn() },
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({
|
||||
annotate: vi.fn(),
|
||||
}));
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
SERVER_PROVIDER_FRESH_WINDOW_MS,
|
||||
readServerProviderHealth,
|
||||
} from "../server-provider-health";
|
||||
|
||||
const NOW = new Date("2026-09-11T12:00:00.000Z");
|
||||
|
||||
function ago(ms: number): Date {
|
||||
return new Date(NOW.getTime() - ms);
|
||||
}
|
||||
|
||||
function ahead(ms: number): Date {
|
||||
return new Date(NOW.getTime() + ms);
|
||||
}
|
||||
|
||||
function rows(
|
||||
...list: Array<{
|
||||
lastResult: string;
|
||||
lastOkAt?: Date | null;
|
||||
nextRetryAt?: Date | null;
|
||||
}>
|
||||
) {
|
||||
vi.mocked(prisma.providerHealth.findMany).mockResolvedValue(
|
||||
list.map((row) => ({
|
||||
lastResult: row.lastResult,
|
||||
lastOkAt: row.lastOkAt ?? null,
|
||||
nextRetryAt: row.nextRetryAt ?? null,
|
||||
})) as never,
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(prisma.providerHealth.findMany).mockReset();
|
||||
vi.mocked(annotate).mockReset();
|
||||
});
|
||||
|
||||
describe("readServerProviderHealth", () => {
|
||||
it("is healthy when some account saw the shared provider work inside the window", async () => {
|
||||
rows({ lastResult: "ok", lastOkAt: ago(60 * 60 * 1000) });
|
||||
await expect(readServerProviderHealth(NOW)).resolves.toBe("healthy");
|
||||
});
|
||||
|
||||
it("is unhealthy when the last success is older than the window", async () => {
|
||||
rows({
|
||||
lastResult: "ok",
|
||||
lastOkAt: ago(SERVER_PROVIDER_FRESH_WINDOW_MS + 60_000),
|
||||
});
|
||||
await expect(readServerProviderHealth(NOW)).resolves.toBe("unhealthy");
|
||||
});
|
||||
|
||||
it("is unhealthy when rows exist but none ever succeeded", async () => {
|
||||
rows({ lastResult: "hard_failed", lastOkAt: null });
|
||||
await expect(readServerProviderHealth(NOW)).resolves.toBe("unhealthy");
|
||||
});
|
||||
|
||||
it("lets a 401 in cooldown beat a fresh ok — a dead key means the same for everyone", async () => {
|
||||
rows(
|
||||
{ lastResult: "ok", lastOkAt: ago(60 * 1000) },
|
||||
{
|
||||
lastResult: "auth_failed",
|
||||
lastOkAt: ago(3 * 60 * 60 * 1000),
|
||||
nextRetryAt: ahead(60 * 60 * 1000),
|
||||
},
|
||||
);
|
||||
await expect(readServerProviderHealth(NOW)).resolves.toBe("unhealthy");
|
||||
});
|
||||
|
||||
it("ignores an auth failure whose cooldown has run out", async () => {
|
||||
rows(
|
||||
{ lastResult: "ok", lastOkAt: ago(60 * 1000) },
|
||||
{
|
||||
lastResult: "auth_failed",
|
||||
lastOkAt: null,
|
||||
nextRetryAt: ago(60 * 1000),
|
||||
},
|
||||
);
|
||||
await expect(readServerProviderHealth(NOW)).resolves.toBe("healthy");
|
||||
});
|
||||
|
||||
it("is unknown when the instance has no rows for either shared provider", async () => {
|
||||
rows();
|
||||
await expect(readServerProviderHealth(NOW)).resolves.toBe("unknown");
|
||||
});
|
||||
|
||||
it("is unknown when the read throws, and says so in the event", async () => {
|
||||
vi.mocked(prisma.providerHealth.findMany).mockRejectedValue(
|
||||
new Error("connection terminated"),
|
||||
);
|
||||
await expect(readServerProviderHealth(NOW)).resolves.toBe("unknown");
|
||||
expect(vi.mocked(annotate)).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: { name: "ai.server_health.read_failed" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("reads only the operator-funded provider tags, and no per-account column", async () => {
|
||||
rows({ lastResult: "ok", lastOkAt: ago(1000) });
|
||||
await readServerProviderHealth(NOW);
|
||||
const args = vi.mocked(prisma.providerHealth.findMany).mock.calls[0]![0]!;
|
||||
expect(args.where).toEqual({
|
||||
providerType: { in: ["admin-openai", "admin-codex"] },
|
||||
});
|
||||
expect(Object.keys(args.select ?? {}).sort()).toEqual([
|
||||
"lastOkAt",
|
||||
"lastResult",
|
||||
"nextRetryAt",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* v1.38.19 (wave D) — "does the shared provider of THIS instance work?",
|
||||
* answered for a user who has never made a single AI call.
|
||||
*
|
||||
* The question comes from the end of the setup flow. The done screen used
|
||||
* to claim "insights work for you right away" on the strength of
|
||||
* `managedBy === "server"`, which is a PRESENCE read: it says the operator
|
||||
* configured a key, never that the key works. On 2026-09-11 the operator's
|
||||
* own instance answered HTTP 500 from its OAuth proxy while the screen kept
|
||||
* making that promise. This module is the evidence the promise needs.
|
||||
*
|
||||
* Why it cannot come from the user's own ledger: `provider_health` is unique
|
||||
* on `(user_id, provider_type)`, and an account walking the wizard has NO
|
||||
* row — for any provider. Its own ledger can only answer "unknown", forever.
|
||||
* So the projection folds rows ACROSS accounts. That is also the reason for
|
||||
* the shape of the return value.
|
||||
*
|
||||
* What leaves this module is a tri-state and nothing else: no counts, no
|
||||
* timestamps, no user ids, not even "how many accounts are tracked". "The
|
||||
* provider of this instance last worked" is a statement about the instance;
|
||||
* "seven accounts used it at 06:42Z" is a statement about other people. The
|
||||
* admin fold (`GET /api/admin/provider-health`) keeps the second kind and
|
||||
* stays behind `requireAdmin()`; this one is readable by any signed-in user
|
||||
* precisely because it carries none of it.
|
||||
*
|
||||
* Fail CLOSED, deliberately against the grain of the ledger it reads. The
|
||||
* ledger fails open (`provider-health-ledger.ts`) because there it protects
|
||||
* a generation already under way — contributing nothing is the safe outcome.
|
||||
* Here the output is a promise made to a person, so a read error, a timeout
|
||||
* or an empty table is `unknown`, and `unknown` buys nothing: the caller
|
||||
* treats it exactly like `unhealthy`.
|
||||
*
|
||||
* No probe. `POST /api/ai/test` exists and would be the direct answer, but it
|
||||
* is a real completion billed to whoever the chain resolves to — usually the
|
||||
* operator. A probe here would charge the operator for every registration,
|
||||
* and a sign-up bot would scale that bill. The design spec's rule holds:
|
||||
* nothing in the flow talks to a provider.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
/**
|
||||
* The whole contract. Three words, no payload.
|
||||
*
|
||||
* healthy — the instance's shared provider demonstrably served somebody
|
||||
* inside the freshness window, and no credential is benched.
|
||||
* unhealthy — it has been tried and it is not working.
|
||||
* unknown — nobody has tried it yet on this instance, or the read
|
||||
* failed. Never an offer; see the module note on failing closed.
|
||||
*/
|
||||
export type ServerProviderHealth = "healthy" | "unhealthy" | "unknown";
|
||||
|
||||
/**
|
||||
* How recent a success has to be to still count.
|
||||
*
|
||||
* The ledger's own cooldowns bracket the choice: 6 h for an auth failure,
|
||||
* 5 min for a hard one. A window under those would read `unhealthy` on a
|
||||
* quiet single-person instance simply because nothing ran that hour; a much
|
||||
* larger one would paint a key that died on Monday as green on Thursday.
|
||||
* A day is long enough for an instance with one briefing per morning to hold
|
||||
* the threshold, short enough that a dead key goes grey within a day.
|
||||
*/
|
||||
export const SERVER_PROVIDER_FRESH_WINDOW_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* The provider tags the operator pays for: the instance-wide OpenAI key and
|
||||
* the shared central Codex account. These are exactly the two tags
|
||||
* `consent-guard.ts` gates on, and that is not a coincidence — the offer
|
||||
* this feeds is the consent for these two and for nothing else.
|
||||
*/
|
||||
const SHARED_PROVIDER_TYPES = ["admin-openai", "admin-codex"] as const;
|
||||
|
||||
export async function readServerProviderHealth(
|
||||
now: Date = new Date(),
|
||||
): Promise<ServerProviderHealth> {
|
||||
let rows: Array<{
|
||||
lastResult: string;
|
||||
lastOkAt: Date | null;
|
||||
nextRetryAt: Date | null;
|
||||
}>;
|
||||
try {
|
||||
rows = await prisma.providerHealth.findMany({
|
||||
where: { providerType: { in: [...SHARED_PROVIDER_TYPES] } },
|
||||
// Three columns, none of which identifies an account. The absence of
|
||||
// `userId` here is load-bearing, not tidiness.
|
||||
select: { lastResult: true, lastOkAt: true, nextRetryAt: true },
|
||||
});
|
||||
} catch {
|
||||
annotate({ action: { name: "ai.server_health.read_failed" } });
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
if (rows.length === 0) return "unknown";
|
||||
|
||||
// A benched credential outranks any success. A 401/403 is not one
|
||||
// account's bad luck — the operator's key is one object, and while it is
|
||||
// in its cooldown it is dead for everybody, including whoever is reading
|
||||
// this screen right now.
|
||||
const credentialBenched = rows.some(
|
||||
(row) =>
|
||||
row.lastResult === "auth_failed" &&
|
||||
row.nextRetryAt !== null &&
|
||||
row.nextRetryAt.getTime() > now.getTime(),
|
||||
);
|
||||
if (credentialBenched) return "unhealthy";
|
||||
|
||||
const floor = now.getTime() - SERVER_PROVIDER_FRESH_WINDOW_MS;
|
||||
const workedRecently = rows.some(
|
||||
(row) => row.lastOkAt !== null && row.lastOkAt.getTime() >= floor,
|
||||
);
|
||||
return workedRecently ? "healthy" : "unhealthy";
|
||||
}
|
||||
Reference in new issue
Block a user