Merge branch 'fix/1384-pregenerate-without-provider' into release/v1.47.6

This commit is contained in:
Marc-André Bombeck committed 2026-11-15 10:46:20 +01:00
commit f2cf316ad8
32 files changed
+675 -94

No files matched your search

+15
View File
@@ -47,6 +47,21 @@
default was rejected as in the future, and onset dates could display as the
wrong calendar day in non-UTC time zones. Saving now anchors to the current
time when before local noon and preserves stored timestamps on edit.
- **No more nightly "job still failing" alert over a provider that cannot
answer.** When the only account on an instance still had an AI provider
saved whose key the provider no longer accepted (removing the key from a
Local or OpenAI-compatible provider keeps its address, so it is still
called), or whose provider HealthLog had paused after repeated errors, the
nightly briefing pass counted that account as a failure, and every night
the admins were told the job was broken. A key the provider refuses is now
that account's matter: the run notes it but does not fail over it, unless
it is the instance's own shared key. An account whose providers are all
paused is skipped, since nothing was sent. The Background jobs card on the
admin status page now lists a run that finished with every account
failing, the same runs the alert counts, and the alert and the card say in
words what failed and which provider, instead of quoting the run's
counters. Those counters also stop counting a fallback text as a warmed
assessment. (#1384)
### Security
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Hintergrundjobs",
"description": "Queues, deren Jobs nach allen Wiederholungen aufgegeben haben.",
"description": "Queues, deren Jobs nach allen Wiederholungen aufgaben oder deren Lauf bei jedem versuchten Konto scheiterte.",
"none": "In den letzten {hours} Stunden ist kein Job fehlgeschlagen.",
"unavailable": "Keine Job-Queue lesbar. Dieses Deployment läuft ohne Hintergrund-Worker.",
"countOne": "{count} Fehlschlag",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Background jobs",
"description": "Queues whose jobs gave up after exhausting their retries.",
"description": "Queues whose jobs gave up after their retries, or whose run ended with every account it tried failing.",
"none": "No job has failed in the last {hours} hours.",
"unavailable": "No job queue to read. This deployment runs without the background worker.",
"countOne": "{count} failure",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Trabajos en segundo plano",
"description": "Colas cuyos trabajos se rindieron tras agotar sus reintentos.",
"description": "Colas cuyos trabajos se rindieron tras sus reintentos o cuya ejecución falló en cada cuenta intentada.",
"none": "Ningún trabajo ha fallado en las últimas {hours} horas.",
"unavailable": "No hay ninguna cola que consultar: este despliegue funciona sin el worker en segundo plano.",
"countOne": "{count} fallo",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Tâches en arrière-plan",
"description": "Files dont les tâches ont abandonné après avoir épuisé leurs tentatives.",
"description": "Files dont les tâches ont abandonné après leurs tentatives, ou dont l’exécution a échoué pour chaque compte.",
"none": "Aucune tâche n’a échoué au cours des {hours} dernières heures.",
"unavailable": "Aucune file à consulter : ce déploiement fonctionne sans le worker en arrière-plan.",
"countOne": "{count} échec",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Processi in background",
"description": "Code i cui processi hanno rinunciato dopo aver esaurito i tentativi.",
"description": "Code i cui processi hanno rinunciato dopo i tentativi o la cui esecuzione è fallita per ogni account.",
"none": "Nessun processo è fallito nelle ultime {hours} ore.",
"unavailable": "Nessuna coda da leggere: questo deployment funziona senza il worker in background.",
"countOne": "{count} errore",
+1 -1
View File
@@ -8743,7 +8743,7 @@
},
"failingJobs": {
"title": "백그라운드 작업",
"description": "재시도를 다 쓰고 포기한 작업이 있는 큐예요.",
"description": "재시도를 다 쓰고 포기했거나, 실행에서 시도한 모든 계정이 실패한 큐예요.",
"none": "최근 {hours}시간 동안 실패한 작업이 없어요.",
"unavailable": "읽을 작업 큐가 없어요. 이 배포본은 백그라운드 워커 없이 돌고 있어요.",
"countOne": "실패 {count}건",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Achtergrondtaken",
"description": "Wachtrijen waarvan taken het hebben opgegeven nadat alle nieuwe pogingen waren verbruikt.",
"description": "Wachtrijen waarvan taken na alle pogingen opgaven, of waarvan een run voor elk geprobeerd account mislukte.",
"none": "Er is de afgelopen {hours} uur geen taak mislukt.",
"unavailable": "Geen takenwachtrij om te lezen. Deze installatie draait zonder de achtergrondworker.",
"countOne": "{count} fout",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Zadania w tle",
"description": "Kolejki, których zadania poddały się po wyczerpaniu ponowień.",
"description": "Kolejki, których zadania poddały się po ponowieniach albo których przebieg zawiódł dla każdego konta.",
"none": "W ciągu ostatnich {hours} godzin żadne zadanie nie zawiodło.",
"unavailable": "Brak kolejki zadań do odczytu: to wdrożenie działa bez workera w tle.",
"countOne": "{count} niepowodzenie",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Tarefas em segundo plano",
"description": "Filas cujas tarefas desistiram depois de esgotar as tentativas.",
"description": "Filas cujas tarefas desistiram após as tentativas, ou cuja execução falhou em todas as contas tentadas.",
"none": "Nenhuma tarefa falhou nas últimas {hours} horas.",
"unavailable": "Nenhuma fila de tarefas para ler. Esta implantação roda sem o worker em segundo plano.",
"countOne": "{count} falha",
+1 -1
View File
@@ -8768,7 +8768,7 @@
},
"failingJobs": {
"title": "Фоновые задания",
"description": "Очереди, задания которых сдались после исчерпания повторных попыток.",
"description": "Очереди, задания которых сдались после всех попыток или чей запуск не удался ни для одного аккаунта.",
"none": "За последние {hours} часов ни одно задание не завершилось неудачей.",
"unavailable": "Нет очереди заданий для чтения. Это развёртывание работает без фонового воркера.",
"countOne": "{count} неудача",
+1 -1
View File
@@ -435,7 +435,7 @@
"lib/jobs/__tests__/document-summary-reaper.test.ts": 1,
"lib/jobs/__tests__/document-summary.test.ts": 19,
"lib/jobs/__tests__/geo-backfill.test.ts": 1,
"lib/jobs/__tests__/insight-pregenerate.test.ts": 45,
"lib/jobs/__tests__/insight-pregenerate.test.ts": 43,
"lib/jobs/__tests__/insight-status-generate.test.ts": 2,
"lib/jobs/__tests__/measurement-maintenance-schedule.test.ts": 2,
"lib/jobs/__tests__/measurement-reminder-byhour.test.ts": 2,
@@ -1,7 +1,7 @@
{
"$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.",
"count": 2863,
"total": 9122,
"total": 9118,
"entries": {
"__tests__/account-payload-consumer-guard.test.ts": 2,
"__tests__/acting-account-boundary-guard.test.ts": 7,
@@ -2134,7 +2134,7 @@
"lib/insights/chart-tokens.ts": 24,
"lib/insights/coach-launch-context.tsx": 10,
"lib/insights/comprehensive-aggregator.ts": 27,
"lib/insights/comprehensive-generate.ts": 45,
"lib/insights/comprehensive-generate.ts": 44,
"lib/insights/correlation-channel-series.ts": 16,
"lib/insights/correlation-discovery.ts": 30,
"lib/insights/correlation-series-builders.ts": 1,
@@ -2299,7 +2299,7 @@
"lib/jobs/google-health-oauth-state-cleanup.ts": 1,
"lib/jobs/google-health-sleep-repair.ts": 1,
"lib/jobs/insight-pregenerate-shared.ts": 1,
"lib/jobs/insight-pregenerate.ts": 36,
"lib/jobs/insight-pregenerate.ts": 33,
"lib/jobs/insight-status-generate-shared.ts": 3,
"lib/jobs/insight-status-generate.ts": 3,
"lib/jobs/intake-auto-skip.ts": 8,
@@ -168,6 +168,11 @@ const ALLOWED: Readonly<Record<string, { count: number; reason: string }>> = {
reason:
"No pgboss schema on a web-only deployment; null means no queue to ask, and the evaluation then judges nothing.",
},
"lib/jobs/job-failures.ts::explainAllFailedRun": {
count: 1,
reason:
"A pg-boss output that does not parse as a run's outcome is not an all-failed run; presentError then shows the raw text as it always did.",
},
"lib/jobs/job-failures.ts::readActiveJobsStartedBefore": {
count: 1,
reason:
@@ -209,4 +209,59 @@ describe("comprehensive warm-path timeout + failure observability", () => {
(failAnnotation![0] as { meta?: { reason?: string } }).meta?.reason,
).toBe("all-providers-failed");
});
// #1384 — the nightly pass needs to know which configured provider failed,
// whose credential a refusal was, and whether anything was sent at all.
it("names the provider that failed first and whether its credential was refused", async () => {
runRawCompletionWithFallback.mockRejectedValue(
Object.assign(new AllProvidersFailedError([]), {
attempts: [
{
providerType: "openai-compatible",
attempt: 1,
failureReason: "401",
httpStatus: 401,
},
],
primaryCredentialExpired: true,
sentNothing: false,
}),
);
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
expect(outcome).toEqual({
status: "failed",
reason: "all-providers-failed",
credentialExpired: true,
provider: "openai-compatible",
});
});
it("says so when every provider was benched and nothing was sent (#1279)", async () => {
runRawCompletionWithFallback.mockRejectedValue(
Object.assign(new AllProvidersFailedError([]), {
attempts: [
{
providerType: "local",
attempt: 1,
failureReason: "benched-for-background-work",
httpStatus: null,
failureKind: "paused",
},
],
primaryCredentialExpired: false,
sentNothing: true,
}),
);
const outcome = await generateComprehensiveInsight("u1", { locale: "de" });
expect(outcome).toEqual({
status: "failed",
reason: "all-providers-failed",
provider: "local",
sentNothing: true,
});
});
});
@@ -134,6 +134,7 @@ describe("generateStatusBatchForUser — the statusText capability comes first",
served: 0,
batched: 0,
fellBack: 0,
fellBackAnswered: 0,
batchCallMade: false,
});
});
@@ -340,6 +341,15 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
vi.mocked(runStatusCompletion).mockResolvedValue({
kind: "error",
} as never);
// #1384 — one card's own call came back with a model note; the other six
// served their fallback. A nightly warm over a provider that only failed
// reported `assessments_warmed: 6`, because every fallback counted.
vi.mocked(runPreparedStatusCard).mockResolvedValueOnce({
hasProvider: true,
text: "note",
cached: false,
updatedAt: "t",
});
const result = await generateStatusBatchForUser("u1", { locale: "en" });
@@ -347,6 +357,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
for (const f of finalizers) expect(f).not.toHaveBeenCalled();
expect(runPreparedStatusCard).toHaveBeenCalledTimes(7);
expect(result.fellBack).toBe(7);
expect(result.fellBackAnswered).toBe(1);
expect(result.batched).toBe(0);
});
@@ -411,6 +422,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => {
for (const f of finalizers) expect(f).not.toHaveBeenCalled();
expect(runPreparedStatusCard).not.toHaveBeenCalled();
expect(result.fellBack).toBe(7);
expect(result.fellBackAnswered).toBe(0);
expect(result.batchCallMade).toBe(false);
});
});
+37
View File
@@ -0,0 +1,37 @@
/**
* What a failed briefing generation knows about its provider chain beyond
* the reason (#1384): which configured provider failed first, whether that
* provider refused the credential, and whether anything was sent at all.
*
* The nightly pass needs all three to settle an account honestly. A refusal
* of the account's own credential is the account's matter, a refusal of the
* operator's shared key is the operator's, and a chain whose every entry the
* provider health ledger benched for background work (#1279) sent no request,
* so the account was not attempted. Without them every such account read as
* a failure, night after night.
*/
import { AllProvidersFailedError } from "@/lib/ai/provider-runner";
export interface ChainFailureFacts {
/**
* The chain's first provider refused the credential (a dead Codex
* sign-in, a revoked key). No retry can succeed until someone reconnects
* or replaces it.
*/
credentialExpired?: boolean;
/** The chain entry that failed first, e.g. `openai-compatible`. */
provider?: string;
/** Every entry was benched for background work; no request left. */
sentNothing?: boolean;
}
/** The facts of a thrown chain failure; empty for any other error. */
export function chainFailureFacts(error: unknown): ChainFailureFacts {
if (!(error instanceof AllProvidersFailedError)) return {};
const provider = error.attempts?.[0]?.providerType;
return {
...(error.primaryCredentialExpired ? { credentialExpired: true } : {}),
...(provider ? { provider } : {}),
...(error.sentNothing ? { sentNothing: true } : {}),
};
}
+6 -16
View File
@@ -93,6 +93,10 @@ import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic";
import { insightResultSchema, type InsightResult } from "@/lib/ai/types";
import { resolveProviderChain } from "@/lib/ai/provider";
import { AllProvidersFailedError } from "@/lib/ai/provider-runner";
import {
chainFailureFacts,
type ChainFailureFacts,
} from "@/lib/insights/chain-failure-facts";
import {
BriefingBudgetExceededError,
runBriefingCompletion,
@@ -178,17 +182,7 @@ export type GenerateOutcome =
| "scope-changed"
| "profile-scope-changed";
}
| {
status: "failed";
reason: string;
/**
* v1.42 — the chain's first provider refused the credential (a dead
* Codex sign-in, a revoked key). The nightly pass reports the account
* as `auth_failed` and does not queue a retry that cannot succeed until
* the person reconnects.
*/
credentialExpired?: boolean;
};
| ({ status: "failed"; reason: string } & ChainFailureFacts);
/** Higher, seedless temperature for the daily-briefing phrasing re-roll. */
const BRIEFING_REROLL_TEMPERATURE = 0.6;
@@ -1290,11 +1284,7 @@ export async function generateComprehensiveInsight(
? (e.attempts?.[0]?.failureKind ?? null)
: providerFailureKind(e).failure,
});
const credentialExpired =
e instanceof AllProvidersFailedError && e.primaryCredentialExpired;
return credentialExpired
? { status: "failed", reason, credentialExpired }
: { status: "failed", reason };
return { status: "failed", reason, ...chainFailureFacts(e) };
}
// Anthropic + local have no native JSON mode, so a ```json-fenced or
+39 -6
View File
@@ -37,6 +37,7 @@ import {
runPreparedStatusCard,
type PendingStatusCard,
type PreparedStatusCard,
type StatusCardResult,
} from "@/lib/insights/status-card-generation";
import { runStatusCompletion } from "@/lib/insights/status-provider";
import {
@@ -77,10 +78,27 @@ export interface StatusBatchResult {
/** Metrics that fell back to a single-card call (omitted by the batch, or
* the batch failed). */
fellBack: number;
/**
* Of `fellBack`, the metrics whose own single-card call came back with a
* model-written assessment. The rest served their deterministic fallback:
* no provider, a failed or cooling-down provider, or a screened reply.
* `batched + fellBackAnswered` is what a warm pass may call warmed (#1384);
* `fellBack` alone counted a night of six failed calls as six assessments.
*/
fellBackAnswered: number;
/** True when the one batched provider call was issued. */
batchCallMade: boolean;
}
/**
* Whether a single-card run ended in a model-written assessment: the persisted
* shape is the only one with a provider and no cache flag. The no-key, the
* timeout / error stub and the screened fallback all carry `cached: true`.
*/
function modelAnswered(result: StatusCardResult): boolean {
return result.hasProvider === true && result.cached === false;
}
/** Pull `perMetric` out of a (possibly fenced) model completion. */
function parsePerMetric(content: string): Record<string, string> | null {
const tryParse = (raw: string): Record<string, string> | null => {
@@ -119,6 +137,7 @@ export async function generateStatusBatchForUser(
served: 0,
batched: 0,
fellBack: 0,
fellBackAnswered: 0,
batchCallMade: false,
};
@@ -171,11 +190,19 @@ export async function generateStatusBatchForUser(
// One metric still needing the LLM is not worth a batch envelope — run it
// through its own single-card path (identical persisted result).
if (pending.length === 1) {
await runPreparedStatusCard(pending[0]);
if (modelAnswered(await runPreparedStatusCard(pending[0]))) {
result.fellBackAnswered++;
}
result.fellBack++;
annotate({
action: { name: "insights.status.batch" },
meta: { served: result.served, batched: 0, fell_back: 1, single: true },
meta: {
served: result.served,
batched: 0,
fell_back: 1,
fell_back_answered: result.fellBackAnswered,
single: true,
},
});
return result;
}
@@ -272,14 +299,19 @@ export async function generateStatusBatchForUser(
// Batch missed this metric (omitted key, or timeout/error/invalid-json):
// fall back to the single-card path. On a batch timeout/error this writes
// the per-card negative stub via the card's own `timeout`.
await runPreparedStatusCard(card);
return "fellBack" as const;
return modelAnswered(await runPreparedStatusCard(card))
? ("fellBackAnswered" as const)
: ("fellBack" as const);
});
const outcomes = await Promise.all(fanOuts);
for (const o of outcomes) {
if (o === "batched") result.batched++;
else result.fellBack++;
if (o === "batched") {
result.batched++;
continue;
}
result.fellBack++;
if (o === "fellBackAnswered") result.fellBackAnswered++;
}
annotate({
@@ -288,6 +320,7 @@ export async function generateStatusBatchForUser(
served: result.served,
batched: result.batched,
fell_back: result.fellBack,
fell_back_answered: result.fellBackAnswered,
pending: pending.length,
},
});
@@ -103,6 +103,7 @@ vi.mock("@/lib/insights/status-batch", () => ({
served: 0,
batched: 0,
fellBack: 0,
fellBackAnswered: 0,
batchCallMade: false,
})),
}));
@@ -132,6 +133,7 @@ import {
INSIGHT_PREGENERATE_EXPIRE_SECONDS,
} from "../insight-pregenerate-shared";
import { handleInsightPregenerateJob } from "../reminder/insights-handlers";
import { judgeJobRun, type reportJobRun } from "../job-run-report";
import { insightRetryOptions } from "../reminder/registrar-shared";
import {
groupOf,
@@ -1556,15 +1558,9 @@ describe("runInsightPregenerate — warm budget honours the response timeout (v1
// withheld (screened) briefing and a dead sign-in queue no provider retry,
// and a time-budget cut-off reports the unreached accounts as deferred.
describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
type Reported = {
queue: string;
runId: string;
candidates: Array<{ key: string; outcome: string; cause?: string }>;
};
function harness(users: Array<{ id: string; locale: string }>) {
const { prisma } = makePrisma(users);
const reportRun = vi.fn(async (input: Reported) => ({
const reportRun = vi.fn<typeof reportJobRun>(async (input) => ({
total: input.candidates.length,
attempted: 0,
failed: 0,
@@ -1606,7 +1602,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
statusGenerators: h.statusGenerators,
warmGenericMetrics: h.warmGenericMetrics,
enqueueRetry: h.enqueueRetry,
reportRun: h.reportRun as never,
reportRun: h.reportRun,
});
expect(h.reportRun).toHaveBeenCalledTimes(1);
@@ -1639,7 +1635,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
statusGenerators: h.statusGenerators,
warmGenericMetrics: h.warmGenericMetrics,
enqueueRetry: h.enqueueRetry,
reportRun: h.reportRun as never,
reportRun: h.reportRun,
});
expect(h.enqueueRetry).toHaveBeenCalledTimes(1);
@@ -1668,7 +1664,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
statusGenerators: h.statusGenerators,
warmGenericMetrics: h.warmGenericMetrics,
enqueueRetry: h.enqueueRetry,
reportRun: h.reportRun as never,
reportRun: h.reportRun,
shouldStop: () => ++calls > 1,
});
@@ -1677,3 +1673,117 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => {
).toEqual(["ok", "deferred", "deferred"]);
});
});
// #1384 — a self-hoster whose only account kept a provider that no longer
// works was told every night that the job was failing. The run used the real
// verdict here, so what the alert reads (`all_failed`) is what is asserted.
describe("runInsightPregenerate — a provider that cannot serve is not a failing run (#1384)", () => {
async function runOne(outcome: Record<string, unknown>) {
const { prisma } = makePrisma([{ id: "only-account", locale: "en" }]);
const reportRun = vi.fn<typeof reportJobRun>(async (input) =>
judgeJobRun(input.candidates),
);
const enqueueRetry = vi.fn().mockResolvedValue(undefined);
const result = await runInsightPregenerate(prisma as never, {
generate: vi.fn().mockResolvedValue(outcome),
statusGenerators: Array.from({ length: 7 }, () =>
vi.fn().mockResolvedValue({ hasProvider: true, cached: true }),
),
warmGenericMetrics: vi.fn().mockResolvedValue(0),
enqueueRetry,
reportRun,
});
return {
result,
enqueueRetry,
candidate: reportRun.mock.calls[0][0].candidates[0],
};
}
it("skips an account whose every provider was benched, so nothing was sent", async () => {
const { result, enqueueRetry, candidate } = await runOne({
status: "failed",
reason: "all-providers-failed",
provider: "local",
sentNothing: true,
});
expect(candidate).toEqual(
expect.objectContaining({
outcome: "skipped",
cause: "provider_benched",
}),
);
expect(result.skipped).toBe(1);
expect(result.failed).toBe(0);
expect(result.allFailed).toBe(false);
expect(result.firstFailure).toBeNull();
// The retry would find the same bench.
expect(enqueueRetry).not.toHaveBeenCalled();
expect(annotateSpy).toHaveBeenCalledWith(
expect.objectContaining({
action: { name: "insights.pregenerate.provider_benched" },
}),
);
});
it("reports a refused credential of the account's own as auth_failed, naming the provider, without failing the run", async () => {
const { result, enqueueRetry, candidate } = await runOne({
status: "failed",
reason: "all-providers-failed",
provider: "openai-compatible",
credentialExpired: true,
});
expect(candidate).toEqual(
expect.objectContaining({
outcome: "auth_failed",
cause: "all-providers-failed:openai-compatible",
}),
);
expect(result.allFailed).toBe(false);
expect(enqueueRetry).not.toHaveBeenCalled();
expect(annotateSpy).toHaveBeenCalledWith(
expect.objectContaining({
action: { name: "insights.pregenerate.comprehensive_failed" },
meta: expect.objectContaining({
provider: "openai-compatible",
credential_refused: true,
}),
}),
);
});
it("still fails the run when the operator's own key was refused, and says which", async () => {
const { result, enqueueRetry, candidate } = await runOne({
status: "failed",
reason: "all-providers-failed",
provider: "admin-openai",
credentialExpired: true,
});
expect(candidate.outcome).toBe("failed");
expect(result.allFailed).toBe(true);
expect(result.firstFailure).toEqual({
reason: "all-providers-failed",
provider: "admin-openai",
});
// A refused key does not heal in 45 minutes.
expect(enqueueRetry).not.toHaveBeenCalled();
});
it("carries the cause and provider of an ordinary failure into the run's facts", async () => {
const { result, enqueueRetry } = await runOne({
status: "failed",
reason: "provider-error",
provider: "local",
});
expect(result.allFailed).toBe(true);
expect(result.firstFailure).toEqual({
reason: "provider-error",
provider: "local",
});
expect(enqueueRetry).toHaveBeenCalledTimes(1);
});
});
@@ -193,3 +193,36 @@ describe("bounded generic JobOutcome serialization", () => {
});
});
});
// #1384 — a generator reason and a provider type become facts the admin page
// and the repeated-failure alert can show.
describe("codeFact", () => {
it("turns our own hyphenated codes into stable fact codes", async () => {
const { codeFact, serializeJobOutcome } = await vi.importActual<
typeof import("@/lib/jobs/job-outcome")
>("@/lib/jobs/job-outcome");
expect(codeFact("all-providers-failed")).toBe("all_providers_failed");
expect(codeFact("openai-compatible")).toBe("openai_compatible");
expect(codeFact("generator:invalid-json")).toBe("generator_invalid_json");
expect(codeFact("timeout")).toBe("timeout");
// Nothing usable: the caller leaves the fact out.
expect(codeFact("")).toBeNull();
expect(codeFact("---")).toBeNull();
expect(codeFact(null)).toBeNull();
// What it produces is a fact the serializer accepts.
expect(
serializeJobOutcome(
jobDone({
failure_cause: codeFact("all-providers-failed")!,
failure_provider: codeFact("admin-openai")!,
}),
),
).toEqual({
ok: true,
did: {
failure_cause: "all_providers_failed",
failure_provider: "admin_openai",
},
});
});
});
+47 -9
View File
@@ -47,13 +47,27 @@ describe("judgeJobRun", () => {
});
});
it("counts a dead credential as a failure", () => {
expect(
judgeJobRun([
{ key: "a", outcome: "auth_failed" },
{ key: "b", outcome: "failed" },
]).allFailed,
).toBe(true);
// #1384 — a key the issuer revoked stays saved and is refused every night.
// Counted as a failure, it made a single-account instance's nightly pass
// read as all-failed forever and alerted the operator about the job.
it("does not fail a run over a refused credential of the account's own", () => {
expect(judgeJobRun([{ key: "a", outcome: "auth_failed" }])).toEqual({
total: 1,
attempted: 1,
failed: 0,
allFailed: false,
partial: true,
});
});
it("reads a refused credential beside a real failure as partial", () => {
const verdict = judgeJobRun([
{ key: "a", outcome: "auth_failed" },
{ key: "b", outcome: "failed" },
]);
expect(verdict.allFailed).toBe(false);
expect(verdict.partial).toBe(true);
expect(verdict.failed).toBe(1);
});
it("reads one success among failures as partial, not failed", () => {
@@ -84,7 +98,7 @@ describe("reportJobRun", () => {
queue: "insight-pregenerate",
runId: "r1",
candidates: [
{ key: "a", outcome: "auth_failed", cause: "all-providers-failed" },
{ key: "a", outcome: "failed", cause: "all-providers-failed" },
{ key: "b", outcome: "skipped", cause: "no-consent" },
],
});
@@ -101,7 +115,31 @@ describe("reportJobRun", () => {
consecutiveFailures: 3,
failed: 1,
total: 1,
causes: ["auth_failed:all-providers-failed"],
causes: ["failed:all-providers-failed"],
},
});
});
it("emits job.run.partial, never job.run.failed, for a run whose only attempt was a refused credential (#1384)", async () => {
const verdict = await reportJobRun({
queue: "insight-pregenerate",
runId: "r1",
candidates: [
{
key: "a",
outcome: "auth_failed",
cause: "all-providers-failed:openai-compatible",
},
],
});
expect(verdict.allFailed).toBe(false);
expect(readConsecutiveRunFailures).not.toHaveBeenCalled();
expect(emitSignal).toHaveBeenCalledTimes(1);
expect(emitSignal.mock.calls[0][0]).toMatchObject({
action: "job.run.partial",
level: "warn",
meta: {
causes: ["auth_failed:all-providers-failed:openai-compatible"],
},
});
});
@@ -87,6 +87,34 @@ describe("judgeQueueRuns", () => {
).toBe("failing");
});
// #1384 — the alert quoted the completed run's own outcome JSON, and the
// operator it sent to the admin page found nothing there.
it("says in words what an all-failed night was, with its cause and provider", () => {
const output = JSON.stringify({
ok: true,
did: {
total: 1,
failed: 1,
all_failed: true,
assessments_warmed: 0,
failure_cause: "all_providers_failed",
failure_provider: "admin_openai",
},
});
const verdict = judgeQueueRuns(
[0, 1, 2].map((ago) =>
run(ago, { state: "completed", allFailed: true, error: output }),
),
);
expect(verdict.state).toBe("failing");
expect(verdict.lastError).toBe(
"The run completed, but every account it attempted failed (1 of 1). " +
"First cause: all_providers_failed from provider admin_openai. " +
"Accounts without a usable provider are skipped and not counted; " +
"job.run.failed in the app log lists each account's outcome.",
);
});
it("explains a bare timeout the way the admin card does", () => {
const verdict = judgeQueueRuns(
[0, 1, 2].map((ago) =>
@@ -0,0 +1,93 @@
/**
* How the nightly briefing pass settles an account whose generation came back
* `failed` (#1384).
*
* A failure used to be one of three things: screened, a dead sign-in, or a
* provider failure worth a retry in 45 minutes. Two cases were missing, and
* both made a self-hoster's only account read as a failing run every night,
* which the repeated-failure alert then reported as a broken job:
*
* - Every provider of the chain was benched for background work by the
* provider health ledger (#1279), so no request was sent. The account was
* not attempted tonight; its earlier failures were counted by the runs
* that made them, and a retry would find the same bench. It is `skipped`.
* - The provider refused the credential. A refused credential does not heal
* in 45 minutes, so no retry is queued, and whose credential it was
* decides who has to act. The operator's shared key or central sign-in is
* the operator's to fix, and a run where it was refused for everyone is a
* failing run: `failed`. The account's own key, gateway or sign-in is its
* owner's to fix: `auth_failed`, which the run reports but does not fail
* over (`job-run-report.ts`). An API key revoked at its issuer stays saved
* and is refused again every night, unlike a ChatGPT sign-in, which the
* refusal parks.
*
* The model answered and the safety screen withheld it (`outbound-screened`)
* stays `screened` with no retry: asking the same model again in 45 minutes
* is the same question, and it was screened eight times a night.
*/
import { isOperatorHeldProvider } from "@/lib/ai/provider-egress";
import type { GenerateOutcome } from "@/lib/insights/comprehensive-generate";
import type { JobRunCandidate, JobRunOutcome } from "@/lib/jobs/job-run-report";
type FailedGeneration = Extract<GenerateOutcome, { status: "failed" }>;
export interface FailedGenerationVerdict {
outcome: JobRunOutcome;
/** The run line's cause: the reason, and the provider that failed first. */
cause: string;
/** False when nothing was sent: the account is tallied as skipped. */
attempted: boolean;
/** Queue the bounded 45-minute retry for this account. */
retry: boolean;
}
export function settleFailedGeneration(
generated: FailedGeneration,
): FailedGenerationVerdict {
if (generated.sentNothing) {
return {
outcome: "skipped",
cause: "provider_benched",
attempted: false,
retry: false,
};
}
const cause = generated.provider
? `${generated.reason}:${generated.provider}`
: generated.reason;
if (generated.reason === "outbound-screened") {
return { outcome: "screened", cause, attempted: true, retry: false };
}
if (generated.credentialExpired) {
const operatorHeld =
generated.provider !== undefined &&
isOperatorHeldProvider(generated.provider);
return {
outcome: operatorHeld ? "failed" : "auth_failed",
cause,
attempted: true,
retry: false,
};
}
return { outcome: "failed", cause, attempted: true, retry: true };
}
/** The first failed account of a run: its reason and the provider, if known. */
export interface FirstFailure {
reason: string;
provider: string | null;
}
/**
* The first failed account's reason and provider, read back off the run's
* causes, for the job's facts: what the admin page and the repeated-failure
* alert show for a run that completed with every account failed.
*/
export function firstFailure(
candidates: readonly JobRunCandidate[],
): FirstFailure | null {
const cause = candidates.find((c) => c.outcome === "failed")?.cause;
if (!cause) return null;
const [reason, provider] = cause.split(":");
return { reason, provider: provider ?? null };
}
+28 -28
View File
@@ -81,6 +81,10 @@ import {
} from "@/lib/insights/metric-status-registry";
import { withTimeout } from "@/lib/insights/with-timeout";
import { comprehensiveWarmBudgetMs } from "@/lib/jobs/model-time";
import {
firstFailure,
settleFailedGeneration,
} from "@/lib/jobs/insight-pregenerate-failure";
import {
INSIGHT_PREGENERATE_QUEUE,
enqueueForceWarm,
@@ -235,6 +239,8 @@ export interface PregenerateRunResult {
* seven specialised cards.
*/
metricAssessmentsWarmed: number;
/** #1384 — the first failed account's reason and provider, as facts. */
firstFailure: ReturnType<typeof firstFailure>;
}
/**
@@ -302,9 +308,9 @@ async function warmPerStatusCaches(
* once and issues ONE provider call for the metrics still needing the LLM,
* fanning the response back into the same per-metric cache rows the
* standalone generators wrote. Replaces the prior seven independent warm
* calls. Returns the count of metrics that got a fresh (batched or
* fallback) assessment so the wide-event tally is comparable to the old
* per-card count.
* calls. Returns the count of metrics that got a fresh model-written
* assessment, batched or through a card's own fallback call, so the
* wide-event tally is comparable to the old per-card count.
*
* Refill-only (`force: false`): a card already generated today and a card
* whose content-hash is unchanged both resolve inside the batch as `served`
@@ -322,7 +328,7 @@ async function warmStatusBatch(
locale,
force: false,
});
warmed += result.batched + result.fellBack;
warmed += result.batched + result.fellBackAnswered;
} catch (err) {
// The batch must never throw the cron's user loop off course — but
// a swallowed batch failure left ~7 cards cold with zero signal.
@@ -555,6 +561,7 @@ export async function runInsightPregenerate(
allFailed: false,
assessmentsWarmed: 0,
metricAssessmentsWarmed: 0,
firstFailure: null,
};
// Master kill-switch — when the operator disabled the briefing
@@ -619,6 +626,7 @@ export async function runInsightPregenerate(
candidates: runOutcomes,
});
result.allFailed = verdict.allFailed;
result.firstFailure = firstFailure(runOutcomes);
return result;
@@ -783,42 +791,34 @@ export async function runInsightPregenerate(
result.skipped++;
settle("skipped", generated.reason);
break;
case "failed":
result.failed++;
// v1.28.30 — THE previously-silent failure path. A generator
// that resolves with `{ status: "failed" }` (features error,
// oversize payload, invalid JSON after retry, abort) bumped
// the tally with no annotation at all — the nightly summary
// showed `failed: 1` and nothing else was greppable. Reuse
// the `comprehensive_failed` shape with the generator's own
// reason as the cause.
case "failed": {
// Benched, screened, a refused credential and whose, or a
// retryable failure: `insight-pregenerate-failure.ts` (#1384).
const verdict = settleFailedGeneration(generated);
if (verdict.attempted) result.failed++;
else result.skipped++;
annotate({
action: { name: "insights.pregenerate.comprehensive_failed" },
action: {
name: verdict.attempted
? "insights.pregenerate.comprehensive_failed"
: "insights.pregenerate.provider_benched",
},
meta: {
locale,
stage: "nightly.generator",
cause: `generator:${generated.reason}`,
provider: generated.provider ?? null,
credential_refused: generated.credentialExpired === true,
budget_ms: warmBudgetMs,
message: null,
},
});
if (generated.reason === "outbound-screened") {
// v1.42 — the model answered and the safety screen withheld
// it, after the generator's own repair. Asking the same model
// again in 45 minutes is not a provider retry; it is the same
// question, and it was screened eight times a night.
settle("screened", generated.reason);
} else if (generated.credentialExpired) {
// v1.42 — a dead sign-in (a Codex refresh token the issuer
// revoked) does not heal in 45 minutes either; the person has
// to reconnect. The refusal parked the connection as expired,
// which Settings shows, and the next run no longer counts it.
settle("auth_failed", generated.reason);
} else {
settle("failed", generated.reason);
settle(verdict.outcome, verdict.cause);
if (verdict.retry) {
await enqueueRetry({ userId: candidate.id, locale });
}
break;
}
}
}
}
+52 -4
View File
@@ -92,7 +92,7 @@ const MAX_ERROR_CHARS = 300;
export interface FailingQueue {
/** The pg-boss queue name, e.g. `environment-fetch`. */
queue: string;
/** Jobs that exhausted their retries inside the window. */
/** Failed runs inside the window: exhausted retries, or all-failed passes. */
failures: number;
/** ISO instant of the newest failure. */
lastFailedAt: string;
@@ -140,10 +140,48 @@ function explainTimeout(message: string, expireSeconds: number | null): string {
);
}
/**
* Say what a completed run that failed every account means (#1384). Such a
* run's `output` is its outcome, `{"ok":true,"did":{…,"all_failed":true}}`,
* and quoting that JSON told the operator nothing: not that the job ran, not
* which provider failed. Null when `raw` is not such an outcome.
*/
function explainAllFailedRun(raw: string): string | null {
if (!raw.startsWith("{")) return null;
let did: Record<string, unknown>;
try {
const parsed = JSON.parse(raw) as { ok?: unknown; did?: unknown };
if (parsed?.ok !== true || typeof parsed.did !== "object") return null;
did = (parsed.did ?? {}) as Record<string, unknown>;
} catch {
return null;
}
if (did.all_failed !== true) return null;
// Every attempted account failed, so the count is both sides of "of".
const failed =
typeof did.failed === "number" && did.failed > 0
? ` (${did.failed} of ${did.failed})`
: "";
// "none" is how the run says it has no cause or provider to name.
const named = (value: unknown): string | null =>
typeof value === "string" && value !== "none" ? value : null;
const cause = named(did.failure_cause);
const provider = named(did.failure_provider);
const why = cause
? ` First cause: ${cause}${provider ? ` from provider ${provider}` : ""}.`
: "";
return (
`The run completed, but every account it attempted failed${failed}.${why} ` +
`Accounts without a usable provider are skipped and not counted; ` +
`job.run.failed in the app log lists each account's outcome.`
);
}
/**
* Turn whatever pg-boss stored in `output` into one line. A thrown `Error`
* serialises to `{ message, stack, … }`; a thrown non-Error can be any JSON at
* all, so the raw text is the fallback rather than an invented placeholder.
* A completed run that reported every account failed is said in words.
*/
export function presentError(
raw: string | null,
@@ -152,6 +190,8 @@ export function presentError(
if (raw === null) return "";
const trimmed = raw.trim();
if (trimmed.length === 0) return "";
const allFailed = explainAllFailedRun(trimmed);
if (allFailed !== null) return allFailed;
if (TIMEOUT_MESSAGE.test(trimmed))
return explainTimeout(trimmed, expireSeconds);
if (HEARTBEAT_MESSAGE.test(trimmed)) {
@@ -168,8 +208,13 @@ export function presentError(
}
/**
* Every queue with at least one exhausted-retry failure in the window, newest
* first. `null` when the queue schema is not present at all.
* Every queue with at least one failed run in the window, newest first:
* a job that exhausted its retries, or a fan-out pass that completed but
* reported that every account it attempted failed (`did.all_failed`). The
* second kind is what the repeated-failure alert counts too
* (`readConsecutiveRunFailures`, `system-job-alert.ts`); the page used to
* list only the first, so an operator the alert sent here read "no job has
* failed" (#1384). `null` when the queue schema is not present at all.
*/
export async function readFailingQueues(
withinHours: number = JOB_FAILURE_WINDOW_HOURS,
@@ -194,7 +239,10 @@ export async function readFailingQueues(
array_agg(expire_seconds ORDER BY completed_on DESC NULLS LAST)
)[1] AS last_expire_seconds
FROM pgboss.job
WHERE state = 'failed'
WHERE (
state = 'failed'
OR (state = 'completed' AND output->'did'->>'all_failed' = 'true')
)
AND completed_on > now() - make_interval(hours => ${withinHours}::int)
GROUP BY name
ORDER BY max(completed_on) DESC
+21
View File
@@ -130,6 +130,11 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"duration_ms",
"errored",
"failed",
// #1384 — a fan-out pass's first failed candidate: its cause and the
// provider that failed for it, as stable codes. What the admin page and
// the repeated-failure alert show for a run that completed all-failed.
"failure_cause",
"failure_provider",
"failures",
"feedback_buckets",
"feedback_total_rows",
@@ -423,6 +428,22 @@ export function percentFact(ratio: number | null | undefined): number {
return wholeFact(ratio * 100);
}
/**
* A short code from our own vocabulary (a generator reason, a provider type)
* as a fact: `all-providers-failed` becomes `all_providers_failed`. Null when
* nothing of it survives, so the caller leaves the fact out.
*/
export function codeFact(value: string | null | undefined): string | null {
if (value == null) return null;
const code = value
.toLowerCase()
.replace(/[^a-z0-9]+/g, "_")
.replace(/^[^a-z]+/, "")
.slice(0, 64)
.replace(/_+$/, "");
return STABLE_CODE_PATTERN.test(code) ? code : null;
}
/** Why one fact is refused, or null when it is a valid fact. */
function invalidFact(key: string, value: unknown): string | null {
if (!JOB_FACT_ALLOWLIST.has(key))
+25 -5
View File
@@ -21,6 +21,18 @@
* attempted but is not a provider failure, so a run of only screened
* candidates is partial, not failed.
*
* `auth_failed` is attempted and is not a failure of the run either (#1384).
* It means the provider refused the account's own credential: a key revoked
* at its issuer, a gateway that now wants a key the account no longer
* stores. Only that account's owner can change it, and the run itself worked.
* It used to count as a failure on the assumption that a refusal is reported
* once and then parked; that holds for a ChatGPT sign-in and for nothing else,
* so an API key the issuer had revoked failed every night for as long as it
* stayed saved. On an instance where that account was the only candidate the
* run then read as all-failed, and the operator was told the job was broken
* night after night. A refusal of the operator's own credential is reported by
* the caller as `failed`, because there the operator is the one who can act.
*
* The verdict comes back to the caller, which writes `all_failed` into its
* `jobDone` facts. That fact is what lets `readConsecutiveRunFailures` count a
* completed-but-all-failed night as part of a streak.
@@ -46,18 +58,26 @@ export interface JobRunVerdict {
total: number;
/** Candidates it actually attempted (not skipped, not deferred). */
attempted: number;
/** `failed` + `auth_failed`. */
/** Candidates that failed (`failed`; a refused credential is not one). */
failed: number;
/** Every attempted candidate failed (and at least one was attempted). */
allFailed: boolean;
/** Something failed or was withheld, but not everything. */
/**
* Something failed, was withheld or had its credential refused, but not
* everything failed.
*/
partial: boolean;
}
/** Candidates listed on the run's line; the counts always cover all of them. */
const MAX_LISTED_CANDIDATES = 50;
const FAILED: ReadonlySet<JobRunOutcome> = new Set(["failed", "auth_failed"]);
const FAILED: ReadonlySet<JobRunOutcome> = new Set(["failed"]);
/** Attempted, not a failure of the run, but worth a `partial` line. */
const WITHHELD: ReadonlySet<JobRunOutcome> = new Set([
"screened",
"auth_failed",
]);
const NOT_ATTEMPTED: ReadonlySet<JobRunOutcome> = new Set([
"skipped",
"deferred",
@@ -72,14 +92,14 @@ export function judgeJobRun(
(c) => !NOT_ATTEMPTED.has(c.outcome),
).length;
const failed = candidates.filter((c) => FAILED.has(c.outcome)).length;
const screened = candidates.filter((c) => c.outcome === "screened").length;
const withheld = candidates.filter((c) => WITHHELD.has(c.outcome)).length;
const allFailed = attempted > 0 && failed === attempted;
return {
total,
attempted,
failed,
allFailed,
partial: !allFailed && (failed > 0 || screened > 0),
partial: !allFailed && (failed > 0 || withheld > 0),
};
}
+6 -2
View File
@@ -21,7 +21,7 @@ import {
type InsightStatusGeneratePayload,
runInsightStatusGenerate,
} from "@/lib/jobs/insight-status-generate";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { codeFact, jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { jobBudget } from "@/lib/jobs/job-budget";
import { withBackgroundEvent } from "@/lib/logging/background";
import { generateStatusBatchForUser } from "@/lib/insights/status-batch";
@@ -80,7 +80,7 @@ export async function runStatusBatchCron(
locale: await resolveJobLocale(user.locale),
force: false,
});
generated += result.batched + result.fellBack;
generated += result.batched + result.fellBackAnswered;
served += result.served;
} catch (error) {
failed++;
@@ -203,6 +203,10 @@ export async function handleInsightPregenerateJob(
all_failed: summary.allFailed,
assessments_warmed: summary.assessmentsWarmed,
metric_assessments_warmed: summary.metricAssessmentsWarmed,
// #1384 — what failed, for the admin page and the alert ("none"
// when nothing did, or the generator could not say).
failure_cause: codeFact(summary.firstFailure?.reason) ?? "none",
failure_provider: codeFact(summary.firstFailure?.provider) ?? "none",
});
} catch (err) {
evt.addWarning(
@@ -460,6 +460,7 @@
"batchCallMade": true,
"batched": 6,
"fellBack": 1,
"fellBackAnswered": 0,
"served": 0
}
},
@@ -1719,6 +1720,7 @@
"batchCallMade": true,
"batched": 0,
"fellBack": 7,
"fellBackAnswered": 0,
"served": 0
}
},
@@ -1831,6 +1833,7 @@
"batchCallMade": false,
"batched": 0,
"fellBack": 1,
"fellBackAnswered": 0,
"served": 6
}
}
@@ -40,6 +40,7 @@ vi.mock("@/lib/insights/status-batch", async () => {
served: 0,
batched: 0,
fellBack: 0,
fellBackAnswered: 0,
batchCallMade: false,
};
if (!(await aiCapabilityForJob(userId, "statusText")).available) {
+35
View File
@@ -126,6 +126,41 @@ describe("readFailingQueues", () => {
await expect(readFailingQueues()).resolves.toEqual([]);
});
// #1384 — the repeated-failure alert counts a nightly pass that completed
// with every account failed, and sent the operator to this page, which
// listed only pg-boss failures and said nothing had failed.
it("lists a run that completed with every account failed, in words, and not a good run", async () => {
const ALL_FAILED_QUEUE = "insight-pregenerate";
await boss.createQueue(ALL_FAILED_QUEUE);
async function completeOne(did: Record<string, unknown>) {
const jobId = await boss.send(ALL_FAILED_QUEUE, {}, { retryLimit: 0 });
const [job] = await boss.fetch(ALL_FAILED_QUEUE);
if (!job || job.id !== jobId) throw new Error("nothing to fetch");
await boss.complete(ALL_FAILED_QUEUE, job.id, { ok: true, did });
}
// A good night, and the night from the report: one account, one failure.
await completeOne({ total: 1, generated: 1, failed: 0, all_failed: false });
await completeOne({
total: 1,
generated: 0,
failed: 1,
all_failed: true,
assessments_warmed: 0,
failure_cause: "all_providers_failed",
failure_provider: "openai_compatible",
});
const failing = await readFailingQueues();
const entry = failing?.find((e) => e.queue === ALL_FAILED_QUEUE);
expect(entry?.failures).toBe(1);
expect(entry?.lastError).toBe(
"The run completed, but every account it attempted failed (1 of 1). " +
"First cause: all_providers_failed from provider openai_compatible. " +
"Accounts without a usable provider are skipped and not counted; " +
"job.run.failed in the app log lists each account's outcome.",
);
});
it("ignores a failure older than the window", async () => {
await failOneJob(QUEUE, { userId: "user-a" }, "old failure");
await getPrismaClient().$executeRawUnsafe(