diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d4156079..de935e7cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,21 @@ default was rejected as in the future, and onset dates could display as the wrong calendar day in non-UTC time zones. Saving now anchors to the current time when before local noon and preserves stored timestamps on edit. +- **No more nightly "job still failing" alert over a provider that cannot + answer.** When the only account on an instance still had an AI provider + saved whose key the provider no longer accepted (removing the key from a + Local or OpenAI-compatible provider keeps its address, so it is still + called), or whose provider HealthLog had paused after repeated errors, the + nightly briefing pass counted that account as a failure, and every night + the admins were told the job was broken. A key the provider refuses is now + that account's matter: the run notes it but does not fail over it, unless + it is the instance's own shared key. An account whose providers are all + paused is skipped, since nothing was sent. The Background jobs card on the + admin status page now lists a run that finished with every account + failing, the same runs the alert counts, and the alert and the card say in + words what failed and which provider, instead of quoting the run's + counters. Those counters also stop counting a fallback text as a warmed + assessment. (#1384) ### Security diff --git a/messages/de.json b/messages/de.json index 66fa04236..06ca76108 100644 --- a/messages/de.json +++ b/messages/de.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Hintergrundjobs", - "description": "Queues, deren Jobs nach allen Wiederholungen aufgegeben haben.", + "description": "Queues, deren Jobs nach allen Wiederholungen aufgaben oder deren Lauf bei jedem versuchten Konto scheiterte.", "none": "In den letzten {hours} Stunden ist kein Job fehlgeschlagen.", "unavailable": "Keine Job-Queue lesbar. Dieses Deployment läuft ohne Hintergrund-Worker.", "countOne": "{count} Fehlschlag", diff --git a/messages/en.json b/messages/en.json index 765ba808b..de6c5d6a6 100644 --- a/messages/en.json +++ b/messages/en.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Background jobs", - "description": "Queues whose jobs gave up after exhausting their retries.", + "description": "Queues whose jobs gave up after their retries, or whose run ended with every account it tried failing.", "none": "No job has failed in the last {hours} hours.", "unavailable": "No job queue to read. This deployment runs without the background worker.", "countOne": "{count} failure", diff --git a/messages/es.json b/messages/es.json index e5f751113..41be21086 100644 --- a/messages/es.json +++ b/messages/es.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Trabajos en segundo plano", - "description": "Colas cuyos trabajos se rindieron tras agotar sus reintentos.", + "description": "Colas cuyos trabajos se rindieron tras sus reintentos o cuya ejecución falló en cada cuenta intentada.", "none": "Ningún trabajo ha fallado en las últimas {hours} horas.", "unavailable": "No hay ninguna cola que consultar: este despliegue funciona sin el worker en segundo plano.", "countOne": "{count} fallo", diff --git a/messages/fr.json b/messages/fr.json index b5e9d4ccf..84226f8ee 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Tâches en arrière-plan", - "description": "Files dont les tâches ont abandonné après avoir épuisé leurs tentatives.", + "description": "Files dont les tâches ont abandonné après leurs tentatives, ou dont l’exécution a échoué pour chaque compte.", "none": "Aucune tâche n’a échoué au cours des {hours} dernières heures.", "unavailable": "Aucune file à consulter : ce déploiement fonctionne sans le worker en arrière-plan.", "countOne": "{count} échec", diff --git a/messages/it.json b/messages/it.json index 8e5512ee2..506eb23f8 100644 --- a/messages/it.json +++ b/messages/it.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Processi in background", - "description": "Code i cui processi hanno rinunciato dopo aver esaurito i tentativi.", + "description": "Code i cui processi hanno rinunciato dopo i tentativi o la cui esecuzione è fallita per ogni account.", "none": "Nessun processo è fallito nelle ultime {hours} ore.", "unavailable": "Nessuna coda da leggere: questo deployment funziona senza il worker in background.", "countOne": "{count} errore", diff --git a/messages/ko.json b/messages/ko.json index 6ce331f81..89fe1732c 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -8743,7 +8743,7 @@ }, "failingJobs": { "title": "백그라운드 작업", - "description": "재시도를 다 쓰고 포기한 작업이 있는 큐예요.", + "description": "재시도를 다 쓰고 포기했거나, 실행에서 시도한 모든 계정이 실패한 큐예요.", "none": "최근 {hours}시간 동안 실패한 작업이 없어요.", "unavailable": "읽을 작업 큐가 없어요. 이 배포본은 백그라운드 워커 없이 돌고 있어요.", "countOne": "실패 {count}건", diff --git a/messages/nl.json b/messages/nl.json index 657a2e61e..ddf85f494 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Achtergrondtaken", - "description": "Wachtrijen waarvan taken het hebben opgegeven nadat alle nieuwe pogingen waren verbruikt.", + "description": "Wachtrijen waarvan taken na alle pogingen opgaven, of waarvan een run voor elk geprobeerd account mislukte.", "none": "Er is de afgelopen {hours} uur geen taak mislukt.", "unavailable": "Geen takenwachtrij om te lezen. Deze installatie draait zonder de achtergrondworker.", "countOne": "{count} fout", diff --git a/messages/pl.json b/messages/pl.json index 5ced39f79..aa2b039d2 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Zadania w tle", - "description": "Kolejki, których zadania poddały się po wyczerpaniu ponowień.", + "description": "Kolejki, których zadania poddały się po ponowieniach albo których przebieg zawiódł dla każdego konta.", "none": "W ciągu ostatnich {hours} godzin żadne zadanie nie zawiodło.", "unavailable": "Brak kolejki zadań do odczytu: to wdrożenie działa bez workera w tle.", "countOne": "{count} niepowodzenie", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index 01ab83ec1..351ec8538 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Tarefas em segundo plano", - "description": "Filas cujas tarefas desistiram depois de esgotar as tentativas.", + "description": "Filas cujas tarefas desistiram após as tentativas, ou cuja execução falhou em todas as contas tentadas.", "none": "Nenhuma tarefa falhou nas últimas {hours} horas.", "unavailable": "Nenhuma fila de tarefas para ler. Esta implantação roda sem o worker em segundo plano.", "countOne": "{count} falha", diff --git a/messages/ru.json b/messages/ru.json index c8ba334ab..acd103c36 100644 --- a/messages/ru.json +++ b/messages/ru.json @@ -8768,7 +8768,7 @@ }, "failingJobs": { "title": "Фоновые задания", - "description": "Очереди, задания которых сдались после исчерпания повторных попыток.", + "description": "Очереди, задания которых сдались после всех попыток или чей запуск не удался ни для одного аккаунта.", "none": "За последние {hours} часов ни одно задание не завершилось неудачей.", "unavailable": "Нет очереди заданий для чтения. Это развёртывание работает без фонового воркера.", "countOne": "{count} неудача", diff --git a/src/__tests__/baselines/test-as-never.json b/src/__tests__/baselines/test-as-never.json index 25d657b68..cddd240c4 100644 --- a/src/__tests__/baselines/test-as-never.json +++ b/src/__tests__/baselines/test-as-never.json @@ -435,7 +435,7 @@ "lib/jobs/__tests__/document-summary-reaper.test.ts": 1, "lib/jobs/__tests__/document-summary.test.ts": 19, "lib/jobs/__tests__/geo-backfill.test.ts": 1, - "lib/jobs/__tests__/insight-pregenerate.test.ts": 45, + "lib/jobs/__tests__/insight-pregenerate.test.ts": 43, "lib/jobs/__tests__/insight-status-generate.test.ts": 2, "lib/jobs/__tests__/measurement-maintenance-schedule.test.ts": 2, "lib/jobs/__tests__/measurement-reminder-byhour.test.ts": 2, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 6bbcd8aee..bbc180535 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2863, - "total": 9122, + "total": 9118, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -2134,7 +2134,7 @@ "lib/insights/chart-tokens.ts": 24, "lib/insights/coach-launch-context.tsx": 10, "lib/insights/comprehensive-aggregator.ts": 27, - "lib/insights/comprehensive-generate.ts": 45, + "lib/insights/comprehensive-generate.ts": 44, "lib/insights/correlation-channel-series.ts": 16, "lib/insights/correlation-discovery.ts": 30, "lib/insights/correlation-series-builders.ts": 1, @@ -2299,7 +2299,7 @@ "lib/jobs/google-health-oauth-state-cleanup.ts": 1, "lib/jobs/google-health-sleep-repair.ts": 1, "lib/jobs/insight-pregenerate-shared.ts": 1, - "lib/jobs/insight-pregenerate.ts": 36, + "lib/jobs/insight-pregenerate.ts": 33, "lib/jobs/insight-status-generate-shared.ts": 3, "lib/jobs/insight-status-generate.ts": 3, "lib/jobs/intake-auto-skip.ts": 8, diff --git a/src/__tests__/error-level-discipline-guard.test.ts b/src/__tests__/error-level-discipline-guard.test.ts index c3432c8be..5376df965 100644 --- a/src/__tests__/error-level-discipline-guard.test.ts +++ b/src/__tests__/error-level-discipline-guard.test.ts @@ -168,6 +168,11 @@ const ALLOWED: Readonly> = { reason: "No pgboss schema on a web-only deployment; null means no queue to ask, and the evaluation then judges nothing.", }, + "lib/jobs/job-failures.ts::explainAllFailedRun": { + count: 1, + reason: + "A pg-boss output that does not parse as a run's outcome is not an all-failed run; presentError then shows the raw text as it always did.", + }, "lib/jobs/job-failures.ts::readActiveJobsStartedBefore": { count: 1, reason: diff --git a/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts b/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts index 5468a5de7..18bb396c5 100644 --- a/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts +++ b/src/lib/insights/__tests__/comprehensive-warm-timeout.test.ts @@ -209,4 +209,59 @@ describe("comprehensive warm-path timeout + failure observability", () => { (failAnnotation![0] as { meta?: { reason?: string } }).meta?.reason, ).toBe("all-providers-failed"); }); + + // #1384 — the nightly pass needs to know which configured provider failed, + // whose credential a refusal was, and whether anything was sent at all. + it("names the provider that failed first and whether its credential was refused", async () => { + runRawCompletionWithFallback.mockRejectedValue( + Object.assign(new AllProvidersFailedError([]), { + attempts: [ + { + providerType: "openai-compatible", + attempt: 1, + failureReason: "401", + httpStatus: 401, + }, + ], + primaryCredentialExpired: true, + sentNothing: false, + }), + ); + + const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); + + expect(outcome).toEqual({ + status: "failed", + reason: "all-providers-failed", + credentialExpired: true, + provider: "openai-compatible", + }); + }); + + it("says so when every provider was benched and nothing was sent (#1279)", async () => { + runRawCompletionWithFallback.mockRejectedValue( + Object.assign(new AllProvidersFailedError([]), { + attempts: [ + { + providerType: "local", + attempt: 1, + failureReason: "benched-for-background-work", + httpStatus: null, + failureKind: "paused", + }, + ], + primaryCredentialExpired: false, + sentNothing: true, + }), + ); + + const outcome = await generateComprehensiveInsight("u1", { locale: "de" }); + + expect(outcome).toEqual({ + status: "failed", + reason: "all-providers-failed", + provider: "local", + sentNothing: true, + }); + }); }); diff --git a/src/lib/insights/__tests__/status-batch.test.ts b/src/lib/insights/__tests__/status-batch.test.ts index 6aaf2dff5..e9c26b9e0 100644 --- a/src/lib/insights/__tests__/status-batch.test.ts +++ b/src/lib/insights/__tests__/status-batch.test.ts @@ -134,6 +134,7 @@ describe("generateStatusBatchForUser — the statusText capability comes first", served: 0, batched: 0, fellBack: 0, + fellBackAnswered: 0, batchCallMade: false, }); }); @@ -340,6 +341,15 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => { vi.mocked(runStatusCompletion).mockResolvedValue({ kind: "error", } as never); + // #1384 — one card's own call came back with a model note; the other six + // served their fallback. A nightly warm over a provider that only failed + // reported `assessments_warmed: 6`, because every fallback counted. + vi.mocked(runPreparedStatusCard).mockResolvedValueOnce({ + hasProvider: true, + text: "note", + cached: false, + updatedAt: "t", + }); const result = await generateStatusBatchForUser("u1", { locale: "en" }); @@ -347,6 +357,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => { for (const f of finalizers) expect(f).not.toHaveBeenCalled(); expect(runPreparedStatusCard).toHaveBeenCalledTimes(7); expect(result.fellBack).toBe(7); + expect(result.fellBackAnswered).toBe(1); expect(result.batched).toBe(0); }); @@ -411,6 +422,7 @@ describe("generateStatusBatchForUser — graceful partial degradation", () => { for (const f of finalizers) expect(f).not.toHaveBeenCalled(); expect(runPreparedStatusCard).not.toHaveBeenCalled(); expect(result.fellBack).toBe(7); + expect(result.fellBackAnswered).toBe(0); expect(result.batchCallMade).toBe(false); }); }); diff --git a/src/lib/insights/chain-failure-facts.ts b/src/lib/insights/chain-failure-facts.ts new file mode 100644 index 000000000..1ac5a0bfb --- /dev/null +++ b/src/lib/insights/chain-failure-facts.ts @@ -0,0 +1,37 @@ +/** + * What a failed briefing generation knows about its provider chain beyond + * the reason (#1384): which configured provider failed first, whether that + * provider refused the credential, and whether anything was sent at all. + * + * The nightly pass needs all three to settle an account honestly. A refusal + * of the account's own credential is the account's matter, a refusal of the + * operator's shared key is the operator's, and a chain whose every entry the + * provider health ledger benched for background work (#1279) sent no request, + * so the account was not attempted. Without them every such account read as + * a failure, night after night. + */ +import { AllProvidersFailedError } from "@/lib/ai/provider-runner"; + +export interface ChainFailureFacts { + /** + * The chain's first provider refused the credential (a dead Codex + * sign-in, a revoked key). No retry can succeed until someone reconnects + * or replaces it. + */ + credentialExpired?: boolean; + /** The chain entry that failed first, e.g. `openai-compatible`. */ + provider?: string; + /** Every entry was benched for background work; no request left. */ + sentNothing?: boolean; +} + +/** The facts of a thrown chain failure; empty for any other error. */ +export function chainFailureFacts(error: unknown): ChainFailureFacts { + if (!(error instanceof AllProvidersFailedError)) return {}; + const provider = error.attempts?.[0]?.providerType; + return { + ...(error.primaryCredentialExpired ? { credentialExpired: true } : {}), + ...(provider ? { provider } : {}), + ...(error.sentNothing ? { sentNothing: true } : {}), + }; +} diff --git a/src/lib/insights/comprehensive-generate.ts b/src/lib/insights/comprehensive-generate.ts index 5d1f6a0f1..3911d54eb 100644 --- a/src/lib/insights/comprehensive-generate.ts +++ b/src/lib/insights/comprehensive-generate.ts @@ -93,6 +93,10 @@ import { usesHourlyMeanDay } from "@/lib/measurements/day-statistic"; import { insightResultSchema, type InsightResult } from "@/lib/ai/types"; import { resolveProviderChain } from "@/lib/ai/provider"; import { AllProvidersFailedError } from "@/lib/ai/provider-runner"; +import { + chainFailureFacts, + type ChainFailureFacts, +} from "@/lib/insights/chain-failure-facts"; import { BriefingBudgetExceededError, runBriefingCompletion, @@ -178,17 +182,7 @@ export type GenerateOutcome = | "scope-changed" | "profile-scope-changed"; } - | { - status: "failed"; - reason: string; - /** - * v1.42 — the chain's first provider refused the credential (a dead - * Codex sign-in, a revoked key). The nightly pass reports the account - * as `auth_failed` and does not queue a retry that cannot succeed until - * the person reconnects. - */ - credentialExpired?: boolean; - }; + | ({ status: "failed"; reason: string } & ChainFailureFacts); /** Higher, seedless temperature for the daily-briefing phrasing re-roll. */ const BRIEFING_REROLL_TEMPERATURE = 0.6; @@ -1290,11 +1284,7 @@ export async function generateComprehensiveInsight( ? (e.attempts?.[0]?.failureKind ?? null) : providerFailureKind(e).failure, }); - const credentialExpired = - e instanceof AllProvidersFailedError && e.primaryCredentialExpired; - return credentialExpired - ? { status: "failed", reason, credentialExpired } - : { status: "failed", reason }; + return { status: "failed", reason, ...chainFailureFacts(e) }; } // Anthropic + local have no native JSON mode, so a ```json-fenced or diff --git a/src/lib/insights/status-batch.ts b/src/lib/insights/status-batch.ts index a5a9a27e6..48aa11f8c 100644 --- a/src/lib/insights/status-batch.ts +++ b/src/lib/insights/status-batch.ts @@ -37,6 +37,7 @@ import { runPreparedStatusCard, type PendingStatusCard, type PreparedStatusCard, + type StatusCardResult, } from "@/lib/insights/status-card-generation"; import { runStatusCompletion } from "@/lib/insights/status-provider"; import { @@ -77,10 +78,27 @@ export interface StatusBatchResult { /** Metrics that fell back to a single-card call (omitted by the batch, or * the batch failed). */ fellBack: number; + /** + * Of `fellBack`, the metrics whose own single-card call came back with a + * model-written assessment. The rest served their deterministic fallback: + * no provider, a failed or cooling-down provider, or a screened reply. + * `batched + fellBackAnswered` is what a warm pass may call warmed (#1384); + * `fellBack` alone counted a night of six failed calls as six assessments. + */ + fellBackAnswered: number; /** True when the one batched provider call was issued. */ batchCallMade: boolean; } +/** + * Whether a single-card run ended in a model-written assessment: the persisted + * shape is the only one with a provider and no cache flag. The no-key, the + * timeout / error stub and the screened fallback all carry `cached: true`. + */ +function modelAnswered(result: StatusCardResult): boolean { + return result.hasProvider === true && result.cached === false; +} + /** Pull `perMetric` out of a (possibly fenced) model completion. */ function parsePerMetric(content: string): Record | null { const tryParse = (raw: string): Record | null => { @@ -119,6 +137,7 @@ export async function generateStatusBatchForUser( served: 0, batched: 0, fellBack: 0, + fellBackAnswered: 0, batchCallMade: false, }; @@ -171,11 +190,19 @@ export async function generateStatusBatchForUser( // One metric still needing the LLM is not worth a batch envelope — run it // through its own single-card path (identical persisted result). if (pending.length === 1) { - await runPreparedStatusCard(pending[0]); + if (modelAnswered(await runPreparedStatusCard(pending[0]))) { + result.fellBackAnswered++; + } result.fellBack++; annotate({ action: { name: "insights.status.batch" }, - meta: { served: result.served, batched: 0, fell_back: 1, single: true }, + meta: { + served: result.served, + batched: 0, + fell_back: 1, + fell_back_answered: result.fellBackAnswered, + single: true, + }, }); return result; } @@ -272,14 +299,19 @@ export async function generateStatusBatchForUser( // Batch missed this metric (omitted key, or timeout/error/invalid-json): // fall back to the single-card path. On a batch timeout/error this writes // the per-card negative stub via the card's own `timeout`. - await runPreparedStatusCard(card); - return "fellBack" as const; + return modelAnswered(await runPreparedStatusCard(card)) + ? ("fellBackAnswered" as const) + : ("fellBack" as const); }); const outcomes = await Promise.all(fanOuts); for (const o of outcomes) { - if (o === "batched") result.batched++; - else result.fellBack++; + if (o === "batched") { + result.batched++; + continue; + } + result.fellBack++; + if (o === "fellBackAnswered") result.fellBackAnswered++; } annotate({ @@ -288,6 +320,7 @@ export async function generateStatusBatchForUser( served: result.served, batched: result.batched, fell_back: result.fellBack, + fell_back_answered: result.fellBackAnswered, pending: pending.length, }, }); diff --git a/src/lib/jobs/__tests__/insight-pregenerate.test.ts b/src/lib/jobs/__tests__/insight-pregenerate.test.ts index 80e182903..f889746eb 100644 --- a/src/lib/jobs/__tests__/insight-pregenerate.test.ts +++ b/src/lib/jobs/__tests__/insight-pregenerate.test.ts @@ -103,6 +103,7 @@ vi.mock("@/lib/insights/status-batch", () => ({ served: 0, batched: 0, fellBack: 0, + fellBackAnswered: 0, batchCallMade: false, })), })); @@ -132,6 +133,7 @@ import { INSIGHT_PREGENERATE_EXPIRE_SECONDS, } from "../insight-pregenerate-shared"; import { handleInsightPregenerateJob } from "../reminder/insights-handlers"; +import { judgeJobRun, type reportJobRun } from "../job-run-report"; import { insightRetryOptions } from "../reminder/registrar-shared"; import { groupOf, @@ -1556,15 +1558,9 @@ describe("runInsightPregenerate — warm budget honours the response timeout (v1 // withheld (screened) briefing and a dead sign-in queue no provider retry, // and a time-budget cut-off reports the unreached accounts as deferred. describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => { - type Reported = { - queue: string; - runId: string; - candidates: Array<{ key: string; outcome: string; cause?: string }>; - }; - function harness(users: Array<{ id: string; locale: string }>) { const { prisma } = makePrisma(users); - const reportRun = vi.fn(async (input: Reported) => ({ + const reportRun = vi.fn(async (input) => ({ total: input.candidates.length, attempted: 0, failed: 0, @@ -1606,7 +1602,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => { statusGenerators: h.statusGenerators, warmGenericMetrics: h.warmGenericMetrics, enqueueRetry: h.enqueueRetry, - reportRun: h.reportRun as never, + reportRun: h.reportRun, }); expect(h.reportRun).toHaveBeenCalledTimes(1); @@ -1639,7 +1635,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => { statusGenerators: h.statusGenerators, warmGenericMetrics: h.warmGenericMetrics, enqueueRetry: h.enqueueRetry, - reportRun: h.reportRun as never, + reportRun: h.reportRun, }); expect(h.enqueueRetry).toHaveBeenCalledTimes(1); @@ -1668,7 +1664,7 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => { statusGenerators: h.statusGenerators, warmGenericMetrics: h.warmGenericMetrics, enqueueRetry: h.enqueueRetry, - reportRun: h.reportRun as never, + reportRun: h.reportRun, shouldStop: () => ++calls > 1, }); @@ -1677,3 +1673,117 @@ describe("runInsightPregenerate — one outcome per candidate (v1.42)", () => { ).toEqual(["ok", "deferred", "deferred"]); }); }); + +// #1384 — a self-hoster whose only account kept a provider that no longer +// works was told every night that the job was failing. The run used the real +// verdict here, so what the alert reads (`all_failed`) is what is asserted. +describe("runInsightPregenerate — a provider that cannot serve is not a failing run (#1384)", () => { + async function runOne(outcome: Record) { + const { prisma } = makePrisma([{ id: "only-account", locale: "en" }]); + const reportRun = vi.fn(async (input) => + judgeJobRun(input.candidates), + ); + const enqueueRetry = vi.fn().mockResolvedValue(undefined); + const result = await runInsightPregenerate(prisma as never, { + generate: vi.fn().mockResolvedValue(outcome), + statusGenerators: Array.from({ length: 7 }, () => + vi.fn().mockResolvedValue({ hasProvider: true, cached: true }), + ), + warmGenericMetrics: vi.fn().mockResolvedValue(0), + enqueueRetry, + reportRun, + }); + return { + result, + enqueueRetry, + candidate: reportRun.mock.calls[0][0].candidates[0], + }; + } + + it("skips an account whose every provider was benched, so nothing was sent", async () => { + const { result, enqueueRetry, candidate } = await runOne({ + status: "failed", + reason: "all-providers-failed", + provider: "local", + sentNothing: true, + }); + + expect(candidate).toEqual( + expect.objectContaining({ + outcome: "skipped", + cause: "provider_benched", + }), + ); + expect(result.skipped).toBe(1); + expect(result.failed).toBe(0); + expect(result.allFailed).toBe(false); + expect(result.firstFailure).toBeNull(); + // The retry would find the same bench. + expect(enqueueRetry).not.toHaveBeenCalled(); + expect(annotateSpy).toHaveBeenCalledWith( + expect.objectContaining({ + action: { name: "insights.pregenerate.provider_benched" }, + }), + ); + }); + + it("reports a refused credential of the account's own as auth_failed, naming the provider, without failing the run", async () => { + const { result, enqueueRetry, candidate } = await runOne({ + status: "failed", + reason: "all-providers-failed", + provider: "openai-compatible", + credentialExpired: true, + }); + + expect(candidate).toEqual( + expect.objectContaining({ + outcome: "auth_failed", + cause: "all-providers-failed:openai-compatible", + }), + ); + expect(result.allFailed).toBe(false); + expect(enqueueRetry).not.toHaveBeenCalled(); + expect(annotateSpy).toHaveBeenCalledWith( + expect.objectContaining({ + action: { name: "insights.pregenerate.comprehensive_failed" }, + meta: expect.objectContaining({ + provider: "openai-compatible", + credential_refused: true, + }), + }), + ); + }); + + it("still fails the run when the operator's own key was refused, and says which", async () => { + const { result, enqueueRetry, candidate } = await runOne({ + status: "failed", + reason: "all-providers-failed", + provider: "admin-openai", + credentialExpired: true, + }); + + expect(candidate.outcome).toBe("failed"); + expect(result.allFailed).toBe(true); + expect(result.firstFailure).toEqual({ + reason: "all-providers-failed", + provider: "admin-openai", + }); + // A refused key does not heal in 45 minutes. + expect(enqueueRetry).not.toHaveBeenCalled(); + }); + + it("carries the cause and provider of an ordinary failure into the run's facts", async () => { + const { result, enqueueRetry } = await runOne({ + status: "failed", + reason: "provider-error", + provider: "local", + }); + + expect(result.allFailed).toBe(true); + expect(result.firstFailure).toEqual({ + reason: "provider-error", + provider: "local", + }); + expect(enqueueRetry).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/jobs/__tests__/job-outcome.test.ts b/src/lib/jobs/__tests__/job-outcome.test.ts index 9f0d79d65..437634129 100644 --- a/src/lib/jobs/__tests__/job-outcome.test.ts +++ b/src/lib/jobs/__tests__/job-outcome.test.ts @@ -193,3 +193,36 @@ describe("bounded generic JobOutcome serialization", () => { }); }); }); + +// #1384 — a generator reason and a provider type become facts the admin page +// and the repeated-failure alert can show. +describe("codeFact", () => { + it("turns our own hyphenated codes into stable fact codes", async () => { + const { codeFact, serializeJobOutcome } = await vi.importActual< + typeof import("@/lib/jobs/job-outcome") + >("@/lib/jobs/job-outcome"); + expect(codeFact("all-providers-failed")).toBe("all_providers_failed"); + expect(codeFact("openai-compatible")).toBe("openai_compatible"); + expect(codeFact("generator:invalid-json")).toBe("generator_invalid_json"); + expect(codeFact("timeout")).toBe("timeout"); + // Nothing usable: the caller leaves the fact out. + expect(codeFact("")).toBeNull(); + expect(codeFact("---")).toBeNull(); + expect(codeFact(null)).toBeNull(); + // What it produces is a fact the serializer accepts. + expect( + serializeJobOutcome( + jobDone({ + failure_cause: codeFact("all-providers-failed")!, + failure_provider: codeFact("admin-openai")!, + }), + ), + ).toEqual({ + ok: true, + did: { + failure_cause: "all_providers_failed", + failure_provider: "admin_openai", + }, + }); + }); +}); diff --git a/src/lib/jobs/__tests__/job-run-report.test.ts b/src/lib/jobs/__tests__/job-run-report.test.ts index 2ccd2dd19..3f17db911 100644 --- a/src/lib/jobs/__tests__/job-run-report.test.ts +++ b/src/lib/jobs/__tests__/job-run-report.test.ts @@ -47,13 +47,27 @@ describe("judgeJobRun", () => { }); }); - it("counts a dead credential as a failure", () => { - expect( - judgeJobRun([ - { key: "a", outcome: "auth_failed" }, - { key: "b", outcome: "failed" }, - ]).allFailed, - ).toBe(true); + // #1384 — a key the issuer revoked stays saved and is refused every night. + // Counted as a failure, it made a single-account instance's nightly pass + // read as all-failed forever and alerted the operator about the job. + it("does not fail a run over a refused credential of the account's own", () => { + expect(judgeJobRun([{ key: "a", outcome: "auth_failed" }])).toEqual({ + total: 1, + attempted: 1, + failed: 0, + allFailed: false, + partial: true, + }); + }); + + it("reads a refused credential beside a real failure as partial", () => { + const verdict = judgeJobRun([ + { key: "a", outcome: "auth_failed" }, + { key: "b", outcome: "failed" }, + ]); + expect(verdict.allFailed).toBe(false); + expect(verdict.partial).toBe(true); + expect(verdict.failed).toBe(1); }); it("reads one success among failures as partial, not failed", () => { @@ -84,7 +98,7 @@ describe("reportJobRun", () => { queue: "insight-pregenerate", runId: "r1", candidates: [ - { key: "a", outcome: "auth_failed", cause: "all-providers-failed" }, + { key: "a", outcome: "failed", cause: "all-providers-failed" }, { key: "b", outcome: "skipped", cause: "no-consent" }, ], }); @@ -101,7 +115,31 @@ describe("reportJobRun", () => { consecutiveFailures: 3, failed: 1, total: 1, - causes: ["auth_failed:all-providers-failed"], + causes: ["failed:all-providers-failed"], + }, + }); + }); + + it("emits job.run.partial, never job.run.failed, for a run whose only attempt was a refused credential (#1384)", async () => { + const verdict = await reportJobRun({ + queue: "insight-pregenerate", + runId: "r1", + candidates: [ + { + key: "a", + outcome: "auth_failed", + cause: "all-providers-failed:openai-compatible", + }, + ], + }); + expect(verdict.allFailed).toBe(false); + expect(readConsecutiveRunFailures).not.toHaveBeenCalled(); + expect(emitSignal).toHaveBeenCalledTimes(1); + expect(emitSignal.mock.calls[0][0]).toMatchObject({ + action: "job.run.partial", + level: "warn", + meta: { + causes: ["auth_failed:all-providers-failed:openai-compatible"], }, }); }); diff --git a/src/lib/jobs/__tests__/system-job-alert.test.ts b/src/lib/jobs/__tests__/system-job-alert.test.ts index 224fb0209..14c390d6f 100644 --- a/src/lib/jobs/__tests__/system-job-alert.test.ts +++ b/src/lib/jobs/__tests__/system-job-alert.test.ts @@ -87,6 +87,34 @@ describe("judgeQueueRuns", () => { ).toBe("failing"); }); + // #1384 — the alert quoted the completed run's own outcome JSON, and the + // operator it sent to the admin page found nothing there. + it("says in words what an all-failed night was, with its cause and provider", () => { + const output = JSON.stringify({ + ok: true, + did: { + total: 1, + failed: 1, + all_failed: true, + assessments_warmed: 0, + failure_cause: "all_providers_failed", + failure_provider: "admin_openai", + }, + }); + const verdict = judgeQueueRuns( + [0, 1, 2].map((ago) => + run(ago, { state: "completed", allFailed: true, error: output }), + ), + ); + expect(verdict.state).toBe("failing"); + expect(verdict.lastError).toBe( + "The run completed, but every account it attempted failed (1 of 1). " + + "First cause: all_providers_failed from provider admin_openai. " + + "Accounts without a usable provider are skipped and not counted; " + + "job.run.failed in the app log lists each account's outcome.", + ); + }); + it("explains a bare timeout the way the admin card does", () => { const verdict = judgeQueueRuns( [0, 1, 2].map((ago) => diff --git a/src/lib/jobs/insight-pregenerate-failure.ts b/src/lib/jobs/insight-pregenerate-failure.ts new file mode 100644 index 000000000..bcad68b35 --- /dev/null +++ b/src/lib/jobs/insight-pregenerate-failure.ts @@ -0,0 +1,93 @@ +/** + * How the nightly briefing pass settles an account whose generation came back + * `failed` (#1384). + * + * A failure used to be one of three things: screened, a dead sign-in, or a + * provider failure worth a retry in 45 minutes. Two cases were missing, and + * both made a self-hoster's only account read as a failing run every night, + * which the repeated-failure alert then reported as a broken job: + * + * - Every provider of the chain was benched for background work by the + * provider health ledger (#1279), so no request was sent. The account was + * not attempted tonight; its earlier failures were counted by the runs + * that made them, and a retry would find the same bench. It is `skipped`. + * - The provider refused the credential. A refused credential does not heal + * in 45 minutes, so no retry is queued, and whose credential it was + * decides who has to act. The operator's shared key or central sign-in is + * the operator's to fix, and a run where it was refused for everyone is a + * failing run: `failed`. The account's own key, gateway or sign-in is its + * owner's to fix: `auth_failed`, which the run reports but does not fail + * over (`job-run-report.ts`). An API key revoked at its issuer stays saved + * and is refused again every night, unlike a ChatGPT sign-in, which the + * refusal parks. + * + * The model answered and the safety screen withheld it (`outbound-screened`) + * stays `screened` with no retry: asking the same model again in 45 minutes + * is the same question, and it was screened eight times a night. + */ +import { isOperatorHeldProvider } from "@/lib/ai/provider-egress"; +import type { GenerateOutcome } from "@/lib/insights/comprehensive-generate"; +import type { JobRunCandidate, JobRunOutcome } from "@/lib/jobs/job-run-report"; + +type FailedGeneration = Extract; + +export interface FailedGenerationVerdict { + outcome: JobRunOutcome; + /** The run line's cause: the reason, and the provider that failed first. */ + cause: string; + /** False when nothing was sent: the account is tallied as skipped. */ + attempted: boolean; + /** Queue the bounded 45-minute retry for this account. */ + retry: boolean; +} + +export function settleFailedGeneration( + generated: FailedGeneration, +): FailedGenerationVerdict { + if (generated.sentNothing) { + return { + outcome: "skipped", + cause: "provider_benched", + attempted: false, + retry: false, + }; + } + const cause = generated.provider + ? `${generated.reason}:${generated.provider}` + : generated.reason; + if (generated.reason === "outbound-screened") { + return { outcome: "screened", cause, attempted: true, retry: false }; + } + if (generated.credentialExpired) { + const operatorHeld = + generated.provider !== undefined && + isOperatorHeldProvider(generated.provider); + return { + outcome: operatorHeld ? "failed" : "auth_failed", + cause, + attempted: true, + retry: false, + }; + } + return { outcome: "failed", cause, attempted: true, retry: true }; +} + +/** The first failed account of a run: its reason and the provider, if known. */ +export interface FirstFailure { + reason: string; + provider: string | null; +} + +/** + * The first failed account's reason and provider, read back off the run's + * causes, for the job's facts: what the admin page and the repeated-failure + * alert show for a run that completed with every account failed. + */ +export function firstFailure( + candidates: readonly JobRunCandidate[], +): FirstFailure | null { + const cause = candidates.find((c) => c.outcome === "failed")?.cause; + if (!cause) return null; + const [reason, provider] = cause.split(":"); + return { reason, provider: provider ?? null }; +} diff --git a/src/lib/jobs/insight-pregenerate.ts b/src/lib/jobs/insight-pregenerate.ts index 59f453448..5c09b727c 100644 --- a/src/lib/jobs/insight-pregenerate.ts +++ b/src/lib/jobs/insight-pregenerate.ts @@ -81,6 +81,10 @@ import { } from "@/lib/insights/metric-status-registry"; import { withTimeout } from "@/lib/insights/with-timeout"; import { comprehensiveWarmBudgetMs } from "@/lib/jobs/model-time"; +import { + firstFailure, + settleFailedGeneration, +} from "@/lib/jobs/insight-pregenerate-failure"; import { INSIGHT_PREGENERATE_QUEUE, enqueueForceWarm, @@ -235,6 +239,8 @@ export interface PregenerateRunResult { * seven specialised cards. */ metricAssessmentsWarmed: number; + /** #1384 — the first failed account's reason and provider, as facts. */ + firstFailure: ReturnType; } /** @@ -302,9 +308,9 @@ async function warmPerStatusCaches( * once and issues ONE provider call for the metrics still needing the LLM, * fanning the response back into the same per-metric cache rows the * standalone generators wrote. Replaces the prior seven independent warm - * calls. Returns the count of metrics that got a fresh (batched or - * fallback) assessment so the wide-event tally is comparable to the old - * per-card count. + * calls. Returns the count of metrics that got a fresh model-written + * assessment, batched or through a card's own fallback call, so the + * wide-event tally is comparable to the old per-card count. * * Refill-only (`force: false`): a card already generated today and a card * whose content-hash is unchanged both resolve inside the batch as `served` @@ -322,7 +328,7 @@ async function warmStatusBatch( locale, force: false, }); - warmed += result.batched + result.fellBack; + warmed += result.batched + result.fellBackAnswered; } catch (err) { // The batch must never throw the cron's user loop off course — but // a swallowed batch failure left ~7 cards cold with zero signal. @@ -555,6 +561,7 @@ export async function runInsightPregenerate( allFailed: false, assessmentsWarmed: 0, metricAssessmentsWarmed: 0, + firstFailure: null, }; // Master kill-switch — when the operator disabled the briefing @@ -619,6 +626,7 @@ export async function runInsightPregenerate( candidates: runOutcomes, }); result.allFailed = verdict.allFailed; + result.firstFailure = firstFailure(runOutcomes); return result; @@ -783,42 +791,34 @@ export async function runInsightPregenerate( result.skipped++; settle("skipped", generated.reason); break; - case "failed": - result.failed++; - // v1.28.30 — THE previously-silent failure path. A generator - // that resolves with `{ status: "failed" }` (features error, - // oversize payload, invalid JSON after retry, abort) bumped - // the tally with no annotation at all — the nightly summary - // showed `failed: 1` and nothing else was greppable. Reuse - // the `comprehensive_failed` shape with the generator's own - // reason as the cause. + case "failed": { + // Benched, screened, a refused credential and whose, or a + // retryable failure: `insight-pregenerate-failure.ts` (#1384). + const verdict = settleFailedGeneration(generated); + if (verdict.attempted) result.failed++; + else result.skipped++; annotate({ - action: { name: "insights.pregenerate.comprehensive_failed" }, + action: { + name: verdict.attempted + ? "insights.pregenerate.comprehensive_failed" + : "insights.pregenerate.provider_benched", + }, meta: { locale, stage: "nightly.generator", cause: `generator:${generated.reason}`, + provider: generated.provider ?? null, + credential_refused: generated.credentialExpired === true, budget_ms: warmBudgetMs, message: null, }, }); - if (generated.reason === "outbound-screened") { - // v1.42 — the model answered and the safety screen withheld - // it, after the generator's own repair. Asking the same model - // again in 45 minutes is not a provider retry; it is the same - // question, and it was screened eight times a night. - settle("screened", generated.reason); - } else if (generated.credentialExpired) { - // v1.42 — a dead sign-in (a Codex refresh token the issuer - // revoked) does not heal in 45 minutes either; the person has - // to reconnect. The refusal parked the connection as expired, - // which Settings shows, and the next run no longer counts it. - settle("auth_failed", generated.reason); - } else { - settle("failed", generated.reason); + settle(verdict.outcome, verdict.cause); + if (verdict.retry) { await enqueueRetry({ userId: candidate.id, locale }); } break; + } } } } diff --git a/src/lib/jobs/job-failures.ts b/src/lib/jobs/job-failures.ts index c9bf7a82b..b96cfd929 100644 --- a/src/lib/jobs/job-failures.ts +++ b/src/lib/jobs/job-failures.ts @@ -92,7 +92,7 @@ const MAX_ERROR_CHARS = 300; export interface FailingQueue { /** The pg-boss queue name, e.g. `environment-fetch`. */ queue: string; - /** Jobs that exhausted their retries inside the window. */ + /** Failed runs inside the window: exhausted retries, or all-failed passes. */ failures: number; /** ISO instant of the newest failure. */ lastFailedAt: string; @@ -140,10 +140,48 @@ function explainTimeout(message: string, expireSeconds: number | null): string { ); } +/** + * Say what a completed run that failed every account means (#1384). Such a + * run's `output` is its outcome, `{"ok":true,"did":{…,"all_failed":true}}`, + * and quoting that JSON told the operator nothing: not that the job ran, not + * which provider failed. Null when `raw` is not such an outcome. + */ +function explainAllFailedRun(raw: string): string | null { + if (!raw.startsWith("{")) return null; + let did: Record; + try { + const parsed = JSON.parse(raw) as { ok?: unknown; did?: unknown }; + if (parsed?.ok !== true || typeof parsed.did !== "object") return null; + did = (parsed.did ?? {}) as Record; + } catch { + return null; + } + if (did.all_failed !== true) return null; + // Every attempted account failed, so the count is both sides of "of". + const failed = + typeof did.failed === "number" && did.failed > 0 + ? ` (${did.failed} of ${did.failed})` + : ""; + // "none" is how the run says it has no cause or provider to name. + const named = (value: unknown): string | null => + typeof value === "string" && value !== "none" ? value : null; + const cause = named(did.failure_cause); + const provider = named(did.failure_provider); + const why = cause + ? ` First cause: ${cause}${provider ? ` from provider ${provider}` : ""}.` + : ""; + return ( + `The run completed, but every account it attempted failed${failed}.${why} ` + + `Accounts without a usable provider are skipped and not counted; ` + + `job.run.failed in the app log lists each account's outcome.` + ); +} + /** * Turn whatever pg-boss stored in `output` into one line. A thrown `Error` * serialises to `{ message, stack, … }`; a thrown non-Error can be any JSON at * all, so the raw text is the fallback rather than an invented placeholder. + * A completed run that reported every account failed is said in words. */ export function presentError( raw: string | null, @@ -152,6 +190,8 @@ export function presentError( if (raw === null) return ""; const trimmed = raw.trim(); if (trimmed.length === 0) return ""; + const allFailed = explainAllFailedRun(trimmed); + if (allFailed !== null) return allFailed; if (TIMEOUT_MESSAGE.test(trimmed)) return explainTimeout(trimmed, expireSeconds); if (HEARTBEAT_MESSAGE.test(trimmed)) { @@ -168,8 +208,13 @@ export function presentError( } /** - * Every queue with at least one exhausted-retry failure in the window, newest - * first. `null` when the queue schema is not present at all. + * Every queue with at least one failed run in the window, newest first: + * a job that exhausted its retries, or a fan-out pass that completed but + * reported that every account it attempted failed (`did.all_failed`). The + * second kind is what the repeated-failure alert counts too + * (`readConsecutiveRunFailures`, `system-job-alert.ts`); the page used to + * list only the first, so an operator the alert sent here read "no job has + * failed" (#1384). `null` when the queue schema is not present at all. */ export async function readFailingQueues( withinHours: number = JOB_FAILURE_WINDOW_HOURS, @@ -194,7 +239,10 @@ export async function readFailingQueues( array_agg(expire_seconds ORDER BY completed_on DESC NULLS LAST) )[1] AS last_expire_seconds FROM pgboss.job - WHERE state = 'failed' + WHERE ( + state = 'failed' + OR (state = 'completed' AND output->'did'->>'all_failed' = 'true') + ) AND completed_on > now() - make_interval(hours => ${withinHours}::int) GROUP BY name ORDER BY max(completed_on) DESC diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 942b5fde6..f47efcf38 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -130,6 +130,11 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "duration_ms", "errored", "failed", + // #1384 — a fan-out pass's first failed candidate: its cause and the + // provider that failed for it, as stable codes. What the admin page and + // the repeated-failure alert show for a run that completed all-failed. + "failure_cause", + "failure_provider", "failures", "feedback_buckets", "feedback_total_rows", @@ -423,6 +428,22 @@ export function percentFact(ratio: number | null | undefined): number { return wholeFact(ratio * 100); } +/** + * A short code from our own vocabulary (a generator reason, a provider type) + * as a fact: `all-providers-failed` becomes `all_providers_failed`. Null when + * nothing of it survives, so the caller leaves the fact out. + */ +export function codeFact(value: string | null | undefined): string | null { + if (value == null) return null; + const code = value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^[^a-z]+/, "") + .slice(0, 64) + .replace(/_+$/, ""); + return STABLE_CODE_PATTERN.test(code) ? code : null; +} + /** Why one fact is refused, or null when it is a valid fact. */ function invalidFact(key: string, value: unknown): string | null { if (!JOB_FACT_ALLOWLIST.has(key)) diff --git a/src/lib/jobs/job-run-report.ts b/src/lib/jobs/job-run-report.ts index 4dadbd0f4..f38f8b4d2 100644 --- a/src/lib/jobs/job-run-report.ts +++ b/src/lib/jobs/job-run-report.ts @@ -21,6 +21,18 @@ * attempted but is not a provider failure, so a run of only screened * candidates is partial, not failed. * + * `auth_failed` is attempted and is not a failure of the run either (#1384). + * It means the provider refused the account's own credential: a key revoked + * at its issuer, a gateway that now wants a key the account no longer + * stores. Only that account's owner can change it, and the run itself worked. + * It used to count as a failure on the assumption that a refusal is reported + * once and then parked; that holds for a ChatGPT sign-in and for nothing else, + * so an API key the issuer had revoked failed every night for as long as it + * stayed saved. On an instance where that account was the only candidate the + * run then read as all-failed, and the operator was told the job was broken + * night after night. A refusal of the operator's own credential is reported by + * the caller as `failed`, because there the operator is the one who can act. + * * The verdict comes back to the caller, which writes `all_failed` into its * `jobDone` facts. That fact is what lets `readConsecutiveRunFailures` count a * completed-but-all-failed night as part of a streak. @@ -46,18 +58,26 @@ export interface JobRunVerdict { total: number; /** Candidates it actually attempted (not skipped, not deferred). */ attempted: number; - /** `failed` + `auth_failed`. */ + /** Candidates that failed (`failed`; a refused credential is not one). */ failed: number; /** Every attempted candidate failed (and at least one was attempted). */ allFailed: boolean; - /** Something failed or was withheld, but not everything. */ + /** + * Something failed, was withheld or had its credential refused, but not + * everything failed. + */ partial: boolean; } /** Candidates listed on the run's line; the counts always cover all of them. */ const MAX_LISTED_CANDIDATES = 50; -const FAILED: ReadonlySet = new Set(["failed", "auth_failed"]); +const FAILED: ReadonlySet = new Set(["failed"]); +/** Attempted, not a failure of the run, but worth a `partial` line. */ +const WITHHELD: ReadonlySet = new Set([ + "screened", + "auth_failed", +]); const NOT_ATTEMPTED: ReadonlySet = new Set([ "skipped", "deferred", @@ -72,14 +92,14 @@ export function judgeJobRun( (c) => !NOT_ATTEMPTED.has(c.outcome), ).length; const failed = candidates.filter((c) => FAILED.has(c.outcome)).length; - const screened = candidates.filter((c) => c.outcome === "screened").length; + const withheld = candidates.filter((c) => WITHHELD.has(c.outcome)).length; const allFailed = attempted > 0 && failed === attempted; return { total, attempted, failed, allFailed, - partial: !allFailed && (failed > 0 || screened > 0), + partial: !allFailed && (failed > 0 || withheld > 0), }; } diff --git a/src/lib/jobs/reminder/insights-handlers.ts b/src/lib/jobs/reminder/insights-handlers.ts index 54fec9bdb..c34506f45 100644 --- a/src/lib/jobs/reminder/insights-handlers.ts +++ b/src/lib/jobs/reminder/insights-handlers.ts @@ -21,7 +21,7 @@ import { type InsightStatusGeneratePayload, runInsightStatusGenerate, } from "@/lib/jobs/insight-status-generate"; -import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { codeFact, jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { jobBudget } from "@/lib/jobs/job-budget"; import { withBackgroundEvent } from "@/lib/logging/background"; import { generateStatusBatchForUser } from "@/lib/insights/status-batch"; @@ -80,7 +80,7 @@ export async function runStatusBatchCron( locale: await resolveJobLocale(user.locale), force: false, }); - generated += result.batched + result.fellBack; + generated += result.batched + result.fellBackAnswered; served += result.served; } catch (error) { failed++; @@ -203,6 +203,10 @@ export async function handleInsightPregenerateJob( all_failed: summary.allFailed, assessments_warmed: summary.assessmentsWarmed, metric_assessments_warmed: summary.metricAssessmentsWarmed, + // #1384 — what failed, for the admin page and the alert ("none" + // when nothing did, or the generator could not say). + failure_cause: codeFact(summary.firstFailure?.reason) ?? "none", + failure_provider: codeFact(summary.firstFailure?.provider) ?? "none", }); } catch (err) { evt.addWarning( diff --git a/tests/integration/__golden__/status-parity/batch.json b/tests/integration/__golden__/status-parity/batch.json index 6091a6647..725ea250c 100644 --- a/tests/integration/__golden__/status-parity/batch.json +++ b/tests/integration/__golden__/status-parity/batch.json @@ -460,6 +460,7 @@ "batchCallMade": true, "batched": 6, "fellBack": 1, + "fellBackAnswered": 0, "served": 0 } }, @@ -1719,6 +1720,7 @@ "batchCallMade": true, "batched": 0, "fellBack": 7, + "fellBackAnswered": 0, "served": 0 } }, @@ -1831,6 +1833,7 @@ "batchCallMade": false, "batched": 0, "fellBack": 1, + "fellBackAnswered": 0, "served": 6 } } diff --git a/tests/integration/background-switch-freshness.test.ts b/tests/integration/background-switch-freshness.test.ts index 99f3ac81f..8ff0db47d 100644 --- a/tests/integration/background-switch-freshness.test.ts +++ b/tests/integration/background-switch-freshness.test.ts @@ -40,6 +40,7 @@ vi.mock("@/lib/insights/status-batch", async () => { served: 0, batched: 0, fellBack: 0, + fellBackAnswered: 0, batchCallMade: false, }; if (!(await aiCapabilityForJob(userId, "statusText")).available) { diff --git a/tests/integration/job-failures.test.ts b/tests/integration/job-failures.test.ts index f13180fd7..997c20f97 100644 --- a/tests/integration/job-failures.test.ts +++ b/tests/integration/job-failures.test.ts @@ -126,6 +126,41 @@ describe("readFailingQueues", () => { await expect(readFailingQueues()).resolves.toEqual([]); }); + // #1384 — the repeated-failure alert counts a nightly pass that completed + // with every account failed, and sent the operator to this page, which + // listed only pg-boss failures and said nothing had failed. + it("lists a run that completed with every account failed, in words, and not a good run", async () => { + const ALL_FAILED_QUEUE = "insight-pregenerate"; + await boss.createQueue(ALL_FAILED_QUEUE); + async function completeOne(did: Record) { + const jobId = await boss.send(ALL_FAILED_QUEUE, {}, { retryLimit: 0 }); + const [job] = await boss.fetch(ALL_FAILED_QUEUE); + if (!job || job.id !== jobId) throw new Error("nothing to fetch"); + await boss.complete(ALL_FAILED_QUEUE, job.id, { ok: true, did }); + } + // A good night, and the night from the report: one account, one failure. + await completeOne({ total: 1, generated: 1, failed: 0, all_failed: false }); + await completeOne({ + total: 1, + generated: 0, + failed: 1, + all_failed: true, + assessments_warmed: 0, + failure_cause: "all_providers_failed", + failure_provider: "openai_compatible", + }); + + const failing = await readFailingQueues(); + const entry = failing?.find((e) => e.queue === ALL_FAILED_QUEUE); + expect(entry?.failures).toBe(1); + expect(entry?.lastError).toBe( + "The run completed, but every account it attempted failed (1 of 1). " + + "First cause: all_providers_failed from provider openai_compatible. " + + "Accounts without a usable provider are skipped and not counted; " + + "job.run.failed in the app log lists each account's outcome.", + ); + }); + it("ignores a failure older than the window", async () => { await failOneJob(QUEUE, { userId: "user-a" }, "old failure"); await getPrismaClient().$executeRawUnsafe(