fix(jobs): stop counting an unusable provider as a failing briefing run

The nightly briefing pass settled an account as failed whenever its
generation failed, including when every provider of its chain was
benched for background work and no request was sent, and when the
provider refused the account's own credential. A key revoked at its
issuer stays saved and is refused every night, so a single-account
instance read as all-failed night after night and the repeated-failure
alert reported a broken job.

- A chain that sent nothing settles as skipped (provider_benched), with
  no 45-minute retry.
- A refused credential of the account's own settles as auth_failed,
  which the run reports as partial but does not fail over; a refusal of
  the operator's shared key still fails the run.
- The generator's failed outcome names the provider that failed first,
  whether its credential was refused and whether anything was sent.
- The run's facts carry failure_cause and failure_provider.

The admin Background jobs card listed only pg-boss failures, while the
alert also counts runs that completed with every account failed, so an
operator sent to the card read that nothing had failed. The card now
lists those runs, and the card and the alert describe them in words
with the cause and provider instead of quoting the run's JSON.

The warm tally counted a status card that fell back to its deterministic
text as a warmed assessment; it now counts model-written notes only.

Refs #1384
This commit is contained in:
Marc-André Bombeck committed 2026-11-15 10:45:21 +01:00
1 parent ac4a4b2fbc
commit 75f8a648dc
32 files changed
+675 -94

No files matched your search

@@ -460,6 +460,7 @@
"batchCallMade": true,
"batched": 6,
"fellBack": 1,
"fellBackAnswered": 0,
"served": 0
}
},
@@ -1719,6 +1720,7 @@
"batchCallMade": true,
"batched": 0,
"fellBack": 7,
"fellBackAnswered": 0,
"served": 0
}
},
@@ -1831,6 +1833,7 @@
"batchCallMade": false,
"batched": 0,
"fellBack": 1,
"fellBackAnswered": 0,
"served": 6
}
}
@@ -40,6 +40,7 @@ vi.mock("@/lib/insights/status-batch", async () => {
served: 0,
batched: 0,
fellBack: 0,
fellBackAnswered: 0,
batchCallMade: false,
};
if (!(await aiCapabilityForJob(userId, "statusText")).available) {
+35
View File
@@ -126,6 +126,41 @@ describe("readFailingQueues", () => {
await expect(readFailingQueues()).resolves.toEqual([]);
});
// #1384 — the repeated-failure alert counts a nightly pass that completed
// with every account failed, and sent the operator to this page, which
// listed only pg-boss failures and said nothing had failed.
it("lists a run that completed with every account failed, in words, and not a good run", async () => {
const ALL_FAILED_QUEUE = "insight-pregenerate";
await boss.createQueue(ALL_FAILED_QUEUE);
async function completeOne(did: Record<string, unknown>) {
const jobId = await boss.send(ALL_FAILED_QUEUE, {}, { retryLimit: 0 });
const [job] = await boss.fetch(ALL_FAILED_QUEUE);
if (!job || job.id !== jobId) throw new Error("nothing to fetch");
await boss.complete(ALL_FAILED_QUEUE, job.id, { ok: true, did });
}
// A good night, and the night from the report: one account, one failure.
await completeOne({ total: 1, generated: 1, failed: 0, all_failed: false });
await completeOne({
total: 1,
generated: 0,
failed: 1,
all_failed: true,
assessments_warmed: 0,
failure_cause: "all_providers_failed",
failure_provider: "openai_compatible",
});
const failing = await readFailingQueues();
const entry = failing?.find((e) => e.queue === ALL_FAILED_QUEUE);
expect(entry?.failures).toBe(1);
expect(entry?.lastError).toBe(
"The run completed, but every account it attempted failed (1 of 1). " +
"First cause: all_providers_failed from provider openai_compatible. " +
"Accounts without a usable provider are skipped and not counted; " +
"job.run.failed in the app log lists each account's outcome.",
);
});
it("ignores a failure older than the window", async () => {
await failOneJob(QUEUE, { userId: "user-a" }, "old failure");
await getPrismaClient().$executeRawUnsafe(