feat(jobs): tell admins when a system job keeps failing

A full rollup backfill that failed on every boot showed only on the
admin page's card of failing background jobs. From the account's side
the only symptom was empty dashboard cards, and nothing pointed the
operator at the card (#1298).

A new hourly evaluation reads the newest terminal runs of every queue
whose failure has no account-facing surface (the system audience in
subsystem-surface.ts) from pg-boss's own ledger. A failed run adds its
attempts, retries included, and the first successful run ends the
streak. Three failed attempts in a row send every admin a SYSTEM_ALERT
once, a reminder at most once a day while the queue keeps failing, and
one notice when it runs successfully again: the decision the backup
health notices use, with the same compare-and-set claim, so two workers
never send a notice twice.

The memory is one row per announced queue in system_job_alert_notices
(migration 0490). The notice names the queue, the streak and the last
error, and opens the admin status page.
This commit is contained in:
Marc-André Bombeck committed 2026-10-29 09:54:43 +01:00
1 parent a571dcc47a
commit f99f87eca6
23 files changed
+756 -3

No files matched your search

@@ -1,5 +1,5 @@
{
"registeredQueueCount": 125,
"registeredQueueCount": 126,
"registeredQueues": [
"account-deletion-purge",
"achievement-unlock-sweep",
@@ -112,6 +112,7 @@
"strava-sync",
"stress-score-compute",
"supplement-nutrient-recompute",
"system-job-alert",
"tls-pin-monitor",
"whoop-backfill",
"whoop-cycle-sync",
@@ -1793,6 +1794,21 @@
"retry_delay_max": null,
"retry_limit": 2
},
{
"dead_letter": null,
"deletion_seconds": 604800,
"expire_seconds": 900,
"heartbeat_seconds": null,
"name": "system-job-alert",
"notify": false,
"partition": false,
"policy": "standard",
"retention_seconds": 1209600,
"retry_backoff": false,
"retry_delay": 0,
"retry_delay_max": null,
"retry_limit": 2
},
{
"dead_letter": null,
"deletion_seconds": 604800,
@@ -2825,6 +2841,18 @@
},
"timezone": "Europe/Berlin"
},
{
"cron": "43 * * * *",
"data": {},
"key": "",
"kind": "cron",
"name": "system-job-alert",
"options": {
"retryLimit": 0,
"tz": "Europe/Berlin"
},
"timezone": "Europe/Berlin"
},
{
"cron": "7 */6 * * *",
"data": {},
@@ -3682,6 +3710,13 @@
"localConcurrency": 1
}
},
{
"queue": "system-job-alert",
"options": {
"localConcurrency": 1,
"pollingIntervalSeconds": 30
}
},
{
"queue": "tls-pin-monitor",
"options": {
+178
View File
@@ -0,0 +1,178 @@
/**
* The system job alert against a ledger pg-boss itself wrote (#1298).
*
* A background job of the instance that keeps failing pushes every admin
* once, a reminder at most once a day while it keeps failing, and once when it
* runs successfully again. Fewer failures than the streak, and queues that
* have an account-facing surface of their own, push nothing.
*
* pg-boss provisions its own schema here and the jobs are failed and completed
* through its own API, as in `job-failures.test.ts`, so the reader is proven
* against the table pg-boss writes rather than one the test wrote.
*/
import { PgBoss } from "pg-boss";
import {
afterAll,
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
} from "vitest";
const dispatched = vi.hoisted(() => ({
calls: [] as Array<{
userId: string;
titleKey: string;
messageKey: string;
params?: Record<string, string | number>;
eventType?: string;
}>,
}));
vi.mock("@/lib/notifications/dispatch-localised", () => ({
dispatchLocalisedNotification: vi.fn(
async (opts: (typeof dispatched.calls)[number]) => {
dispatched.calls.push(opts);
},
),
}));
import {
notifySystemJobFailures,
SYSTEM_JOB_FAILURE_STREAK,
} from "@/lib/jobs/system-job-alert";
import { makeUser } from "../kit/factories";
import { getPrismaClient, truncateAllTables } from "./setup";
const SYSTEM_QUEUE = "rollup-full-backfill";
const ACCOUNT_QUEUE = "google-health-sync";
const HOUR_MS = 3_600_000;
let boss: PgBoss;
async function runOnce(queue: string, outcome: "fail" | "complete") {
const id = await boss.send(queue, { userId: "u" }, { retryLimit: 0 });
if (!id) throw new Error("queue rejected the job");
const [job] = await boss.fetch(queue);
if (!job) throw new Error("nothing to fetch");
if (outcome === "fail") {
await boss.fail(
queue,
job.id,
new Error("canceling statement due to statement timeout"),
);
} else {
await boss.complete(queue, job.id, { ok: true });
}
}
beforeAll(async () => {
boss = new PgBoss({
connectionString: process.env.DATABASE_URL,
supervise: false,
schedule: false,
});
await boss.start();
await boss.createQueue(SYSTEM_QUEUE);
await boss.createQueue(ACCOUNT_QUEUE);
}, 60_000);
afterAll(async () => {
await boss?.stop({ graceful: false, close: true });
await getPrismaClient().$executeRawUnsafe(
`DROP SCHEMA IF EXISTS pgboss CASCADE`,
);
});
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
await getPrismaClient().$executeRawUnsafe(`DELETE FROM pgboss.job`);
dispatched.calls.length = 0;
});
describe("notifySystemJobFailures", () => {
it("tells every admin once, reminds a day later, and announces the recovery once", async () => {
const admin = await makeUser({ role: "ADMIN" });
const second = await makeUser({ role: "ADMIN" });
await makeUser({ role: "USER" });
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK; i++) {
await runOnce(SYSTEM_QUEUE, "fail");
}
const now = new Date();
const first = await notifySystemJobFailures(now);
expect(first.failing).toBe(1);
expect(first.sent).toEqual([{ queue: SYSTEM_QUEUE, kind: "problem" }]);
expect(dispatched.calls.map((c) => c.userId).sort()).toEqual(
[admin.id, second.id].sort(),
);
const call = dispatched.calls[0];
expect(call.eventType).toBe("SYSTEM_ALERT");
expect(call.titleKey).toBe("notifications.systemJobAlert.titleFailing");
expect(call.params).toMatchObject({
queue: SYSTEM_QUEUE,
failures: SYSTEM_JOB_FAILURE_STREAK,
});
expect(String(call.params?.error)).toContain("statement timeout");
// The next hour: still failing, already told.
dispatched.calls.length = 0;
const again = await notifySystemJobFailures(
new Date(now.getTime() + HOUR_MS),
);
expect(again.sent).toEqual([]);
expect(dispatched.calls).toEqual([]);
// A day later: one reminder.
const reminder = await notifySystemJobFailures(
new Date(now.getTime() + 25 * HOUR_MS),
);
expect(reminder.sent).toEqual([{ queue: SYSTEM_QUEUE, kind: "reminder" }]);
// It runs again: one recovery notice, then silence.
await runOnce(SYSTEM_QUEUE, "complete");
dispatched.calls.length = 0;
const recovered = await notifySystemJobFailures(
new Date(now.getTime() + 26 * HOUR_MS),
);
expect(recovered.sent).toEqual([
{ queue: SYSTEM_QUEUE, kind: "recovered" },
]);
expect(dispatched.calls[0].titleKey).toBe(
"notifications.systemJobAlert.titleRecovered",
);
const quiet = await notifySystemJobFailures(
new Date(now.getTime() + 27 * HOUR_MS),
);
expect(quiet.sent).toEqual([]);
});
it("stays quiet below the streak and for queues with an account-facing surface", async () => {
await makeUser({ role: "ADMIN" });
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK - 1; i++) {
await runOnce(SYSTEM_QUEUE, "fail");
}
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK + 2; i++) {
await runOnce(ACCOUNT_QUEUE, "fail");
}
const outcome = await notifySystemJobFailures();
expect(outcome.failing).toBe(0);
expect(outcome.sent).toEqual([]);
expect(dispatched.calls).toEqual([]);
});
it("counts a success between failures as the end of the streak", async () => {
await makeUser({ role: "ADMIN" });
await runOnce(SYSTEM_QUEUE, "fail");
await runOnce(SYSTEM_QUEUE, "fail");
await runOnce(SYSTEM_QUEUE, "complete");
await runOnce(SYSTEM_QUEUE, "fail");
const outcome = await notifySystemJobFailures();
expect(outcome.failing).toBe(0);
expect(outcome.sent).toEqual([]);
});
});