mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 15:23:29 +00:00
A night on which every account was skipped, for instance because every provider was paused for background work, wrote all_failed false. The repeated-failure alert read that as a successful run: it ended the streak and announced a recovery while the provider was still down, and nights then alternated between failed and paused so the streak never reached the alert again. The run now also records run_attempted. A completed run with run_attempted 0 is neither a failure nor a recovery: the alert and readConsecutiveRunFailures step over it. Refs #1384
266 lines
8.8 KiB
TypeScript
266 lines
8.8 KiB
TypeScript
/**
|
|
* The system job alert against a ledger pg-boss itself wrote (#1298).
|
|
*
|
|
* A background job of the instance that keeps failing pushes every admin
|
|
* once, a reminder at most once a day while it keeps failing, and once when it
|
|
* runs successfully again. Fewer failures than the streak, and queues that
|
|
* have an account-facing surface of their own, push nothing. The streak
|
|
* counts jobs, not retries, and is kept per account: one account failing
|
|
* every hour beside accounts that succeed is one problem and one recovery,
|
|
* and a job that does not run again is announced once, not every day.
|
|
*
|
|
* pg-boss provisions its own schema here and the jobs are failed and completed
|
|
* through its own API, as in `job-failures.test.ts`, so the reader is proven
|
|
* against the table pg-boss writes rather than one the test wrote.
|
|
*/
|
|
import { PgBoss } from "pg-boss";
|
|
import {
|
|
afterAll,
|
|
beforeAll,
|
|
beforeEach,
|
|
describe,
|
|
expect,
|
|
it,
|
|
vi,
|
|
} from "vitest";
|
|
|
|
const dispatched = vi.hoisted(() => ({
|
|
calls: [] as Array<{
|
|
userId: string;
|
|
titleKey: string;
|
|
messageKey: string;
|
|
params?: Record<string, string | number>;
|
|
eventType?: string;
|
|
}>,
|
|
}));
|
|
|
|
vi.mock("@/lib/notifications/dispatch-localised", () => ({
|
|
dispatchLocalisedNotification: vi.fn(
|
|
async (opts: (typeof dispatched.calls)[number]) => {
|
|
dispatched.calls.push(opts);
|
|
},
|
|
),
|
|
}));
|
|
|
|
import {
|
|
notifySystemJobFailures,
|
|
SYSTEM_JOB_FAILURE_STREAK,
|
|
} from "@/lib/jobs/system-job-alert";
|
|
|
|
import { makeUser } from "../kit/factories";
|
|
|
|
import { getPrismaClient, truncateAllTables } from "./setup";
|
|
|
|
const SYSTEM_QUEUE = "rollup-full-backfill";
|
|
const ACCOUNT_QUEUE = "google-health-sync";
|
|
const HOUR_MS = 3_600_000;
|
|
|
|
let boss: PgBoss;
|
|
|
|
async function runOnce(
|
|
queue: string,
|
|
outcome: "fail" | "complete",
|
|
userId = "u",
|
|
retryLimit = 0,
|
|
output: Record<string, unknown> = { ok: true },
|
|
) {
|
|
const id = await boss.send(
|
|
queue,
|
|
{ userId },
|
|
{ retryLimit, retryDelay: 0, retryBackoff: false },
|
|
);
|
|
if (!id) throw new Error("queue rejected the job");
|
|
// Every attempt, retries included, until pg-boss settles the job.
|
|
for (let attempt = 0; attempt <= retryLimit; attempt++) {
|
|
const [job] = await boss.fetch(queue);
|
|
if (!job) throw new Error("nothing to fetch");
|
|
if (outcome === "fail") {
|
|
await boss.fail(
|
|
queue,
|
|
job.id,
|
|
new Error(`canceling statement due to statement timeout (${userId})`),
|
|
);
|
|
} else {
|
|
await boss.complete(queue, job.id, output);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
beforeAll(async () => {
|
|
boss = new PgBoss({
|
|
connectionString: process.env.DATABASE_URL,
|
|
supervise: false,
|
|
schedule: false,
|
|
});
|
|
await boss.start();
|
|
await boss.createQueue(SYSTEM_QUEUE);
|
|
await boss.createQueue(ACCOUNT_QUEUE);
|
|
}, 60_000);
|
|
|
|
afterAll(async () => {
|
|
await boss?.stop({ graceful: false, close: true });
|
|
await getPrismaClient().$executeRawUnsafe(
|
|
`DROP SCHEMA IF EXISTS pgboss CASCADE`,
|
|
);
|
|
});
|
|
|
|
beforeEach(async () => {
|
|
await truncateAllTables(getPrismaClient());
|
|
await getPrismaClient().$executeRawUnsafe(`DELETE FROM pgboss.job`);
|
|
dispatched.calls.length = 0;
|
|
});
|
|
|
|
describe("notifySystemJobFailures", () => {
|
|
it("tells every admin once, reminds a day later, and announces the recovery once", async () => {
|
|
const admin = await makeUser({ role: "ADMIN" });
|
|
const second = await makeUser({ role: "ADMIN" });
|
|
await makeUser({ role: "USER" });
|
|
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK; i++) {
|
|
await runOnce(SYSTEM_QUEUE, "fail");
|
|
}
|
|
const now = new Date();
|
|
|
|
const first = await notifySystemJobFailures(now);
|
|
expect(first.failing).toBe(1);
|
|
expect(first.sent).toEqual([{ queue: SYSTEM_QUEUE, kind: "problem" }]);
|
|
expect(dispatched.calls.map((c) => c.userId).sort()).toEqual(
|
|
[admin.id, second.id].sort(),
|
|
);
|
|
const call = dispatched.calls[0];
|
|
expect(call.eventType).toBe("SYSTEM_ALERT");
|
|
expect(call.titleKey).toBe("notifications.systemJobAlert.titleFailing");
|
|
expect(call.params).toMatchObject({
|
|
queue: SYSTEM_QUEUE,
|
|
failures: SYSTEM_JOB_FAILURE_STREAK,
|
|
});
|
|
expect(String(call.params?.error)).toContain("statement timeout");
|
|
|
|
// The next hour: still failing, already told.
|
|
dispatched.calls.length = 0;
|
|
const again = await notifySystemJobFailures(
|
|
new Date(now.getTime() + HOUR_MS),
|
|
);
|
|
expect(again.sent).toEqual([]);
|
|
expect(dispatched.calls).toEqual([]);
|
|
|
|
// It fails once more; a day after the notice: one reminder.
|
|
await runOnce(SYSTEM_QUEUE, "fail");
|
|
const reminder = await notifySystemJobFailures(
|
|
new Date(now.getTime() + 25 * HOUR_MS),
|
|
);
|
|
expect(reminder.sent).toEqual([{ queue: SYSTEM_QUEUE, kind: "reminder" }]);
|
|
|
|
// It runs again: one recovery notice, then silence.
|
|
await runOnce(SYSTEM_QUEUE, "complete");
|
|
dispatched.calls.length = 0;
|
|
const recovered = await notifySystemJobFailures(
|
|
new Date(now.getTime() + 26 * HOUR_MS),
|
|
);
|
|
expect(recovered.sent).toEqual([
|
|
{ queue: SYSTEM_QUEUE, kind: "recovered" },
|
|
]);
|
|
expect(dispatched.calls[0].titleKey).toBe(
|
|
"notifications.systemJobAlert.titleRecovered",
|
|
);
|
|
const quiet = await notifySystemJobFailures(
|
|
new Date(now.getTime() + 27 * HOUR_MS),
|
|
);
|
|
expect(quiet.sent).toEqual([]);
|
|
});
|
|
|
|
it("stays quiet below the streak and for queues with an account-facing surface", async () => {
|
|
await makeUser({ role: "ADMIN" });
|
|
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK - 1; i++) {
|
|
await runOnce(SYSTEM_QUEUE, "fail");
|
|
}
|
|
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK + 2; i++) {
|
|
await runOnce(ACCOUNT_QUEUE, "fail");
|
|
}
|
|
const outcome = await notifySystemJobFailures();
|
|
expect(outcome.failing).toBe(0);
|
|
expect(outcome.sent).toEqual([]);
|
|
expect(dispatched.calls).toEqual([]);
|
|
});
|
|
|
|
it("counts a success between failures as the end of the streak", async () => {
|
|
await makeUser({ role: "ADMIN" });
|
|
await runOnce(SYSTEM_QUEUE, "fail");
|
|
await runOnce(SYSTEM_QUEUE, "fail");
|
|
await runOnce(SYSTEM_QUEUE, "complete");
|
|
await runOnce(SYSTEM_QUEUE, "fail");
|
|
const outcome = await notifySystemJobFailures();
|
|
expect(outcome.failing).toBe(0);
|
|
expect(outcome.sent).toEqual([]);
|
|
});
|
|
|
|
it("lets a run that attempted nobody neither end nor extend the streak (#1384)", async () => {
|
|
await makeUser({ role: "ADMIN" });
|
|
// What the briefing pass writes on a night every provider was paused.
|
|
const benched = {
|
|
ok: true,
|
|
did: { all_failed: false, run_attempted: 0, skipped: 1 },
|
|
};
|
|
await runOnce(SYSTEM_QUEUE, "fail");
|
|
await runOnce(SYSTEM_QUEUE, "fail");
|
|
await runOnce(SYSTEM_QUEUE, "complete", "u", 0, benched);
|
|
expect((await notifySystemJobFailures()).failing).toBe(0);
|
|
await runOnce(SYSTEM_QUEUE, "fail");
|
|
expect((await notifySystemJobFailures()).failing).toBe(1);
|
|
});
|
|
|
|
it("counts a job that used up its retries as one failure", async () => {
|
|
await makeUser({ role: "ADMIN" });
|
|
await runOnce(SYSTEM_QUEUE, "fail", "u", 2);
|
|
const outcome = await notifySystemJobFailures();
|
|
expect(outcome.failing).toBe(0);
|
|
expect(outcome.sent).toEqual([]);
|
|
});
|
|
|
|
it("tells once about one account failing every hour beside accounts that succeed", async () => {
|
|
await makeUser({ role: "ADMIN" });
|
|
const start = Date.now();
|
|
const sent: string[] = [];
|
|
// Ten hours: account `a` fails every hour, the others succeed after it,
|
|
// so the queue's newest job is always a success.
|
|
for (let hour = 0; hour < 10; hour++) {
|
|
await runOnce(SYSTEM_QUEUE, "fail", "a");
|
|
await runOnce(SYSTEM_QUEUE, "complete", "b");
|
|
await runOnce(SYSTEM_QUEUE, "complete", "c");
|
|
const outcome = await notifySystemJobFailures(
|
|
new Date(start + hour * HOUR_MS),
|
|
);
|
|
sent.push(...outcome.sent.map((n) => n.kind));
|
|
}
|
|
expect(sent).toEqual(["problem"]);
|
|
expect(String(dispatched.calls[0].params?.error)).toContain("(a)");
|
|
|
|
// `a` succeeds: one recovery.
|
|
await runOnce(SYSTEM_QUEUE, "complete", "a");
|
|
const recovered = await notifySystemJobFailures(
|
|
new Date(start + 10 * HOUR_MS),
|
|
);
|
|
expect(recovered.sent).toEqual([
|
|
{ queue: SYSTEM_QUEUE, kind: "recovered" },
|
|
]);
|
|
});
|
|
|
|
it("does not remind about a job that has not run again since the notice", async () => {
|
|
await makeUser({ role: "ADMIN" });
|
|
for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK; i++) {
|
|
await runOnce(SYSTEM_QUEUE, "fail");
|
|
}
|
|
const now = new Date(Date.now() + 1_000);
|
|
expect((await notifySystemJobFailures(now)).sent).toEqual([
|
|
{ queue: SYSTEM_QUEUE, kind: "problem" },
|
|
]);
|
|
for (const days of [1, 2, 5]) {
|
|
const later = await notifySystemJobFailures(
|
|
new Date(now.getTime() + days * 24 * HOUR_MS),
|
|
);
|
|
expect(later.failing).toBe(1);
|
|
expect(later.sent).toEqual([]);
|
|
}
|
|
});
|
|
});
|