mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 07:13:29 +00:00
The nightly briefing pass settled an account as failed whenever its generation failed, including when every provider of its chain was benched for background work and no request was sent, and when the provider refused the account's own credential. A key revoked at its issuer stays saved and is refused every night, so a single-account instance read as all-failed night after night and the repeated-failure alert reported a broken job. - A chain that sent nothing settles as skipped (provider_benched), with no 45-minute retry. - A refused credential of the account's own settles as auth_failed, which the run reports as partial but does not fail over; a refusal of the operator's shared key still fails the run. - The generator's failed outcome names the provider that failed first, whether its credential was refused and whether anything was sent. - The run's facts carry failure_cause and failure_provider. The admin Background jobs card listed only pg-boss failures, while the alert also counts runs that completed with every account failed, so an operator sent to the card read that nothing had failed. The card now lists those runs, and the card and the alert describe them in words with the cause and provider instead of quoting the run's JSON. The warm tally counted a status card that fell back to its deterministic text as a warmed assessment; it now counts model-written notes only. Refs #1384
295 lines
11 KiB
TypeScript
295 lines
11 KiB
TypeScript
/**
|
|
* #646 — a broken background job says its own name.
|
|
*
|
|
* The environment fetch threw on every run for the whole life of the module.
|
|
* It was logged and counted, and neither reached a screen: `recordError()`
|
|
* increments one unnamed integer, and the worker log only exists where someone
|
|
* is tailing the container. The queue held the answer the entire time.
|
|
*
|
|
* These readers query `pgboss.job` directly, so the only test worth having is
|
|
* one against a schema pg-boss itself created. A hand-written `CREATE TABLE`
|
|
* would prove the query matches the table the test wrote, which is exactly the
|
|
* mistake being fixed. So pg-boss provisions its own schema here and the jobs
|
|
* are failed through its own API.
|
|
*/
|
|
import { PgBoss } from "pg-boss";
|
|
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
|
|
|
import {
|
|
readActiveJobsStartedBefore,
|
|
readConsecutiveRunFailures,
|
|
readFailingQueues,
|
|
readQueueFailureForUser,
|
|
} from "@/lib/jobs/job-failures";
|
|
|
|
import { getPrismaClient } from "./setup";
|
|
|
|
const QUEUE = "environment-fetch";
|
|
const OTHER_QUEUE = "reminder-check";
|
|
|
|
let boss: PgBoss;
|
|
|
|
/** Send a job, take it, and fail it terminally so it lands in `state='failed'`. */
|
|
async function failOneJob(
|
|
queue: string,
|
|
data: object,
|
|
error: string,
|
|
expireInSeconds?: number,
|
|
) {
|
|
const jobId = await boss.send(queue, data, {
|
|
retryLimit: 0,
|
|
...(expireInSeconds === undefined ? {} : { expireInSeconds }),
|
|
});
|
|
if (!jobId) throw new Error("queue rejected the job");
|
|
const [job] = await boss.fetch(queue);
|
|
if (!job) throw new Error("nothing to fetch");
|
|
await boss.fail(queue, job.id, new Error(error));
|
|
return jobId;
|
|
}
|
|
|
|
async function startBoss(): Promise<void> {
|
|
boss = new PgBoss({
|
|
connectionString: process.env.DATABASE_URL,
|
|
// No maintenance or monitoring timers — the test drives every transition.
|
|
supervise: false,
|
|
schedule: false,
|
|
});
|
|
await boss.start();
|
|
await boss.createQueue(QUEUE);
|
|
await boss.createQueue(OTHER_QUEUE);
|
|
}
|
|
|
|
beforeAll(startBoss, 60_000);
|
|
|
|
// Every case reads the queue it filled itself: no job a case before it sent,
|
|
// failed or left active is still there to be counted or fetched.
|
|
beforeEach(async () => {
|
|
await getPrismaClient().$executeRawUnsafe(`DELETE FROM pgboss.job`);
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await boss?.stop({ graceful: false, close: true });
|
|
await getPrismaClient().$executeRawUnsafe(
|
|
`DROP SCHEMA IF EXISTS pgboss CASCADE`,
|
|
);
|
|
});
|
|
|
|
describe("readFailingQueues", () => {
|
|
it("names the queue, counts the failures, and carries the last error", async () => {
|
|
await failOneJob(
|
|
QUEUE,
|
|
{ userId: "user-a" },
|
|
'type "public.EnvironmentLocationSource" does not exist',
|
|
);
|
|
await failOneJob(QUEUE, { userId: "user-b" }, "upstream feed unavailable");
|
|
await failOneJob(OTHER_QUEUE, {}, "reminder dispatch failed");
|
|
|
|
const failing = await readFailingQueues();
|
|
expect(failing).not.toBeNull();
|
|
|
|
const environment = failing?.find((entry) => entry.queue === QUEUE);
|
|
expect(environment?.failures).toBe(2);
|
|
expect(environment?.lastError).toBe("upstream feed unavailable");
|
|
expect(Number.isFinite(Date.parse(environment?.lastFailedAt ?? ""))).toBe(
|
|
true,
|
|
);
|
|
|
|
const other = failing?.find((entry) => entry.queue === OTHER_QUEUE);
|
|
expect(other?.failures).toBe(1);
|
|
expect(other?.lastError).toBe("reminder dispatch failed");
|
|
});
|
|
|
|
it("says what a bare timeout means, with the job's own limit (#1031)", async () => {
|
|
await getPrismaClient().$executeRawUnsafe(
|
|
`DELETE FROM pgboss.job WHERE state = 'failed'`,
|
|
);
|
|
// pg-boss writes these words both for a pass too slow for its limit and
|
|
// for a job whose process died under it; alone they sent an operator
|
|
// nowhere.
|
|
await failOneJob(QUEUE, {}, "job timed out", 900);
|
|
await failOneJob(OTHER_QUEUE, {}, "job timed out", 7200);
|
|
|
|
const failing = await readFailingQueues();
|
|
const fifteen = failing?.find((entry) => entry.queue === QUEUE);
|
|
expect(fifteen?.lastError).toMatch(
|
|
/^job timed out: did not finish within its 15 min limit\./,
|
|
);
|
|
expect(fifteen?.lastError).toMatch(/restarted/);
|
|
const twoHours = failing?.find((entry) => entry.queue === OTHER_QUEUE);
|
|
expect(twoHours?.lastError).toMatch(/within its 2 h limit/);
|
|
});
|
|
|
|
it("reports an empty list — not null — when nothing is failing", async () => {
|
|
await getPrismaClient().$executeRawUnsafe(
|
|
`DELETE FROM pgboss.job WHERE state = 'failed'`,
|
|
);
|
|
await expect(readFailingQueues()).resolves.toEqual([]);
|
|
});
|
|
|
|
// #1384 — the repeated-failure alert counts a nightly pass that completed
|
|
// with every account failed, and sent the operator to this page, which
|
|
// listed only pg-boss failures and said nothing had failed.
|
|
it("lists a run that completed with every account failed, in words, and not a good run", async () => {
|
|
const ALL_FAILED_QUEUE = "insight-pregenerate";
|
|
await boss.createQueue(ALL_FAILED_QUEUE);
|
|
async function completeOne(did: Record<string, unknown>) {
|
|
const jobId = await boss.send(ALL_FAILED_QUEUE, {}, { retryLimit: 0 });
|
|
const [job] = await boss.fetch(ALL_FAILED_QUEUE);
|
|
if (!job || job.id !== jobId) throw new Error("nothing to fetch");
|
|
await boss.complete(ALL_FAILED_QUEUE, job.id, { ok: true, did });
|
|
}
|
|
// A good night, and the night from the report: one account, one failure.
|
|
await completeOne({ total: 1, generated: 1, failed: 0, all_failed: false });
|
|
await completeOne({
|
|
total: 1,
|
|
generated: 0,
|
|
failed: 1,
|
|
all_failed: true,
|
|
assessments_warmed: 0,
|
|
failure_cause: "all_providers_failed",
|
|
failure_provider: "openai_compatible",
|
|
});
|
|
|
|
const failing = await readFailingQueues();
|
|
const entry = failing?.find((e) => e.queue === ALL_FAILED_QUEUE);
|
|
expect(entry?.failures).toBe(1);
|
|
expect(entry?.lastError).toBe(
|
|
"The run completed, but every account it attempted failed (1 of 1). " +
|
|
"First cause: all_providers_failed from provider openai_compatible. " +
|
|
"Accounts without a usable provider are skipped and not counted; " +
|
|
"job.run.failed in the app log lists each account's outcome.",
|
|
);
|
|
});
|
|
|
|
it("ignores a failure older than the window", async () => {
|
|
await failOneJob(QUEUE, { userId: "user-a" }, "old failure");
|
|
await getPrismaClient().$executeRawUnsafe(
|
|
`UPDATE pgboss.job SET completed_on = now() - interval '10 days' WHERE state = 'failed'`,
|
|
);
|
|
await expect(readFailingQueues(72)).resolves.toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe("readQueueFailureForUser", () => {
|
|
it("matches on the userId the job payload carries", async () => {
|
|
await getPrismaClient().$executeRawUnsafe(
|
|
`DELETE FROM pgboss.job WHERE state = 'failed'`,
|
|
);
|
|
await failOneJob(QUEUE, { userId: "user-a" }, "first");
|
|
await failOneJob(QUEUE, { userId: "user-a" }, "second");
|
|
await failOneJob(QUEUE, { userId: "user-b" }, "someone else");
|
|
|
|
const mine = await readQueueFailureForUser(QUEUE, "user-a");
|
|
expect(mine?.failures).toBe(2);
|
|
expect(Number.isFinite(Date.parse(mine?.lastFailedAt ?? ""))).toBe(true);
|
|
|
|
// No error text on this side — the message is written for an operator.
|
|
expect(Object.keys(mine ?? {}).sort()).toEqual([
|
|
"failures",
|
|
"lastFailedAt",
|
|
]);
|
|
});
|
|
|
|
it("answers null for an account whose jobs did not fail", async () => {
|
|
await expect(
|
|
readQueueFailureForUser(QUEUE, "user-with-no-failures"),
|
|
).resolves.toBeNull();
|
|
});
|
|
|
|
it("does not attribute another queue's failure to this one", async () => {
|
|
await getPrismaClient().$executeRawUnsafe(
|
|
`DELETE FROM pgboss.job WHERE state = 'failed'`,
|
|
);
|
|
await failOneJob(OTHER_QUEUE, { userId: "user-a" }, "reminder failed");
|
|
await expect(readQueueFailureForUser(QUEUE, "user-a")).resolves.toBeNull();
|
|
});
|
|
});
|
|
|
|
describe("readActiveJobsStartedBefore (#1031)", () => {
|
|
it("lists a job still marked active from before the given instant, and nothing after it", async () => {
|
|
const jobId = await boss.send(
|
|
OTHER_QUEUE,
|
|
{},
|
|
{ retryLimit: 0, expireInSeconds: 7200 },
|
|
);
|
|
const [job] = await boss.fetch(OTHER_QUEUE);
|
|
expect(job?.id).toBe(jobId);
|
|
|
|
const cutOff = await readActiveJobsStartedBefore(
|
|
new Date(Date.now() + 60_000),
|
|
);
|
|
expect(cutOff).toContainEqual(
|
|
expect.objectContaining({
|
|
queue: OTHER_QUEUE,
|
|
id: jobId,
|
|
expireSeconds: 7200,
|
|
}),
|
|
);
|
|
await expect(
|
|
readActiveJobsStartedBefore(new Date(Date.now() - 60 * 60_000)),
|
|
).resolves.toEqual([]);
|
|
|
|
await boss.complete(OTHER_QUEUE, jobId!);
|
|
});
|
|
});
|
|
|
|
describe("readConsecutiveRunFailures (v1.42)", () => {
|
|
const STREAK_QUEUE = "insight-pregenerate";
|
|
|
|
/** Complete one job with the facts a handler returned. */
|
|
async function completeOneJob(did: Record<string, unknown>) {
|
|
const jobId = await boss.send(STREAK_QUEUE, {}, { retryLimit: 0 });
|
|
const [job] = await boss.fetch(STREAK_QUEUE);
|
|
if (!job || job.id !== jobId) throw new Error("nothing to fetch");
|
|
await boss.complete(STREAK_QUEUE, job.id, { ok: true, did });
|
|
return jobId!;
|
|
}
|
|
|
|
/** Pin the finishing order so the test does not race the clock. */
|
|
async function finishedMinutesAgo(jobId: string, minutes: number) {
|
|
await getPrismaClient().$executeRawUnsafe(
|
|
`UPDATE pgboss.job SET completed_on = now() - make_interval(mins => ${minutes}) WHERE id = '${jobId}'`,
|
|
);
|
|
}
|
|
|
|
it("counts failed runs and all-failed nights, skips runs that do not say, and stops at a good night", async () => {
|
|
await boss.createQueue(STREAK_QUEUE);
|
|
// Oldest first: a good night, then a pg-boss failure, an on-demand job
|
|
// that reports nothing about the run, and an all-failed night.
|
|
await finishedMinutesAgo(await completeOneJob({ all_failed: false }), 40);
|
|
await finishedMinutesAgo(
|
|
await failOneJob(STREAK_QUEUE, {}, "worker died"),
|
|
30,
|
|
);
|
|
await finishedMinutesAgo(await completeOneJob({ forced: 1 }), 20);
|
|
await finishedMinutesAgo(await completeOneJob({ all_failed: true }), 10);
|
|
|
|
// Two earlier failures plus the run failing now.
|
|
await expect(readConsecutiveRunFailures(STREAK_QUEUE)).resolves.toBe(3);
|
|
});
|
|
|
|
it("answers 1 for a queue with no history: the run failing now", async () => {
|
|
await expect(readConsecutiveRunFailures("never-ran")).resolves.toBe(1);
|
|
});
|
|
});
|
|
|
|
describe("no queue schema at all", () => {
|
|
it("answers null rather than throwing, so the status page still renders", async () => {
|
|
const prisma = getPrismaClient();
|
|
await boss.stop({ graceful: false, close: true });
|
|
await prisma.$executeRawUnsafe(`ALTER SCHEMA pgboss RENAME TO pgboss_gone`);
|
|
try {
|
|
await expect(readFailingQueues()).resolves.toBeNull();
|
|
await expect(
|
|
readQueueFailureForUser(QUEUE, "user-a"),
|
|
).resolves.toBeNull();
|
|
} finally {
|
|
await prisma.$executeRawUnsafe(
|
|
`ALTER SCHEMA pgboss_gone RENAME TO pgboss`,
|
|
);
|
|
// Running again for whichever case comes next.
|
|
await startBoss();
|
|
}
|
|
});
|
|
});
|