mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-16 07:13:29 +00:00
fix(jobs): step over a briefing night that attempted nobody
A night on which every account was skipped, for instance because every provider was paused for background work, wrote all_failed false. The repeated-failure alert read that as a successful run: it ended the streak and announced a recovery while the provider was still down, and nights then alternated between failed and paused so the streak never reached the alert again. The run now also records run_attempted. A completed run with run_attempted 0 is neither a failure nor a recovery: the alert and readConsecutiveRunFailures step over it. Refs #1384
This commit is contained in:
1 parent
75f8a648dc
commit
5152e68586
9 files changed
+62
-7
No files matched your search
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
|
||||
"count": 84,
|
||||
"total": 93816,
|
||||
"total": 93819,
|
||||
"entries": {
|
||||
"app/medications/page-client.tsx": 842,
|
||||
"components/charts/mood-chart.tsx": 1166,
|
||||
@@ -64,7 +64,7 @@
|
||||
"lib/insights/mood-aggregates.ts": 1053,
|
||||
"lib/integrations/status.ts": 1448,
|
||||
"lib/jobs/coach-nudge.ts": 1084,
|
||||
"lib/jobs/insight-pregenerate.ts": 1189,
|
||||
"lib/jobs/insight-pregenerate.ts": 1192,
|
||||
"lib/jobs/medication-intake-import.ts": 819,
|
||||
"lib/jobs/offhost-backup.ts": 1094,
|
||||
"lib/jobs/reaction-line.ts": 803,
|
||||
|
||||
@@ -1717,6 +1717,9 @@ describe("runInsightPregenerate — a provider that cannot serve is not a failin
|
||||
expect(result.skipped).toBe(1);
|
||||
expect(result.failed).toBe(0);
|
||||
expect(result.allFailed).toBe(false);
|
||||
// Nobody attempted: the handler then writes no `all_failed`, so a benched
|
||||
// night neither extends nor ends the alert's streak.
|
||||
expect(result.attempted).toBe(0);
|
||||
expect(result.firstFailure).toBeNull();
|
||||
// The retry would find the same bench.
|
||||
expect(enqueueRetry).not.toHaveBeenCalled();
|
||||
@@ -1742,6 +1745,7 @@ describe("runInsightPregenerate — a provider that cannot serve is not a failin
|
||||
}),
|
||||
);
|
||||
expect(result.allFailed).toBe(false);
|
||||
expect(result.attempted).toBe(1);
|
||||
expect(enqueueRetry).not.toHaveBeenCalled();
|
||||
expect(annotateSpy).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
|
||||
@@ -216,12 +216,13 @@ export interface PregenerateRunResult {
|
||||
/** The job's time budget ran out before every candidate was reached. */
|
||||
stoppedEarly: boolean;
|
||||
/**
|
||||
* v1.42 — every candidate the run attempted failed (see `reportJobRun`).
|
||||
* Written into the job's facts as `all_failed`, which is how a completed
|
||||
* night with nothing generated still counts toward the repeated-failure
|
||||
* streak.
|
||||
* v1.42 — every candidate the run attempted failed (see `reportJobRun`),
|
||||
* written into the job's facts as `all_failed` so a completed night with
|
||||
* nothing generated counts toward the repeated-failure streak. `attempted`
|
||||
* (#1384) goes out as `run_attempted`; at 0 the streak steps over the night.
|
||||
*/
|
||||
allFailed: boolean;
|
||||
attempted: number;
|
||||
/**
|
||||
* Count of per-metric assessment caches written warm across the whole
|
||||
* run. The comprehensive generator evicts the seven `*-status`
|
||||
@@ -559,6 +560,7 @@ export async function runInsightPregenerate(
|
||||
budgetBlocked: 0,
|
||||
stoppedEarly: false,
|
||||
allFailed: false,
|
||||
attempted: 0,
|
||||
assessmentsWarmed: 0,
|
||||
metricAssessmentsWarmed: 0,
|
||||
firstFailure: null,
|
||||
@@ -626,6 +628,7 @@ export async function runInsightPregenerate(
|
||||
candidates: runOutcomes,
|
||||
});
|
||||
result.allFailed = verdict.allFailed;
|
||||
result.attempted = verdict.attempted;
|
||||
result.firstFailure = firstFailure(runOutcomes);
|
||||
|
||||
return result;
|
||||
|
||||
@@ -497,6 +497,11 @@ export async function readConsecutiveRunFailures(
|
||||
AND state IN ('completed', 'failed')
|
||||
AND completed_on IS NOT NULL
|
||||
AND (state = 'failed' OR output->'did'->>'all_failed' IS NOT NULL)
|
||||
-- #1384: a run that attempted nobody neither fails nor recovers.
|
||||
AND NOT (
|
||||
state = 'completed'
|
||||
AND COALESCE(output->'did'->>'run_attempted', '') = '0'
|
||||
)
|
||||
ORDER BY completed_on DESC
|
||||
LIMIT ${CONSECUTIVE_LOOKBACK_ROWS}
|
||||
`;
|
||||
|
||||
@@ -244,6 +244,9 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
|
||||
"removed",
|
||||
"resting_corrected",
|
||||
"rows_written",
|
||||
// #1384 — how many accounts a fan-out pass attempted; 0 marks a run that
|
||||
// neither failed nor recovered, which the failure streak steps over.
|
||||
"run_attempted",
|
||||
// v1.39.1 — the background restore of a stored backup, and the sweep that
|
||||
// re-queues one a stopped worker left running. Counts and a claim flag.
|
||||
"restore_claimed",
|
||||
|
||||
@@ -201,6 +201,11 @@ export async function handleInsightPregenerateJob(
|
||||
// v1.42 — read back by `readConsecutiveRunFailures`, so a night that
|
||||
// completed with nothing generated counts toward the failure streak.
|
||||
all_failed: summary.allFailed,
|
||||
// #1384 — a night that attempted no account (every provider paused)
|
||||
// is neither a failure nor a recovery. `run_attempted: 0` is how the
|
||||
// streak readers (`readConsecutiveRunFailures`, the repeated-failure
|
||||
// alert) know to step over it instead of ending the streak.
|
||||
run_attempted: summary.attempted,
|
||||
assessments_warmed: summary.assessmentsWarmed,
|
||||
metric_assessments_warmed: summary.metricAssessmentsWarmed,
|
||||
// #1384 — what failed, for the admin page and the alert ("none"
|
||||
|
||||
@@ -197,6 +197,12 @@ export async function readTerminalRuns(
|
||||
WHERE name = q.name
|
||||
AND completed_on IS NOT NULL
|
||||
AND state IN ('completed', 'failed')
|
||||
-- #1384: a fan-out run that attempted nobody neither fails nor
|
||||
-- recovers; it must not end a streak.
|
||||
AND NOT (
|
||||
state = 'completed'
|
||||
AND COALESCE(output->'did'->>'run_attempted', '') = '0'
|
||||
)
|
||||
ORDER BY completed_on DESC
|
||||
LIMIT ${LOOKBACK_ROWS}
|
||||
) j
|
||||
|
||||
@@ -268,6 +268,19 @@ describe("readConsecutiveRunFailures (v1.42)", () => {
|
||||
await expect(readConsecutiveRunFailures(STREAK_QUEUE)).resolves.toBe(3);
|
||||
});
|
||||
|
||||
it("steps over a night that attempted nobody instead of ending the streak (#1384)", async () => {
|
||||
await boss.createQueue(STREAK_QUEUE);
|
||||
// Failed, then every provider paused (nothing attempted), then failed.
|
||||
await finishedMinutesAgo(await completeOneJob({ all_failed: true }), 30);
|
||||
await finishedMinutesAgo(
|
||||
await completeOneJob({ all_failed: false, run_attempted: 0 }),
|
||||
20,
|
||||
);
|
||||
await finishedMinutesAgo(await completeOneJob({ all_failed: true }), 10);
|
||||
|
||||
await expect(readConsecutiveRunFailures(STREAK_QUEUE)).resolves.toBe(3);
|
||||
});
|
||||
|
||||
it("answers 1 for a queue with no history: the run failing now", async () => {
|
||||
await expect(readConsecutiveRunFailures("never-ran")).resolves.toBe(1);
|
||||
});
|
||||
|
||||
@@ -62,6 +62,7 @@ async function runOnce(
|
||||
outcome: "fail" | "complete",
|
||||
userId = "u",
|
||||
retryLimit = 0,
|
||||
output: Record<string, unknown> = { ok: true },
|
||||
) {
|
||||
const id = await boss.send(
|
||||
queue,
|
||||
@@ -80,7 +81,7 @@ async function runOnce(
|
||||
new Error(`canceling statement due to statement timeout (${userId})`),
|
||||
);
|
||||
} else {
|
||||
await boss.complete(queue, job.id, { ok: true });
|
||||
await boss.complete(queue, job.id, output);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -193,6 +194,21 @@ describe("notifySystemJobFailures", () => {
|
||||
expect(outcome.sent).toEqual([]);
|
||||
});
|
||||
|
||||
it("lets a run that attempted nobody neither end nor extend the streak (#1384)", async () => {
|
||||
await makeUser({ role: "ADMIN" });
|
||||
// What the briefing pass writes on a night every provider was paused.
|
||||
const benched = {
|
||||
ok: true,
|
||||
did: { all_failed: false, run_attempted: 0, skipped: 1 },
|
||||
};
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
await runOnce(SYSTEM_QUEUE, "complete", "u", 0, benched);
|
||||
expect((await notifySystemJobFailures()).failing).toBe(0);
|
||||
await runOnce(SYSTEM_QUEUE, "fail");
|
||||
expect((await notifySystemJobFailures()).failing).toBe(1);
|
||||
});
|
||||
|
||||
it("counts a job that used up its retries as one failure", async () => {
|
||||
await makeUser({ role: "ADMIN" });
|
||||
await runOnce(SYSTEM_QUEUE, "fail", "u", 2);
|
||||
|
||||
Reference in new issue
Block a user