diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index 8511c398b..f736accfa 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 84, - "total": 93816, + "total": 93819, "entries": { "app/medications/page-client.tsx": 842, "components/charts/mood-chart.tsx": 1166, @@ -64,7 +64,7 @@ "lib/insights/mood-aggregates.ts": 1053, "lib/integrations/status.ts": 1448, "lib/jobs/coach-nudge.ts": 1084, - "lib/jobs/insight-pregenerate.ts": 1189, + "lib/jobs/insight-pregenerate.ts": 1192, "lib/jobs/medication-intake-import.ts": 819, "lib/jobs/offhost-backup.ts": 1094, "lib/jobs/reaction-line.ts": 803, diff --git a/src/lib/jobs/__tests__/insight-pregenerate.test.ts b/src/lib/jobs/__tests__/insight-pregenerate.test.ts index f889746eb..667f9dca3 100644 --- a/src/lib/jobs/__tests__/insight-pregenerate.test.ts +++ b/src/lib/jobs/__tests__/insight-pregenerate.test.ts @@ -1717,6 +1717,9 @@ describe("runInsightPregenerate — a provider that cannot serve is not a failin expect(result.skipped).toBe(1); expect(result.failed).toBe(0); expect(result.allFailed).toBe(false); + // Nobody attempted: the handler then writes no `all_failed`, so a benched + // night neither extends nor ends the alert's streak. + expect(result.attempted).toBe(0); expect(result.firstFailure).toBeNull(); // The retry would find the same bench. expect(enqueueRetry).not.toHaveBeenCalled(); @@ -1742,6 +1745,7 @@ describe("runInsightPregenerate — a provider that cannot serve is not a failin }), ); expect(result.allFailed).toBe(false); + expect(result.attempted).toBe(1); expect(enqueueRetry).not.toHaveBeenCalled(); expect(annotateSpy).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/src/lib/jobs/insight-pregenerate.ts b/src/lib/jobs/insight-pregenerate.ts index 5c09b727c..0c3fb8a57 100644 --- a/src/lib/jobs/insight-pregenerate.ts +++ b/src/lib/jobs/insight-pregenerate.ts @@ -216,12 +216,13 @@ export interface PregenerateRunResult { /** The job's time budget ran out before every candidate was reached. */ stoppedEarly: boolean; /** - * v1.42 — every candidate the run attempted failed (see `reportJobRun`). - * Written into the job's facts as `all_failed`, which is how a completed - * night with nothing generated still counts toward the repeated-failure - * streak. + * v1.42 — every candidate the run attempted failed (see `reportJobRun`), + * written into the job's facts as `all_failed` so a completed night with + * nothing generated counts toward the repeated-failure streak. `attempted` + * (#1384) goes out as `run_attempted`; at 0 the streak steps over the night. */ allFailed: boolean; + attempted: number; /** * Count of per-metric assessment caches written warm across the whole * run. The comprehensive generator evicts the seven `*-status` @@ -559,6 +560,7 @@ export async function runInsightPregenerate( budgetBlocked: 0, stoppedEarly: false, allFailed: false, + attempted: 0, assessmentsWarmed: 0, metricAssessmentsWarmed: 0, firstFailure: null, @@ -626,6 +628,7 @@ export async function runInsightPregenerate( candidates: runOutcomes, }); result.allFailed = verdict.allFailed; + result.attempted = verdict.attempted; result.firstFailure = firstFailure(runOutcomes); return result; diff --git a/src/lib/jobs/job-failures.ts b/src/lib/jobs/job-failures.ts index b96cfd929..5ee93e195 100644 --- a/src/lib/jobs/job-failures.ts +++ b/src/lib/jobs/job-failures.ts @@ -497,6 +497,11 @@ export async function readConsecutiveRunFailures( AND state IN ('completed', 'failed') AND completed_on IS NOT NULL AND (state = 'failed' OR output->'did'->>'all_failed' IS NOT NULL) + -- #1384: a run that attempted nobody neither fails nor recovers. + AND NOT ( + state = 'completed' + AND COALESCE(output->'did'->>'run_attempted', '') = '0' + ) ORDER BY completed_on DESC LIMIT ${CONSECUTIVE_LOOKBACK_ROWS} `; diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index f47efcf38..d4baeae2f 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -244,6 +244,9 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "removed", "resting_corrected", "rows_written", + // #1384 — how many accounts a fan-out pass attempted; 0 marks a run that + // neither failed nor recovered, which the failure streak steps over. + "run_attempted", // v1.39.1 — the background restore of a stored backup, and the sweep that // re-queues one a stopped worker left running. Counts and a claim flag. "restore_claimed", diff --git a/src/lib/jobs/reminder/insights-handlers.ts b/src/lib/jobs/reminder/insights-handlers.ts index c34506f45..680c06973 100644 --- a/src/lib/jobs/reminder/insights-handlers.ts +++ b/src/lib/jobs/reminder/insights-handlers.ts @@ -201,6 +201,11 @@ export async function handleInsightPregenerateJob( // v1.42 — read back by `readConsecutiveRunFailures`, so a night that // completed with nothing generated counts toward the failure streak. all_failed: summary.allFailed, + // #1384 — a night that attempted no account (every provider paused) + // is neither a failure nor a recovery. `run_attempted: 0` is how the + // streak readers (`readConsecutiveRunFailures`, the repeated-failure + // alert) know to step over it instead of ending the streak. + run_attempted: summary.attempted, assessments_warmed: summary.assessmentsWarmed, metric_assessments_warmed: summary.metricAssessmentsWarmed, // #1384 — what failed, for the admin page and the alert ("none" diff --git a/src/lib/jobs/system-job-alert.ts b/src/lib/jobs/system-job-alert.ts index ff6187462..b5313fea2 100644 --- a/src/lib/jobs/system-job-alert.ts +++ b/src/lib/jobs/system-job-alert.ts @@ -197,6 +197,12 @@ export async function readTerminalRuns( WHERE name = q.name AND completed_on IS NOT NULL AND state IN ('completed', 'failed') + -- #1384: a fan-out run that attempted nobody neither fails nor + -- recovers; it must not end a streak. + AND NOT ( + state = 'completed' + AND COALESCE(output->'did'->>'run_attempted', '') = '0' + ) ORDER BY completed_on DESC LIMIT ${LOOKBACK_ROWS} ) j diff --git a/tests/integration/job-failures.test.ts b/tests/integration/job-failures.test.ts index 997c20f97..efe0206e9 100644 --- a/tests/integration/job-failures.test.ts +++ b/tests/integration/job-failures.test.ts @@ -268,6 +268,19 @@ describe("readConsecutiveRunFailures (v1.42)", () => { await expect(readConsecutiveRunFailures(STREAK_QUEUE)).resolves.toBe(3); }); + it("steps over a night that attempted nobody instead of ending the streak (#1384)", async () => { + await boss.createQueue(STREAK_QUEUE); + // Failed, then every provider paused (nothing attempted), then failed. + await finishedMinutesAgo(await completeOneJob({ all_failed: true }), 30); + await finishedMinutesAgo( + await completeOneJob({ all_failed: false, run_attempted: 0 }), + 20, + ); + await finishedMinutesAgo(await completeOneJob({ all_failed: true }), 10); + + await expect(readConsecutiveRunFailures(STREAK_QUEUE)).resolves.toBe(3); + }); + it("answers 1 for a queue with no history: the run failing now", async () => { await expect(readConsecutiveRunFailures("never-ran")).resolves.toBe(1); }); diff --git a/tests/integration/system-job-alert.test.ts b/tests/integration/system-job-alert.test.ts index dd9518143..8baeb05fe 100644 --- a/tests/integration/system-job-alert.test.ts +++ b/tests/integration/system-job-alert.test.ts @@ -62,6 +62,7 @@ async function runOnce( outcome: "fail" | "complete", userId = "u", retryLimit = 0, + output: Record = { ok: true }, ) { const id = await boss.send( queue, @@ -80,7 +81,7 @@ async function runOnce( new Error(`canceling statement due to statement timeout (${userId})`), ); } else { - await boss.complete(queue, job.id, { ok: true }); + await boss.complete(queue, job.id, output); return; } } @@ -193,6 +194,21 @@ describe("notifySystemJobFailures", () => { expect(outcome.sent).toEqual([]); }); + it("lets a run that attempted nobody neither end nor extend the streak (#1384)", async () => { + await makeUser({ role: "ADMIN" }); + // What the briefing pass writes on a night every provider was paused. + const benched = { + ok: true, + did: { all_failed: false, run_attempted: 0, skipped: 1 }, + }; + await runOnce(SYSTEM_QUEUE, "fail"); + await runOnce(SYSTEM_QUEUE, "fail"); + await runOnce(SYSTEM_QUEUE, "complete", "u", 0, benched); + expect((await notifySystemJobFailures()).failing).toBe(0); + await runOnce(SYSTEM_QUEUE, "fail"); + expect((await notifySystemJobFailures()).failing).toBe(1); + }); + it("counts a job that used up its retries as one failure", async () => { await makeUser({ role: "ADMIN" }); await runOnce(SYSTEM_QUEUE, "fail", "u", 2);