diff --git a/eslint-plugins/healthlog/__tests__/job-handler-outcome.test.mjs b/eslint-plugins/healthlog/__tests__/job-handler-outcome.test.mjs new file mode 100644 index 000000000..7d608ec10 --- /dev/null +++ b/eslint-plugins/healthlog/__tests__/job-handler-outcome.test.mjs @@ -0,0 +1,67 @@ +/** + * Unit tests for the `healthlog/job-handler-outcome` rule — the bare + * `boss.work(` ban, the one exempt module that owns `createAndWork`, the + * test-file exemption, and the scope limit to `src/lib/jobs/`. + * + * The `invalid` cases below also stand as the rule's own documented blind + * spots: an aliased or renamed receiver is not detected, and neither is a + * handler that returns `{ ok: true }` after swallowing. Those sit in `valid` + * on purpose, so a future reader can see what this rule does not claim. + */ +import { describe, it } from "vitest"; +import { RuleTester } from "eslint"; +import rule from "../job-handler-outcome.js"; + +RuleTester.describe = describe; +RuleTester.it = it; +RuleTester.itOnly = it.only; + +const ruleTester = new RuleTester({ + languageOptions: { ecmaVersion: 2023, sourceType: "module" }, +}); + +ruleTester.run("job-handler-outcome", rule, { + valid: [ + { + code: "await createAndWork(boss, QUEUE, { localConcurrency: 1 }, handleX);", + filename: "/repo/src/lib/jobs/reminder/register-status.ts", + }, + { + // The wrapper's own call — this module is the one legal caller. + code: "await boss.work(queue, options, runJob(queue, handler));", + filename: "/repo/src/lib/jobs/reminder/registrar-shared.ts", + }, + { + // Tests mock `boss.work` and assert against it directly. + code: 'boss.work("q", {}, async () => {});', + filename: "/repo/src/lib/jobs/__tests__/whoop-queues.test.ts", + }, + { + // Out of scope: not the worker tree. + code: 'boss.work("q", {}, async () => {});', + filename: "/repo/src/lib/integrations/status.ts", + }, + { + // Documented blind spot: an aliased receiver is not detected. + code: 'const b = boss; b.work("q", {}, async () => {});', + filename: "/repo/src/lib/jobs/reminder/register-rollup.ts", + }, + { + // Documented blind spot: this rule cannot see a swallowed failure. + code: "await createAndWork(boss, QUEUE, {}, async () => ({ ok: true, did: {} }));", + filename: "/repo/src/lib/jobs/reminder/register-rollup.ts", + }, + ], + invalid: [ + { + code: 'await boss.work("scratch", async () => {});', + filename: "/repo/src/lib/jobs/reminder/register-maintenance.ts", + errors: [{ messageId: "bareBossWork" }], + }, + { + code: "await boss.work(QUEUE, { localConcurrency: 1 }, handleX);", + filename: "/repo/src/lib/jobs/worker-status.ts", + errors: [{ messageId: "bareBossWork" }], + }, + ], +}); diff --git a/eslint-plugins/healthlog/index.js b/eslint-plugins/healthlog/index.js index d1529a8bc..8147b1540 100644 --- a/eslint-plugins/healthlog/index.js +++ b/eslint-plugins/healthlog/index.js @@ -8,6 +8,8 @@ * - `no-raw-palette-color` — ban raw Tailwind palette utilities in app UI. * - `spacing-scale` — no pt-/pb- overrides on gap-based Card slots, * and no off-scale `5` step on a bg-card shell. + * - `job-handler-outcome` — pg-boss bindings route through createAndWork, + * whose handler returns a JobOutcome. */ "use strict"; @@ -17,6 +19,7 @@ const safeFetchRequired = require("./safe-fetch-required.js"); const apiFetchRequired = require("./api-fetch-required.js"); const noRawPaletteColor = require("./no-raw-palette-color.js"); const spacingScale = require("./spacing-scale.js"); +const jobHandlerOutcome = require("./job-handler-outcome.js"); module.exports = { rules: { @@ -30,5 +33,6 @@ module.exports = { // named rule (now error-level too; the staged warn phase ended with // the semantic sweep). See the rule header. "no-dracula-utility": noRawPaletteColor, + "job-handler-outcome": jobHandlerOutcome, }, }; diff --git a/eslint-plugins/healthlog/job-handler-outcome.js b/eslint-plugins/healthlog/job-handler-outcome.js new file mode 100644 index 000000000..9dfdbafc4 --- /dev/null +++ b/eslint-plugins/healthlog/job-handler-outcome.js @@ -0,0 +1,107 @@ +/** + * @fileoverview ESLint rule — a pg-boss queue binding must go through + * `createAndWork`, never a bare `boss.work(`. + * + * `boss.work(queue, options, handler)` completes a job whenever the handler's + * promise resolves, whatever it resolves to. Every binding in this tree used + * to be written that way, so a handler that caught an error, logged a warning + * and returned looked exactly like one that did its work. `createAndWork` + * (`src/lib/jobs/reminder/registrar-shared.ts`) types its handler as + * `(jobs) => Promise` and wraps it in `runJob`, which fails the + * pg-boss job on `ok: false`. Routing every binding through it is what makes + * "the handler returned nothing" a compile error rather than a silent pass. + * + * WHAT THIS RULE PROVES, AND WHAT IT DOES NOT + * + * It proves that no queue in this tree is bound by a path that accepts a + * void handler. That is all it proves. It does NOT detect a handler that + * returns `{ ok: true }` after swallowing an exception — that compiles, it + * passes this rule, and it is exactly as invisible as the defect the outcome + * type was written for. The rule converts an omission (forgetting to signal + * failure) into a commission (claiming a success that did not happen). The + * second is a smaller class and a reviewable one; it is not zero. + * + * Two further evasions are known and unhandled: aliasing the method + * (`const w = boss.work.bind(boss)`, `const { work } = boss`) and calling + * `work` on an object not named `boss`. Both are syntactic dodges a reviewer + * would see in the diff; neither is defended against here. + * + * Scope: `src/lib/jobs/`, excluding the registrar module that owns the one + * legal `boss.work` call and excluding test files, which mock `boss.work` + * and assert against it directly. + * + * @see src/lib/jobs/run-job.ts + * @see src/lib/jobs/job-outcome.ts + * @see src/lib/jobs/reminder/registrar-shared.ts + */ + +"use strict"; + +// The one module allowed to call `boss.work` — `createAndWork` lives here +// and is the wrapper every other binding has to use. +const EXEMPT_FILES = ["src/lib/jobs/reminder/registrar-shared.ts"]; + +// The worker tree. A `boss.work` outside it would not be a queue binding. +const ENFORCED_ROOTS = ["src/lib/jobs/"]; + +function toPosix(filename) { + return filename.replace(/\\/g, "/"); +} + +function isTestFile(posix) { + return ( + /\.test\.[cm]?[jt]sx?$/.test(posix) || + /\.spec\.[cm]?[jt]sx?$/.test(posix) || + posix.includes("/__tests__/") || + posix.includes("/__mocks__/") + ); +} + +function isEnforced(filename) { + const posix = toPosix(filename); + if (!ENFORCED_ROOTS.some((root) => posix.includes(root))) return false; + if (EXEMPT_FILES.some((f) => posix.includes(f))) return false; + if (isTestFile(posix)) return false; + return true; +} + +/** @type {import("eslint").Rule.RuleModule} */ +const jobHandlerOutcomeRule = { + meta: { + type: "problem", + docs: { + description: + "pg-boss queue bindings must go through createAndWork so the handler returns a JobOutcome.", + }, + schema: [], + messages: { + bareBossWork: + 'boss.work() completes a job whenever the handler resolves, so a swallowed failure reads as success. Bind the queue with createAndWork(boss, QUEUE, options, handler) from "./registrar-shared" — its handler type is (jobs) => Promise and runJob fails the job on ok:false.', + }, + }, + create(context) { + const filename = context.filename ?? context.getFilename?.(); + if (!filename || !isEnforced(filename)) { + return {}; + } + + return { + CallExpression(node) { + const callee = node.callee; + if ( + callee.type !== "MemberExpression" || + callee.computed || + callee.property.type !== "Identifier" || + callee.property.name !== "work" || + callee.object.type !== "Identifier" || + callee.object.name !== "boss" + ) { + return; + } + context.report({ node, messageId: "bareBossWork" }); + }, + }; + }, +}; + +module.exports = jobHandlerOutcomeRule; diff --git a/eslint.config.mjs b/eslint.config.mjs index 96ad3e1fd..82fa8cfae 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -75,6 +75,13 @@ const eslintConfig = defineConfig([ // trip it. Density lives on the Card itself (`gap-2 py-3 md:py-4`) or on // a swept `p-4 md:p-6` shell. See the rule header. "healthlog/spacing-scale": "error", + // Every pg-boss queue binding under src/lib/jobs goes through + // `createAndWork`, whose handler type is `(jobs) => Promise`. + // A bare `boss.work(` accepts a handler that resolves to nothing, which + // is how a swallowed failure came to read as a completed job. The rule + // catches the omission only — a handler that returns `{ ok: true }` + // after swallowing still passes, and its header says so. + "healthlog/job-handler-outcome": "error", }, }, // v1.28.17 — every recharts-rendering component funnels through the diff --git a/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts b/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts index 13e240628..219978aae 100644 --- a/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts @@ -73,12 +73,16 @@ describe("periodic reconcile — wiring", () => { /const schedules[\s\S]*?=\s*\[([\s\S]*?)\];/, ); expect(schedules).not.toBeNull(); + // The third element pins the retry policy. It is load-bearing now that + // the handler reports a failed outcome instead of swallowing: without + // `cronIsTheRetry` a database outage would run the same doomed sweep + // three times per tick, and the next tick is already the retry. expect(schedules![1]).toMatch( - /\[IMPORT_JOB_RECONCILE_QUEUE,\s*IMPORT_JOB_RECONCILE_CRON\]/, + /\[IMPORT_JOB_RECONCILE_QUEUE,\s*IMPORT_JOB_RECONCILE_CRON,\s*cronIsTheRetry\]/, ); expect(maintenanceSource).toMatch( - /boss\.work\(\s*IMPORT_JOB_RECONCILE_QUEUE[\s\S]{0,120}handleImportJobReconcileTick/, + /createAndWork\(\s*boss,\s*IMPORT_JOB_RECONCILE_QUEUE[\s\S]{0,120}handleImportJobReconcileTick/, ); }); }); @@ -163,24 +167,30 @@ describe("reconcileOrphanImportJobs", () => { }); describe("handleImportJobReconcileTick", () => { - it("delegates to reconcileOrphanImportJobs and resolves on success", async () => { + it("delegates to reconcileOrphanImportJobs and reports a done outcome", async () => { mocks.findMany.mockResolvedValue([]); - await expect( - handleImportJobReconcileTick([] as never), - ).resolves.toBeUndefined(); + await expect(handleImportJobReconcileTick([] as never)).resolves.toEqual({ + ok: true, + did: {}, + }); expect(mocks.findMany).toHaveBeenCalledTimes(1); }); - it("swallows a reconcile failure instead of throwing", async () => { - mocks.findMany.mockRejectedValue(new Error("db unavailable")); + it("reports a failed outcome when the reconcile pass throws", async () => { + const cause = new Error("db unavailable"); + mocks.findMany.mockRejectedValue(cause); - // A background sweep this cheap must never spam pg-boss's - // retry/backoff machinery — it just re-runs on the next 15-minute - // tick. The handler must resolve even when the underlying reconcile - // pass throws. - await expect( - handleImportJobReconcileTick([] as never), - ).resolves.toBeUndefined(); + // The handler used to swallow this and resolve, which left a sweep + // that had not run for hours indistinguishable from one that found + // nothing to do. It now names the failure, `runJob` rethrows it, and + // pg-boss records a failed job. The queue carries `cronIsTheRetry` + // (retryLimit 0) because the next 15-minute tick is the retry — a + // database that is down stays down for an immediate re-run. + await expect(handleImportJobReconcileTick([] as never)).resolves.toEqual({ + ok: false, + reason: "apple health import reconcile failed", + cause, + }); }); }); diff --git a/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts b/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts index 5e5de5c6c..d28594276 100644 --- a/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts @@ -87,11 +87,14 @@ describe("apple health import — parser revision boundary", () => { expect(allQueues).not.toBeNull(); expect(allQueues![1]).toMatch(/\bAPPLE_HEALTH_IMPORT_V2_QUEUE\b/); expect(allQueues![1]).toMatch(/\bAPPLE_HEALTH_IMPORT_LEGACY_QUEUE\b/); + // The v2 window is the wider of the two: its binding wraps the batch in + // a loop that keeps the first failed outcome, so the handler call sits + // further from the queue name than the legacy bridge's does. expect(maintenanceSource).toMatch( - /boss\.work[\s\S]{0,200}APPLE_HEALTH_IMPORT_V2_QUEUE[\s\S]{0,400}handleAppleHealthImport/, + /createAndWork[\s\S]{0,200}APPLE_HEALTH_IMPORT_V2_QUEUE[\s\S]{0,600}handleAppleHealthImport/, ); expect(maintenanceSource).toMatch( - /boss\.work[\s\S]{0,200}APPLE_HEALTH_IMPORT_LEGACY_QUEUE[\s\S]{0,400}migrateLegacyAppleHealthImport/, + /createAndWork[\s\S]{0,200}APPLE_HEALTH_IMPORT_LEGACY_QUEUE[\s\S]{0,400}migrateLegacyAppleHealthImport/, ); }); diff --git a/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts b/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts index a8ad973da..bb6147eab 100644 --- a/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts +++ b/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts @@ -4,7 +4,7 @@ * * Same source-text-grep approach as the period-narrative / stress-strain * guards: assert the queue is imported, registered in `allQueues`, and wired - * to a `boss.work` handler that runs the refresh pipeline — without booting + * to a `createAndWork` handler that runs the refresh pipeline — without booting * pg-boss + Prisma. An unregistered queue silently never runs (the recurring * past bug this guards against; v1.4.37 W10 caught exactly this class). */ @@ -41,9 +41,9 @@ describe("reminder-worker — coach-memory-refresh wiring", () => { expect(allQueuesBlock()).toMatch(/\bCOACH_MEMORY_REFRESH_QUEUE\b/); }); - it("registers a boss.work handler that runs the refresh pipeline", () => { + it("registers a createAndWork handler that runs the refresh pipeline", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}COACH_MEMORY_REFRESH_QUEUE[\s\S]{0,400}runCoachMemoryRefresh/, + /createAndWork[\s\S]{0,200}COACH_MEMORY_REFRESH_QUEUE[\s\S]{0,600}runCoachMemoryRefresh/, ); }); }); diff --git a/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts b/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts index e00d24206..0567b4e40 100644 --- a/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts +++ b/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts @@ -2,7 +2,7 @@ * v1.18.7 — coach-message-cleanup queue registration guard. * * Same source-text-grep approach as the other queue-wiring guards: assert the - * queue is registered in `allQueues`, scheduled, and wired to a `boss.work` + * queue is registered in `allQueues`, scheduled, and wired to a `createAndWork` * handler — without booting pg-boss + Prisma. An unregistered queue silently * never drains (the recurring v1.4.37 dead-queue bug), which would leave the * encrypted Coach history growing unbounded with the retention job dark. @@ -41,13 +41,13 @@ describe("reminder-worker — coach-message-cleanup wiring", () => { it("schedules the coach-message-cleanup cron", () => { expect(workerSource).toMatch( - /\[COACH_MESSAGE_CLEANUP_QUEUE,\s*COACH_MESSAGE_CLEANUP_CRON\]/, + /\[COACH_MESSAGE_CLEANUP_QUEUE,\s*COACH_MESSAGE_CLEANUP_CRON,\s*cronIsTheRetry\]/, ); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(workerSource).toMatch( - /boss\.work[\s\S]{0,200}COACH_MESSAGE_CLEANUP_QUEUE[\s\S]{0,200}handleCoachMessageCleanup/, + /createAndWork[\s\S]{0,200}COACH_MESSAGE_CLEANUP_QUEUE[\s\S]{0,200}handleCoachMessageCleanup/, ); }); diff --git a/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts b/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts index 61aa8c699..18a28299b 100644 --- a/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts +++ b/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts @@ -3,7 +3,7 @@ * registration guard. * * Same source-text-grep approach as the step-consolidation-repair guard: - * assert the queue is registered in `allQueues`, a `boss.work` handler is + * assert the queue is registered in `allQueues`, a `createAndWork` handler is * wired against it, and the boot-discovery enqueue helper fires — without * booting pg-boss + Prisma. An unregistered queue silently never drains * (the v1.4.37 dead-queue class). @@ -34,9 +34,9 @@ describe("reminder-worker — cumulative-pr-rederive wiring", () => { expect(allQueuesMatch![1]).toMatch(/\bCUMULATIVE_PR_REDERIVE_QUEUE\b/); }); - it("registers a boss.work handler against the queue", () => { + it("registers a createAndWork handler against the queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}CUMULATIVE_PR_REDERIVE_QUEUE[\s\S]{0,400}runCumulativePrRederivationForUser/, + /createAndWork[\s\S]{0,200}CUMULATIVE_PR_REDERIVE_QUEUE[\s\S]{0,400}runCumulativePrRederivationForUser/, ); }); diff --git a/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts b/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts index e060711e6..8c9158af2 100644 --- a/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts +++ b/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts @@ -5,7 +5,7 @@ * consolidation registration guards: assert the cycle-reminder queue is * declared, listed in `allQueues` (the v1.4.37 dead-queue lesson — an * unregistered queue is never provisioned and the schedule silently - * no-ops), scheduled on its cron, and wired to a `boss.work` handler that + * no-ops), scheduled on its cron, and wired to a `createAndWork` handler that * delegates to `runCycleReminderTick`. No pg-boss / Prisma boot. */ import { readFileSync } from "node:fs"; @@ -57,9 +57,9 @@ describe("reminder-worker — cycle-reminder wiring", () => { expect(source).toMatch(/\[CYCLE_REMINDER_QUEUE,\s*CYCLE_REMINDER_CRON\]/); }); - it("registers a boss.work handler that runs the cycle-reminder tick", () => { + it("registers a createAndWork handler that runs the cycle-reminder tick", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}CYCLE_REMINDER_QUEUE[\s\S]{0,200}handleCycleReminderCheck/, + /createAndWork[\s\S]{0,200}CYCLE_REMINDER_QUEUE[\s\S]{0,200}handleCycleReminderCheck/, ); expect(source).toMatch( /handleCycleReminderCheck[\s\S]{0,400}runCycleReminderTick/, diff --git a/src/lib/jobs/__tests__/daily-briefing.test.ts b/src/lib/jobs/__tests__/daily-briefing.test.ts index 8ae057493..ed34641db 100644 --- a/src/lib/jobs/__tests__/daily-briefing.test.ts +++ b/src/lib/jobs/__tests__/daily-briefing.test.ts @@ -98,9 +98,9 @@ describe("reminder-worker — daily-briefing wiring", () => { ); }); - it("wires a boss.work handler that runs the tick", () => { + it("wires a createAndWork handler that runs the tick", () => { expect(registrar).toMatch( - /boss\.work[\s\S]{0,200}DAILY_BRIEFING_QUEUE[\s\S]{0,400}runDailyBriefingTick/, + /createAndWork[\s\S]{0,200}DAILY_BRIEFING_QUEUE[\s\S]{0,400}runDailyBriefingTick/, ); }); diff --git a/src/lib/jobs/__tests__/data-arrival-queue.test.ts b/src/lib/jobs/__tests__/data-arrival-queue.test.ts index 9deeb634e..a86f8057f 100644 --- a/src/lib/jobs/__tests__/data-arrival-queue.test.ts +++ b/src/lib/jobs/__tests__/data-arrival-queue.test.ts @@ -39,9 +39,9 @@ describe("data-arrival queue wiring", () => { expect(allQueues![1]).toMatch(/\bDATA_ARRIVAL_QUEUE\b/); }); - it("binds a boss.work handler that drains it", () => { + it("binds a createAndWork handler that drains it", () => { expect(statusRegistrar).toMatch( - /boss\.work[\s\S]{0,200}DATA_ARRIVAL_QUEUE[\s\S]{0,200}handleDataArrival/, + /createAndWork[\s\S]{0,200}DATA_ARRIVAL_QUEUE[\s\S]{0,200}handleDataArrival/, ); }); @@ -61,7 +61,11 @@ describe("data-arrival queue wiring", () => { }); it("has no cron schedule — it is send-only, driven by ingest", () => { - expect(statusRegistrar).not.toMatch(/\[DATA_ARRIVAL_QUEUE,\s*\w+_CRON\]/); + // Tolerates the optional third schedule element, so a cron sneaking in + // with send options still trips this. + expect(statusRegistrar).not.toMatch( + /\[\s*DATA_ARRIVAL_QUEUE,\s*\w+_CRON\s*[,\]]/, + ); }); it("registers the retention job that prunes the markers", () => { @@ -71,10 +75,10 @@ describe("data-arrival queue wiring", () => { expect(allQueues).not.toBeNull(); expect(allQueues![1]).toMatch(/\bARRIVAL_REACTION_CLEANUP_QUEUE\b/); expect(maintenanceSource).toMatch( - /\[ARRIVAL_REACTION_CLEANUP_QUEUE,\s*ARRIVAL_REACTION_CLEANUP_CRON\]/, + /\[\s*ARRIVAL_REACTION_CLEANUP_QUEUE,\s*ARRIVAL_REACTION_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, ); expect(maintenanceSource).toMatch( - /boss\.work[\s\S]{0,200}ARRIVAL_REACTION_CLEANUP_QUEUE[\s\S]{0,200}handleArrivalReactionCleanup/, + /createAndWork[\s\S]{0,200}ARRIVAL_REACTION_CLEANUP_QUEUE[\s\S]{0,200}handleArrivalReactionCleanup/, ); }); diff --git a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts index 4bfc499cb..b904a30dd 100644 --- a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts +++ b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts @@ -50,15 +50,15 @@ describe("reminder-worker — drainPerSampleCumulative nightly schedule", () => expect(source).toMatch(/\bDRAIN_CUMULATIVE_CUTOFF_HOURS\b/); }); - it("registers a boss.work handler against the drain queue", () => { + it("registers a createAndWork handler against the drain queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,400}drainPerSampleCumulative/, + /createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,900}drainPerSampleCumulative/, ); }); it("schedules the drain cron via boss.schedule (allQueues + schedules)", () => { expect(source).toMatch( - /\[DRAIN_CUMULATIVE_QUEUE,\s*DRAIN_CUMULATIVE_CRON\]/, + /\[DRAIN_CUMULATIVE_QUEUE,\s*DRAIN_CUMULATIVE_CRON,\s*cronIsTheRetry\]/, ); }); diff --git a/src/lib/jobs/__tests__/fitbit-queues.test.ts b/src/lib/jobs/__tests__/fitbit-queues.test.ts index 3e6145540..a9e9fbbc6 100644 --- a/src/lib/jobs/__tests__/fitbit-queues.test.ts +++ b/src/lib/jobs/__tests__/fitbit-queues.test.ts @@ -58,22 +58,22 @@ describe("reminder-worker — Fitbit sync queues", () => { it("schedules the hourly poll cron + the daily oauth-state sweep", () => { expect(source).toMatch(/\[FITBIT_SYNC_QUEUE,\s*FITBIT_SYNC_CRON\]/); expect(source).toMatch( - /\[FITBIT_OAUTH_STATE_CLEANUP_QUEUE,\s*FITBIT_OAUTH_STATE_CLEANUP_CRON\]/, + /\[\s*FITBIT_OAUTH_STATE_CLEANUP_QUEUE,\s*FITBIT_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, ); }); - it("registers a boss.work handler for every Fitbit queue", () => { + it("registers a createAndWork handler for every Fitbit queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,160}FITBIT_SYNC_QUEUE[\s\S]{0,160}handleFitbitSync/, + /createAndWork[\s\S]{0,160}FITBIT_SYNC_QUEUE[\s\S]{0,160}handleFitbitSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,200}FITBIT_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, + /createAndWork[\s\S]{0,200}FITBIT_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,200}FITBIT_SLEEP_REPAIR_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, + /createAndWork[\s\S]{0,200}FITBIT_SLEEP_REPAIR_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,160}FITBIT_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,160}handleFitbitOAuthStateCleanup/, + /createAndWork[\s\S]{0,160}FITBIT_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,160}handleFitbitOAuthStateCleanup/, ); }); diff --git a/src/lib/jobs/__tests__/google-health-queues.test.ts b/src/lib/jobs/__tests__/google-health-queues.test.ts index 179091be9..783593da4 100644 --- a/src/lib/jobs/__tests__/google-health-queues.test.ts +++ b/src/lib/jobs/__tests__/google-health-queues.test.ts @@ -61,19 +61,19 @@ describe("reminder-worker — Google Health sync queues", () => { /\[GOOGLE_HEALTH_SYNC_QUEUE,\s*GOOGLE_HEALTH_SYNC_CRON\]/, ); expect(source).toMatch( - /\[\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE,\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON,?\s*\]/, + /\[\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE,\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, ); }); - it("registers a boss.work handler for every Google Health queue", () => { + it("registers a createAndWork handler for every Google Health queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}GOOGLE_HEALTH_SYNC_QUEUE[\s\S]{0,200}handleGoogleHealthSync/, + /createAndWork[\s\S]{0,200}GOOGLE_HEALTH_SYNC_QUEUE[\s\S]{0,200}handleGoogleHealthSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,320}GOOGLE_HEALTH_BACKFILL_QUEUE[\s\S]{0,320}enqueueIntegrationBackfillAdmission/, + /createAndWork[\s\S]{0,320}GOOGLE_HEALTH_BACKFILL_QUEUE[\s\S]{0,320}enqueueIntegrationBackfillAdmission/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,200}GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,200}handleGoogleHealthOAuthStateCleanup/, + /createAndWork[\s\S]{0,200}GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,200}handleGoogleHealthOAuthStateCleanup/, ); }); diff --git a/src/lib/jobs/__tests__/insight-pregenerate.test.ts b/src/lib/jobs/__tests__/insight-pregenerate.test.ts index e8bb06131..38c92003e 100644 --- a/src/lib/jobs/__tests__/insight-pregenerate.test.ts +++ b/src/lib/jobs/__tests__/insight-pregenerate.test.ts @@ -1181,9 +1181,9 @@ describe("queue registration", () => { ); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(workerSrc).toMatch( - /boss\.work[\s\S]{0,80}INSIGHT_PREGENERATE_QUEUE/, + /createAndWork[\s\S]{0,80}INSIGHT_PREGENERATE_QUEUE/, ); }); diff --git a/src/lib/jobs/__tests__/insight-status-generate.test.ts b/src/lib/jobs/__tests__/insight-status-generate.test.ts index fe4a9b116..06ab5ed4e 100644 --- a/src/lib/jobs/__tests__/insight-status-generate.test.ts +++ b/src/lib/jobs/__tests__/insight-status-generate.test.ts @@ -6,7 +6,7 @@ * - an unknown metric is annotated and skipped, not thrown; * - the enqueue helper no-ops without a global boss instance and de-dupes * with a singletonKey when one is present; - * - the queue is registered in `allQueues` AND wired to a `boss.work` + * - the queue is registered in `allQueues` AND wired to a `createAndWork` * handler in reminder-worker.ts (the v1.4.37 W10 unregistered-queue * catch — a queue built but never registered silently never drains). */ @@ -171,9 +171,9 @@ describe("queue registration", () => { expect(match![1]).toMatch(/\bINSIGHT_STATUS_GENERATE_QUEUE\b/); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(workerSrc).toMatch( - /boss\.work[\s\S]{0,120}INSIGHT_STATUS_GENERATE_QUEUE/, + /createAndWork[\s\S]{0,120}INSIGHT_STATUS_GENERATE_QUEUE/, ); }); diff --git a/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts b/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts index 16e1ae0ad..a99672474 100644 --- a/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts +++ b/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts @@ -3,7 +3,7 @@ * * Same source-text-grep approach as the drain + step-consolidation + * mean-consolidation regression guards: assert the queue is registered - * in `allQueues`, a `boss.work` handler is wired against it, and the + * in `allQueues`, a `createAndWork` handler is wired against it, and the * boot-discovery enqueue helper is called — without booting pg-boss + * Prisma. An unregistered queue silently never drains (the past incident * this guard exists to prevent). @@ -42,12 +42,12 @@ describe("reminder-worker — intake-slot-dedup wiring", () => { expect(allQueuesMatch![1]).toMatch(/\bINTAKE_SLOT_DEDUP_QUEUE\b/); }); - it("registers a boss.work handler against the intake-slot-dedup queue", () => { + it("registers a createAndWork handler against the intake-slot-dedup queue", () => { // v1.15.19 — span widened from {0,400}: the handler gained the daily // cron-tick dispatch branch (no `userId` → discovery fan-out) between // the queue name and the per-user dedup call. expect(source).toMatch( - /boss\.work[\s\S]{0,200}INTAKE_SLOT_DEDUP_QUEUE[\s\S]{0,1200}dedupeUserIntakeSlots/, + /createAndWork[\s\S]{0,200}INTAKE_SLOT_DEDUP_QUEUE[\s\S]{0,1400}dedupeUserIntakeSlots/, ); }); diff --git a/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts b/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts index 6215ec22f..7f42958a7 100644 --- a/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts +++ b/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts @@ -2,7 +2,7 @@ * MCP Phase 3 (M2) — mcp-token-cleanup queue registration guard. * * Source-text-grep guard like the other queue-wiring checks: assert the queue - * is in `allQueues`, scheduled, and wired to a `boss.work` handler. An + * is in `allQueues`, scheduled, and wired to a `createAndWork` handler. An * unregistered queue silently never drains (the recurring v1.4.37 dead-queue * bug), which would leave expired MCP connector access tokens growing unbounded. */ @@ -40,13 +40,13 @@ describe("reminder-worker — mcp-token-cleanup wiring", () => { it("schedules the mcp-token-cleanup cron", () => { expect(workerSource).toMatch( - /\[MCP_TOKEN_CLEANUP_QUEUE,\s*MCP_TOKEN_CLEANUP_CRON\]/, + /\[MCP_TOKEN_CLEANUP_QUEUE,\s*MCP_TOKEN_CLEANUP_CRON,\s*cronIsTheRetry\]/, ); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(workerSource).toMatch( - /boss\.work[\s\S]{0,200}MCP_TOKEN_CLEANUP_QUEUE[\s\S]{0,200}handleMcpTokenCleanup/, + /createAndWork[\s\S]{0,200}MCP_TOKEN_CLEANUP_QUEUE[\s\S]{0,200}handleMcpTokenCleanup/, ); }); diff --git a/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts b/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts index 3b5956ec6..4dbfd6d11 100644 --- a/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts +++ b/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts @@ -3,7 +3,7 @@ * * Same source-text-grep approach as the drain + step-consolidation * regression guards: assert the queue is registered in `allQueues`, a - * `boss.work` handler is wired against it, and the boot-discovery + * `createAndWork` handler is wired against it, and the boot-discovery * enqueue helper is called — without booting pg-boss + Prisma. An * unregistered queue silently never drains. */ @@ -34,9 +34,9 @@ describe("reminder-worker — mean-consolidation wiring", () => { expect(allQueuesMatch![1]).toMatch(/\bMEAN_CONSOLIDATION_QUEUE\b/); }); - it("registers a boss.work handler against the mean-consolidation queue", () => { + it("registers a createAndWork handler against the mean-consolidation queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}MEAN_CONSOLIDATION_QUEUE[\s\S]{0,400}runMeanConsolidationForUser/, + /createAndWork[\s\S]{0,200}MEAN_CONSOLIDATION_QUEUE[\s\S]{0,400}runMeanConsolidationForUser/, ); }); diff --git a/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts b/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts index 9c36703d4..564b9cb95 100644 --- a/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts +++ b/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts @@ -5,7 +5,7 @@ * guards: assert the Vorsorge-reminder queue is declared, listed in * `allQueues` (the v1.4.37 dead-queue lesson — an unregistered queue is * never provisioned and the schedule silently no-ops), scheduled on its - * cron, and wired to a `boss.work` handler that delegates to + * cron, and wired to a `createAndWork` handler that delegates to * `runMeasurementReminderTick`. No pg-boss / Prisma boot. */ import { readFileSync } from "node:fs"; @@ -56,9 +56,9 @@ describe("reminder-worker — measurement-reminder wiring", () => { ); }); - it("registers a boss.work handler that runs the tick", () => { + it("registers a createAndWork handler that runs the tick", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}MEASUREMENT_REMINDER_QUEUE[\s\S]{0,200}handleMeasurementReminderCheck/, + /createAndWork[\s\S]{0,200}MEASUREMENT_REMINDER_QUEUE[\s\S]{0,200}handleMeasurementReminderCheck/, ); expect(source).toMatch( /handleMeasurementReminderCheck[\s\S]{0,400}runMeasurementReminderTick/, @@ -80,12 +80,12 @@ describe("reminder-worker — eventful reminder-satisfy wiring (v1.18.1)", () => expect(allQueuesMatch![1]).toMatch(/\bREMINDER_SATISFY_QUEUE\b/); }); - it("binds a boss.work handler for the satisfy queue", () => { + it("binds a createAndWork handler for the satisfy queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}REMINDER_SATISFY_QUEUE[\s\S]{0,200}handleReminderSatisfy/, + /createAndWork[\s\S]{0,200}REMINDER_SATISFY_QUEUE[\s\S]{0,200}handleReminderSatisfy/, ); expect(source).toMatch( - /handleReminderSatisfy[\s\S]{0,600}runReminderSatisfyForUser/, + /handleReminderSatisfy[\s\S]{0,800}runReminderSatisfyForUser/, ); }); }); diff --git a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts index ed96efe98..fa68a9100 100644 --- a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts +++ b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts @@ -7,7 +7,7 @@ * recently-tombstoned rows untouched. * 2. Source-grep wiring (same approach as the drain-cumulative guard): * the queue is registered in `allQueues`, scheduled, and bound to a - * `boss.work` handler — a missing `allQueues` entry silently no-ops + * `createAndWork` handler — a missing `allQueues` entry silently no-ops * the schedule under pg-boss v12. */ import { readFileSync } from "node:fs"; @@ -114,9 +114,9 @@ describe("reminder-worker — measurement-tombstone-cleanup schedule", () => { ); }); - it("binds a boss.work handler to the queue", () => { + it("binds a createAndWork handler to the queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE[\s\S]{0,200}handleMeasurementTombstoneCleanup/, + /createAndWork[\s\S]{0,200}MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE[\s\S]{0,200}handleMeasurementTombstoneCleanup/, ); }); }); diff --git a/src/lib/jobs/__tests__/nightscout-queues.test.ts b/src/lib/jobs/__tests__/nightscout-queues.test.ts index 5873b8382..6c994d55a 100644 --- a/src/lib/jobs/__tests__/nightscout-queues.test.ts +++ b/src/lib/jobs/__tests__/nightscout-queues.test.ts @@ -50,9 +50,9 @@ describe("reminder-worker — Nightscout sync queue", () => { expect(source).toMatch(/\[NIGHTSCOUT_SYNC_QUEUE,\s*NIGHTSCOUT_SYNC_CRON\]/); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,160}NIGHTSCOUT_SYNC_QUEUE[\s\S]{0,160}handleNightscoutSync/, + /createAndWork[\s\S]{0,160}NIGHTSCOUT_SYNC_QUEUE[\s\S]{0,160}handleNightscoutSync/, ); }); }); @@ -85,12 +85,12 @@ describe("reminder-worker — Polar / Oura sync queues", () => { expect(source).toMatch(/\[OURA_SYNC_QUEUE,\s*OURA_SYNC_CRON\]/); }); - it("registers a boss.work handler for each queue", () => { + it("registers a createAndWork handler for each queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,160}POLAR_SYNC_QUEUE[\s\S]{0,160}handlePolarSync/, + /createAndWork[\s\S]{0,160}POLAR_SYNC_QUEUE[\s\S]{0,160}handlePolarSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,160}OURA_SYNC_QUEUE[\s\S]{0,160}handleOuraSync/, + /createAndWork[\s\S]{0,160}OURA_SYNC_QUEUE[\s\S]{0,160}handleOuraSync/, ); }); }); diff --git a/src/lib/jobs/__tests__/period-narrative-warm.test.ts b/src/lib/jobs/__tests__/period-narrative-warm.test.ts index 45838182e..5c9e6ea83 100644 --- a/src/lib/jobs/__tests__/period-narrative-warm.test.ts +++ b/src/lib/jobs/__tests__/period-narrative-warm.test.ts @@ -196,8 +196,10 @@ describe("queue registration", () => { ); }); - it("registers a boss.work handler for the queue", () => { - expect(workerSrc).toMatch(/boss\.work[\s\S]{0,120}PERIOD_NARRATIVE_QUEUE/); + it("registers a createAndWork handler for the queue", () => { + expect(workerSrc).toMatch( + /createAndWork[\s\S]{0,120}PERIOD_NARRATIVE_QUEUE/, + ); }); it("exposes a sane queue name + nightly cron", () => { diff --git a/src/lib/jobs/__tests__/recovery-score-queue.test.ts b/src/lib/jobs/__tests__/recovery-score-queue.test.ts index 5e7a15866..65628b9e9 100644 --- a/src/lib/jobs/__tests__/recovery-score-queue.test.ts +++ b/src/lib/jobs/__tests__/recovery-score-queue.test.ts @@ -3,7 +3,7 @@ * * Same source-text-grep approach as the other queue-wiring guards: assert * the queue is imported, registered in `allQueues`, scheduled, and wired to - * a `boss.work` handler — without booting pg-boss + Prisma. An unregistered + * a `createAndWork` handler — without booting pg-boss + Prisma. An unregistered * queue silently never drains (the recurring past bug this guards against). * * Also pins the iOS-write rejection of the server-owned COMPUTED source on @@ -57,9 +57,9 @@ describe("reminder-worker — recovery-score wiring", () => { ); }); - it("registers a boss.work handler that runs the recovery-score pass", () => { + it("registers a createAndWork handler that runs the recovery-score pass", () => { expect(workerSource).toMatch( - /boss\.work[\s\S]{0,200}RECOVERY_SCORE_QUEUE[\s\S]{0,400}runRecoveryScore/, + /createAndWork[\s\S]{0,200}RECOVERY_SCORE_QUEUE[\s\S]{0,400}runRecoveryScore/, ); }); }); diff --git a/src/lib/jobs/__tests__/run-job.test.ts b/src/lib/jobs/__tests__/run-job.test.ts new file mode 100644 index 000000000..6973cb89a --- /dev/null +++ b/src/lib/jobs/__tests__/run-job.test.ts @@ -0,0 +1,112 @@ +/** + * Unit tests for the job-outcome spine. + * + * The whole point of `runJob` is that an `ok: false` outcome reaches pg-boss + * as a failed job. That is a behaviour claim, not a typing one, and it is the + * claim the retention cleanups now depend on — so it is pinned here: the + * failure is reported once with the handler's own facts, it rethrows, and a + * handler that throws on its own is passed through untouched rather than + * rewrapped into a stack that points at this file. + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import type { Job } from "pg-boss"; + +const reportWorkerError = vi.fn(); + +vi.mock("../report-worker-error", () => ({ + reportWorkerError: (...a: unknown[]) => reportWorkerError(...a), +})); + +import { jobDone, jobFailed } from "../job-outcome"; +import { JobFailure, runJob } from "../run-job"; + +const noJobs: Job[] = []; + +beforeEach(() => { + vi.clearAllMocks(); + reportWorkerError.mockResolvedValue(undefined); +}); + +describe("runJob", () => { + it("resolves and reports nothing when the handler says it did the work", async () => { + const wrapped = runJob("rate-limit-cleanup", async () => + jobDone({ deleted: 12 }), + ); + + await expect(wrapped(noJobs)).resolves.toBeUndefined(); + expect(reportWorkerError).not.toHaveBeenCalled(); + }); + + it("treats a run with nothing to do as a success", async () => { + const wrapped = runJob("mood-reminder-cleanup", async () => + jobDone({ deleted: 0 }), + ); + + await expect(wrapped(noJobs)).resolves.toBeUndefined(); + expect(reportWorkerError).not.toHaveBeenCalled(); + }); + + it("rethrows a failed outcome so pg-boss records a failed job", async () => { + const cause = new Error("statement timeout"); + const wrapped = runJob("push-attempt-cleanup", async () => + jobFailed("push-attempt cleanup failed", cause, { deleted: 0 }), + ); + + await expect(wrapped(noJobs)).rejects.toBeInstanceOf(JobFailure); + await expect(wrapped(noJobs)).rejects.toThrow( + "push-attempt-cleanup: push-attempt cleanup failed", + ); + }); + + it("reports the failure with the handler's own facts and the cause", async () => { + const cause = new Error("statement timeout"); + const wrapped = runJob("push-attempt-cleanup", async () => + jobFailed("push-attempt cleanup failed", cause, { deleted: 0 }), + ); + + await expect(wrapped(noJobs)).rejects.toThrow(); + + expect(reportWorkerError).toHaveBeenCalledTimes(1); + const [queue, error, meta] = reportWorkerError.mock.calls[0]; + expect(queue).toBe("push-attempt-cleanup"); + expect(error).toBeInstanceOf(JobFailure); + expect(meta).toEqual({ deleted: 0, cause: "statement timeout" }); + }); + + it("carries the thrown error's stack instead of its own frame", async () => { + const cause = new Error("statement timeout"); + cause.stack = "Error: statement timeout\n at deleteMany (prisma.ts:1:1)"; + const wrapped = runJob("mcp-token-cleanup", async () => + jobFailed("mcp-token cleanup failed", cause), + ); + + const failure = await wrapped(noJobs).catch((err: unknown) => err); + expect(failure).toBeInstanceOf(JobFailure); + expect((failure as JobFailure).stack).toContain("at deleteMany"); + expect((failure as JobFailure).cause).toBe(cause); + }); + + it("passes a throwing handler through untouched", async () => { + const thrown = new Error("cohort discovery failed"); + const wrapped = runJob("withings-sync", async () => { + throw thrown; + }); + + await expect(wrapped(noJobs)).rejects.toBe(thrown); + // The throw already fails the job. Reporting it here would double every + // handler that reports for itself, and rewrapping would bury the stack. + expect(reportWorkerError).not.toHaveBeenCalled(); + }); + + it("hands the batch to the handler unchanged", async () => { + const handler = vi.fn(async () => jobDone({ jobs: 2 })); + const jobs = [ + { id: "a", data: { userId: "u1" } }, + { id: "b", data: { userId: "u2" } }, + ] as unknown as Job<{ userId: string }>[]; + + await runJob("data-arrival", handler)(jobs); + + expect(handler).toHaveBeenCalledWith(jobs); + }); +}); diff --git a/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts b/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts index c6375c871..cc803ecc5 100644 --- a/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts +++ b/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts @@ -3,7 +3,7 @@ * * The reminder-worker imports heavy infrastructure, so we read the source as * text and assert the queue constant is declared, registered in `allQueues`, - * wired to a `boss.work` handler, and enqueued at boot — the fast guard that + * wired to a `createAndWork` handler, and enqueued at boot — the fast guard that * catches the v1.4.37 dead-queue class (a queue declared but never registered, * which silently never drains). */ @@ -30,9 +30,9 @@ describe("reminder-worker — sleep-timeline backfill queue", () => { expect(block![1]!).toContain("SLEEP_TIMELINE_BACKFILL_QUEUE"); }); - it("wires a boss.work handler through the shared admission queue", () => { + it("wires a createAndWork handler through the shared admission queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,260}SLEEP_TIMELINE_BACKFILL_QUEUE[\s\S]{0,300}enqueueIntegrationBackfillAdmission/, + /createAndWork[\s\S]{0,260}SLEEP_TIMELINE_BACKFILL_QUEUE[\s\S]{0,300}enqueueIntegrationBackfillAdmission/, ); }); diff --git a/src/lib/jobs/__tests__/status-cron-candidates.test.ts b/src/lib/jobs/__tests__/status-cron-candidates.test.ts index 15c13baa8..4f2690a1f 100644 --- a/src/lib/jobs/__tests__/status-cron-candidates.test.ts +++ b/src/lib/jobs/__tests__/status-cron-candidates.test.ts @@ -196,8 +196,8 @@ describe("mood-status queue registration (dead-queue guard)", () => { expect(workerSrc).toMatch(/MOOD_STATUS_CRON\s*=\s*"30 2 \* \* \*"/); }); - it("registers a boss.work handler for the queue", () => { - expect(workerSrc).toMatch(/boss\.work[\s\S]{0,120}MOOD_STATUS_QUEUE/); + it("registers a createAndWork handler for the queue", () => { + expect(workerSrc).toMatch(/createAndWork[\s\S]{0,120}MOOD_STATUS_QUEUE/); }); it("drives all seven status crons through the shared discovery", () => { diff --git a/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts b/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts index 8e0eeab75..e34c295cd 100644 --- a/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts +++ b/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts @@ -4,7 +4,7 @@ * * Same source-text-grep approach as the drain + step-consolidation + * mean-consolidation guards: assert the repair queue is registered in - * `allQueues`, a `boss.work` handler is wired against it, and the + * `allQueues`, a `createAndWork` handler is wired against it, and the * boot-discovery enqueue helper fires — without booting pg-boss + Prisma. * An unregistered queue silently never drains (the dead-queue class). */ @@ -36,9 +36,9 @@ describe("reminder-worker — step-consolidation-repair wiring", () => { expect(allQueuesMatch![1]).toMatch(/\bSTEP_CONSOLIDATION_REPAIR_QUEUE\b/); }); - it("registers a boss.work handler against the repair queue", () => { + it("registers a createAndWork handler against the repair queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}STEP_CONSOLIDATION_REPAIR_QUEUE[\s\S]{0,400}runStepConsolidationRepairForUser/, + /createAndWork[\s\S]{0,200}STEP_CONSOLIDATION_REPAIR_QUEUE[\s\S]{0,400}runStepConsolidationRepairForUser/, ); }); diff --git a/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts b/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts index 839c2b575..969d80fba 100644 --- a/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts +++ b/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts @@ -5,7 +5,7 @@ * * Same source-text-grep approach as the WX-C recovery-score guard: assert * each queue is imported, registered in `allQueues`, scheduled (the two - * cron-driven score jobs), and wired to a `boss.work` handler — without + * cron-driven score jobs), and wired to a `createAndWork` handler — without * booting pg-boss + Prisma. An unregistered queue silently never drains * (the recurring past bug this guards against). */ @@ -67,9 +67,9 @@ describe("reminder-worker — stress-score wiring", () => { expect(source).toMatch(/\[STRESS_SCORE_QUEUE,\s*STRESS_SCORE_CRON\]/); }); - it("registers a boss.work handler that runs the stress-score pass", () => { + it("registers a createAndWork handler that runs the stress-score pass", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}STRESS_SCORE_QUEUE[\s\S]{0,400}runStressScore/, + /createAndWork[\s\S]{0,200}STRESS_SCORE_QUEUE[\s\S]{0,400}runStressScore/, ); }); }); @@ -90,9 +90,9 @@ describe("reminder-worker — strain-score wiring", () => { expect(source).toMatch(/\[STRAIN_SCORE_QUEUE,\s*STRAIN_SCORE_CRON\]/); }); - it("registers a boss.work handler that runs the strain-score pass", () => { + it("registers a createAndWork handler that runs the strain-score pass", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,200}STRAIN_SCORE_QUEUE[\s\S]{0,400}runStrainScore/, + /createAndWork[\s\S]{0,200}STRAIN_SCORE_QUEUE[\s\S]{0,400}runStrainScore/, ); }); }); @@ -111,9 +111,9 @@ describe("reminder-worker — dense intra-day retention wiring", () => { expect(allQueuesBlock()).toMatch(/\bDENSE_INTRADAY_RETENTION_QUEUE\b/); }); - it("registers a boss.work handler against the dense-intraday-retention queue", () => { + it("registers a createAndWork handler against the dense-intraday-retention queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,250}DENSE_INTRADAY_RETENTION_QUEUE[\s\S]{0,500}runDenseIntradayRetentionForUser/, + /createAndWork[\s\S]{0,250}DENSE_INTRADAY_RETENTION_QUEUE[\s\S]{0,500}runDenseIntradayRetentionForUser/, ); }); @@ -163,9 +163,9 @@ describe("reminder-worker — dense hourly-rebuild wiring", () => { expect(allQueuesBlock()).toMatch(/\bDENSE_INTRADAY_HOURLY_REBUILD_QUEUE\b/); }); - it("registers a boss.work handler against the hourly-rebuild queue", () => { + it("registers a createAndWork handler against the hourly-rebuild queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,250}DENSE_INTRADAY_HOURLY_REBUILD_QUEUE[\s\S]{0,500}runDenseIntradayHourlyRebuildForUser/, + /createAndWork[\s\S]{0,250}DENSE_INTRADAY_HOURLY_REBUILD_QUEUE[\s\S]{0,500}runDenseIntradayHourlyRebuildForUser/, ); }); diff --git a/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts b/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts index 68764d932..9fe67d0b0 100644 --- a/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts +++ b/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts @@ -3,7 +3,7 @@ * * Same source-text-grep approach as the other queue-wiring guards: assert * the queue is imported, registered in `allQueues`, scheduled, and wired to - * a `boss.work` handler — without booting pg-boss + Prisma. An unregistered + * a `createAndWork` handler — without booting pg-boss + Prisma. An unregistered * queue silently never drains (the recurring v1.4.37 dead-queue bug), which * would leave the pinned-leaf-rotation alarm dark. */ @@ -50,9 +50,9 @@ describe("reminder-worker — tls-pin-monitor wiring", () => { ); }); - it("registers a boss.work handler for the queue", () => { + it("registers a createAndWork handler for the queue", () => { expect(workerSource).toMatch( - /boss\.work[\s\S]{0,200}TLS_PIN_MONITOR_QUEUE[\s\S]{0,200}handleTlsPinMonitor/, + /createAndWork[\s\S]{0,200}TLS_PIN_MONITOR_QUEUE[\s\S]{0,200}handleTlsPinMonitor/, ); }); diff --git a/src/lib/jobs/__tests__/whoop-queues.test.ts b/src/lib/jobs/__tests__/whoop-queues.test.ts index 7d2c514c1..a1ad468b4 100644 --- a/src/lib/jobs/__tests__/whoop-queues.test.ts +++ b/src/lib/jobs/__tests__/whoop-queues.test.ts @@ -77,25 +77,25 @@ describe("reminder-worker — WHOOP sync queues", () => { /\[WHOOP_CYCLE_SYNC_QUEUE,\s*WHOOP_CYCLE_SYNC_CRON\]/, ); expect(source).toMatch( - /\[WHOOP_OAUTH_STATE_CLEANUP_QUEUE,\s*WHOOP_OAUTH_STATE_CLEANUP_CRON\]/, + /\[\s*WHOOP_OAUTH_STATE_CLEANUP_QUEUE,\s*WHOOP_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, ); }); - it("registers a boss.work handler for every WHOOP sync queue", () => { + it("registers a createAndWork handler for every WHOOP sync queue", () => { expect(source).toMatch( - /boss\.work[\s\S]{0,160}WHOOP_RECOVERY_SYNC_QUEUE[\s\S]{0,160}handleWhoopRecoverySync/, + /createAndWork[\s\S]{0,160}WHOOP_RECOVERY_SYNC_QUEUE[\s\S]{0,160}handleWhoopRecoverySync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,160}WHOOP_SLEEP_SYNC_QUEUE[\s\S]{0,160}handleWhoopSleepSync/, + /createAndWork[\s\S]{0,160}WHOOP_SLEEP_SYNC_QUEUE[\s\S]{0,160}handleWhoopSleepSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,160}WHOOP_WORKOUT_SYNC_QUEUE[\s\S]{0,160}handleWhoopWorkoutSync/, + /createAndWork[\s\S]{0,160}WHOOP_WORKOUT_SYNC_QUEUE[\s\S]{0,160}handleWhoopWorkoutSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,160}WHOOP_CYCLE_SYNC_QUEUE[\s\S]{0,160}handleWhoopCycleSync/, + /createAndWork[\s\S]{0,160}WHOOP_CYCLE_SYNC_QUEUE[\s\S]{0,160}handleWhoopCycleSync/, ); expect(source).toMatch( - /boss\.work[\s\S]{0,200}WHOOP_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, + /createAndWork[\s\S]{0,200}WHOOP_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, ); }); diff --git a/src/lib/jobs/__tests__/whoop-resource-sync.test.ts b/src/lib/jobs/__tests__/whoop-resource-sync.test.ts index 6359d2ecb..cbe181b6d 100644 --- a/src/lib/jobs/__tests__/whoop-resource-sync.test.ts +++ b/src/lib/jobs/__tests__/whoop-resource-sync.test.ts @@ -128,7 +128,18 @@ describe("runWhoopResourceSync dispatch", () => { findManyMock.mockResolvedValue([{ userId: "a" }, { userId: "b" }]); syncUserWorkout.mockRejectedValueOnce(new Error("boom for a")); - await expect(handleWhoopWorkoutSync([job({})])).resolves.toBeUndefined(); + // The pass is what the outcome judges: the sweep ran, so the job is + // done. The revoked-grant user rides out as `users_failed` rather than + // failing the queue and retrying the whole cohort over one account. + await expect(handleWhoopWorkoutSync([job({})])).resolves.toEqual({ + ok: true, + did: { + total: 2, + users_synced: 1, + users_failed: 1, + measurements_imported: 0, + }, + }); // Both users were attempted despite the first throwing. expect(syncUserWorkout).toHaveBeenCalledTimes(2); }); diff --git a/src/lib/jobs/__tests__/withings-ecg-worker.test.ts b/src/lib/jobs/__tests__/withings-ecg-worker.test.ts index cb11bde9e..e40bf868c 100644 --- a/src/lib/jobs/__tests__/withings-ecg-worker.test.ts +++ b/src/lib/jobs/__tests__/withings-ecg-worker.test.ts @@ -93,7 +93,15 @@ describe("handleWithingsEcgSync", () => { enddate: 1715000060, }); - await expect(handleWithingsEcgSync([queued])).resolves.toBeUndefined(); + await expect(handleWithingsEcgSync([queued])).resolves.toEqual({ + ok: true, + did: { + total: 1, + users_synced: 1, + users_failed: 0, + recordings_imported: 1, + }, + }); expect(ecgFindMany).not.toHaveBeenCalled(); expect(syncUserEcg).toHaveBeenCalledTimes(1); @@ -120,14 +128,24 @@ describe("handleWithingsEcgSync", () => { it("one user's failure never starves the rest of the cohort", async () => { ecgFindMany.mockResolvedValue([{ userId: "a" }, { userId: "b" }]); + syncUserEcg.mockResolvedValue(0); syncUserEcg.mockRejectedValueOnce(new Error("boom for a")); // Per-user catch-and-warn (mirrors the activity / sleep handlers): the - // handler resolves, both users attempted, the leaf records its own ledger - // failure so honesty is preserved without rejecting the whole batch. + // pass ran, so the job is done and the failing user rides out as a + // `users_failed` count. The leaf records its own ledger failure, so + // honesty is preserved without rejecting the whole batch. await expect( handleWithingsEcgSync([job({} as WithingsEcgSyncPayload)]), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ + ok: true, + did: { + total: 2, + users_synced: 1, + users_failed: 1, + recordings_imported: 0, + }, + }); expect(syncUserEcg).toHaveBeenCalledTimes(2); }); }); @@ -143,7 +161,7 @@ describe("withings ECG queue registration", () => { /\[WITHINGS_ECG_SYNC_QUEUE\]:\s*\{[\s\S]*?policy:\s*"short"/, ); expect(registrarSource).toMatch( - /boss\.work\([\s\S]*?WITHINGS_ECG_SYNC_QUEUE[\s\S]*?handleWithingsEcgSync/, + /createAndWork\(\s*boss,\s*WITHINGS_ECG_SYNC_QUEUE[\s\S]{0,120}handleWithingsEcgSync/, ); }); }); diff --git a/src/lib/jobs/__tests__/withings-queues.test.ts b/src/lib/jobs/__tests__/withings-queues.test.ts index 5dc7dac08..9861c41a8 100644 --- a/src/lib/jobs/__tests__/withings-queues.test.ts +++ b/src/lib/jobs/__tests__/withings-queues.test.ts @@ -50,14 +50,14 @@ describe("reminder-worker — Withings activity + sleep v2 queues", () => { it("registers the activity-sync handler against pg-boss", () => { expect(source).toMatch(/handleWithingsActivitySync\b/); expect(source).toMatch( - /boss\.work[\s\S]{0,200}WITHINGS_ACTIVITY_QUEUE[\s\S]{0,200}handleWithingsActivitySync/, + /createAndWork[\s\S]{0,200}WITHINGS_ACTIVITY_QUEUE[\s\S]{0,200}handleWithingsActivitySync/, ); }); it("registers the sleep-sync handler against pg-boss", () => { expect(source).toMatch(/handleWithingsSleepSync\b/); expect(source).toMatch( - /boss\.work[\s\S]{0,200}WITHINGS_SLEEP_QUEUE[\s\S]{0,200}handleWithingsSleepSync/, + /createAndWork[\s\S]{0,200}WITHINGS_SLEEP_QUEUE[\s\S]{0,200}handleWithingsSleepSync/, ); }); @@ -78,7 +78,7 @@ describe("reminder-worker — Withings activity + sleep v2 queues", () => { }); it("schedules the ECG catch-net cron at :41 so watch-only accounts pull strips", () => { - // The queue is already in allQueues + boss.work-bound; W-1 adds the missing + // The queue is already in allQueues + createAndWork-bound; W-1 adds the missing // schedule entry so a payload-less cron tick walks the connection cohort. expect(source).toMatch( /WITHINGS_ECG_SYNC_CRON\s*=\s*["']41 \* \* \* \*["']/, diff --git a/src/lib/jobs/__tests__/withings-subscription-retry.test.ts b/src/lib/jobs/__tests__/withings-subscription-retry.test.ts index e873c9ca5..be6de5b0e 100644 --- a/src/lib/jobs/__tests__/withings-subscription-retry.test.ts +++ b/src/lib/jobs/__tests__/withings-subscription-retry.test.ts @@ -72,7 +72,16 @@ describe("handleWithingsFallbackSync subscription repair", () => { data: { triggeredAt: "2026-07-21T10:00:00.000Z" }, } as Job; - await handleWithingsFallbackSync([queued]); + await expect(handleWithingsFallbackSync([queued])).resolves.toEqual({ + ok: true, + did: { + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + subscription_repair_failed: false, + }, + }); expect(retryDueWithingsWebhookSubscriptions).toHaveBeenCalledTimes(1); expect(syncUserMeasurements).not.toHaveBeenCalled(); @@ -89,7 +98,19 @@ describe("handleWithingsFallbackSync subscription repair", () => { data: { triggeredAt: "2026-07-21T11:00:00.000Z" }, } as Job; - await expect(handleWithingsFallbackSync([queued])).resolves.toBeUndefined(); + // The repair is an auxiliary leg, not the pass this queue is named for: + // the fallback sync still ran for everyone, so the job is done and the + // repair failure rides out as a fact on the outcome. + await expect(handleWithingsFallbackSync([queued])).resolves.toEqual({ + ok: true, + did: { + total: 2, + users_synced: 2, + users_failed: 0, + measurements_imported: 0, + subscription_repair_failed: true, + }, + }); expect(findMany).toHaveBeenCalledWith({ select: { userId: true } }); expect(syncUserMeasurements).toHaveBeenNthCalledWith( diff --git a/src/lib/jobs/__tests__/workout-insight-queue.test.ts b/src/lib/jobs/__tests__/workout-insight-queue.test.ts index 4fdc3ca15..7af32f112 100644 --- a/src/lib/jobs/__tests__/workout-insight-queue.test.ts +++ b/src/lib/jobs/__tests__/workout-insight-queue.test.ts @@ -32,9 +32,9 @@ describe("workout-insight-generate queue wiring", () => { expect(allQueues![1]).toMatch(/\bWORKOUT_INSIGHT_GENERATE_QUEUE\b/); }); - it("binds a boss.work handler that drains it", () => { + it("binds a createAndWork handler that drains it", () => { expect(statusRegistrar).toMatch( - /boss\.work[\s\S]{0,300}WORKOUT_INSIGHT_GENERATE_QUEUE[\s\S]{0,300}handleWorkoutInsightGenerate/, + /createAndWork[\s\S]{0,300}WORKOUT_INSIGHT_GENERATE_QUEUE[\s\S]{0,300}handleWorkoutInsightGenerate/, ); }); @@ -50,8 +50,10 @@ describe("workout-insight-generate queue wiring", () => { it("has no cron schedule — it is dispatched by arrival only", () => { // A cron here would be a standing invitation to regenerate, which is the // saturation surface the design refuses. + // Tolerates the optional third schedule element, so a cron sneaking in + // with send options still trips this. expect(statusRegistrar).not.toMatch( - /\[WORKOUT_INSIGHT_GENERATE_QUEUE,\s*\w+_CRON\]/, + /\[\s*WORKOUT_INSIGHT_GENERATE_QUEUE,\s*\w+_CRON\s*[,\]]/, ); }); diff --git a/src/lib/jobs/apple-health-import-worker.ts b/src/lib/jobs/apple-health-import-worker.ts index 9bf197017..b71dfa0ab 100644 --- a/src/lib/jobs/apple-health-import-worker.ts +++ b/src/lib/jobs/apple-health-import-worker.ts @@ -31,6 +31,7 @@ import type { Job } from "pg-boss"; import { extractExportXml } from "@/lib/import/unzip-export-xml"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { streamParseExportXml, type ImportJobProgress, @@ -66,23 +67,26 @@ export const IMPORT_JOB_RECONCILE_CRON = "*/15 * * * *"; * the same `reconcileOrphanImportJobs()` the boot path calls — the * heartbeat + live pg-boss-job check already make it safe to re-run * on a healthy worker (a live import's fresh heartbeat and `active` - * pg-boss state both keep its row untouched). Errors are logged, not - * rethrown — a reconcile miss must never spam pg-boss's retry/backoff - * machinery for a background sweep this cheap to just re-run in 15 - * minutes. + * pg-boss state both keep its row untouched). A sweep that throws now + * reports a failed outcome instead of only leaving a warning behind: + * this pass exists to end a silent stall, and a stall in the pass + * itself was the one condition nothing surfaced. The next tick is 15 + * minutes away regardless, so nothing rests on the retry. */ export async function handleImportJobReconcileTick( jobs: Job[], -): Promise { +): Promise { void jobs; - await withBackgroundEvent( + return withBackgroundEvent( "job.apple_health_import_reconcile", async (evt) => { try { await reconcileOrphanImportJobs(); } catch (err) { evt.addWarning(`apple-health-import-reconcile failed: ${err}`); + return jobFailed("apple health import reconcile failed", err); } + return jobDone(); }, ); } @@ -230,7 +234,7 @@ async function writeProgress( */ export async function handleAppleHealthImport( job: Job, -): Promise { +): Promise { const { userId, uploadPath, uploadBytes, triggeredByAdminId } = job.data; const prisma = getWorkerPrisma(); @@ -280,7 +284,7 @@ export async function handleAppleHealthImport( ` — row is already terminal (${importJob.status}); the staged upload` + " was consumed by the first run and a re-run could only mask its outcome", ); - return; + return jobDone({ skipped: "already_terminal" }); } // Extracted-XML path, hoisted so the failure path can clean it up — @@ -364,6 +368,12 @@ export async function handleAppleHealthImport( // the end, scoped to the user's full measurement span, so // post-import reads of the analytics + comprehensive surfaces // hit the warm rollup table on first paint. + // + // The miss rides out as a fact rather than as a failed job: the + // import is committed and its mirror row already reads `done`, so + // failing here would report a completed import as failed and + // contradict the row the status poll surfaces to the operator. + let rollupFailed = false; try { const span = await prisma.measurement.aggregate({ where: { userId }, @@ -382,6 +392,7 @@ export async function handleAppleHealthImport( } catch (rollupErr) { // Rollup failure is non-fatal — the next read falls through to // live aggregation. Log but don't poison the import. + rollupFailed = true; console.warn( `[apple-health-import] Rollup recompute failed for user ${userId}`, rollupErr, @@ -392,6 +403,13 @@ export async function handleAppleHealthImport( // periodically swept on the host. safeUnlink(unzip.xmlPath); safeUnlink(uploadPath); + + return jobDone({ + records_read: result.totals.recordsRead, + rows_upserted: result.totals.rowsUpserted, + duration_ms: result.totals.durationMs, + rollup_failed: rollupFailed, + }); } catch (err) { // v1.28.33 (issue #486) — a missing staging file is an operational // condition, not a parse failure: `/tmp` is wiped on a container diff --git a/src/lib/jobs/data-arrival.ts b/src/lib/jobs/data-arrival.ts index d51132bd9..cb4318ac6 100644 --- a/src/lib/jobs/data-arrival.ts +++ b/src/lib/jobs/data-arrival.ts @@ -30,6 +30,7 @@ import type { Job } from "pg-boss"; import { invalidateUserDashboardSnapshot } from "@/lib/cache/invalidate"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { annotate } from "@/lib/logging/context"; import { withBackgroundEvent } from "@/lib/logging/background"; @@ -248,14 +249,17 @@ export async function runDataArrival( export async function handleDataArrival( jobs: Job[], -): Promise { - await withBackgroundEvent("job.data_arrival", async (evt) => { +): Promise { + return withBackgroundEvent("job.data_arrival", async (evt) => { + let processed = 0; + let skipped = 0; for (const job of jobs) { const arrival = job.data; try { const outcome = await runDataArrival(getWorkerPrisma(), arrival); if (outcome.status === "skipped") { + skipped++; annotate({ action: { name: `arrival.${arrival.kind}.skipped` }, meta: { reason: outcome.reason, source: arrival.source }, @@ -264,6 +268,7 @@ export async function handleDataArrival( continue; } + processed++; annotate({ action: { name: `arrival.${arrival.kind}.processed` }, meta: { @@ -285,5 +290,9 @@ export async function handleDataArrival( throw err; } } + // Both statuses are successes — a refused arrival is a decision the spine + // made, not work it failed to do. The counts are what make the split + // between them readable at all. + return jobDone({ total: jobs.length, processed, skipped }); }); } diff --git a/src/lib/jobs/document-purge.ts b/src/lib/jobs/document-purge.ts index 8f136be07..eebb5eb7e 100644 --- a/src/lib/jobs/document-purge.ts +++ b/src/lib/jobs/document-purge.ts @@ -11,6 +11,7 @@ */ import { type Job } from "pg-boss"; import { withBackgroundEvent } from "@/lib/logging/background"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { getWorkerPrisma } from "@/lib/jobs/reminder/shared"; import type { PrismaClient } from "@/generated/prisma/client"; @@ -43,16 +44,22 @@ export async function purgeExpiredDocumentTombstones( return count; } -export async function handleDocumentPurge(jobs: Job[]) { +export async function handleDocumentPurge( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.document_tombstone_purge", async (evt) => { + return withBackgroundEvent("job.document_tombstone_purge", async (evt) => { const prisma = getWorkerPrisma(); try { const purged = await purgeExpiredDocumentTombstones(prisma); evt.setAction({ name: "documents.vault.purge" }); evt.addMeta("document_purge_deleted", purged); + // A night with no expired tombstone purges nothing and is still a run + // that did its work — the zero is the fact, not an absence of one. + return jobDone({ document_purge_deleted: purged }); } catch (err) { evt.addWarning(`document-tombstone-purge failed: ${err}`); + return jobFailed("document tombstone purge failed", err); } }); } diff --git a/src/lib/jobs/encryption-key-rotate.ts b/src/lib/jobs/encryption-key-rotate.ts index be6652341..a33588426 100644 --- a/src/lib/jobs/encryption-key-rotate.ts +++ b/src/lib/jobs/encryption-key-rotate.ts @@ -24,6 +24,7 @@ */ import { type Job } from "pg-boss"; import { withBackgroundEvent } from "@/lib/logging/background"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { auditLog } from "@/lib/auth/audit"; import { rotateCorpus, @@ -60,8 +61,8 @@ export async function runEncryptionKeyRotation(): Promise<{ export async function handleEncryptionKeyRotate( jobs: Job[], -) { - await withBackgroundEvent("job.encryption_key_rotate", async (evt) => { +): Promise { + return withBackgroundEvent("job.encryption_key_rotate", async (evt) => { const requestedBy = jobs[0]?.data?.requestedByUserId ?? null; try { const result = await runEncryptionKeyRotation(); @@ -78,6 +79,15 @@ export async function handleEncryptionKeyRotate( errors: result.totalErrors, }, }); + // Per-row errors are the fail-closed skip of a row under a key the + // deployment no longer configures. The pass still ran, so they ride out + // as a count rather than failing (and retrying) the whole corpus. + return jobDone({ + rotate_active_key_id: result.activeKeyId, + rotate_scanned: result.totalScanned, + rotate_rotated: result.totalRotated, + rotate_errors: result.totalErrors, + }); } catch (err) { evt.addWarning(`encryption-key-rotate failed: ${err}`); await auditLog("admin.encryption.rotate.failed", { diff --git a/src/lib/jobs/environment-fetch.ts b/src/lib/jobs/environment-fetch.ts index b21d8379d..84c8dd7b2 100644 --- a/src/lib/jobs/environment-fetch.ts +++ b/src/lib/jobs/environment-fetch.ts @@ -33,6 +33,7 @@ import { utcDayKey, } from "@/lib/environment/service"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { workerLog } from "./reminder/shared"; export const ENVIRONMENT_FETCH_QUEUE = "environment-fetch"; @@ -152,14 +153,14 @@ export async function enqueueEnvironmentFetchDiscovery( export async function handleEnvironmentFetch( boss: PgBoss, payload: EnvironmentFetchPayload, -): Promise { +): Promise { if (!payload.userId) { const { enqueued, skipped } = await enqueueEnvironmentFetchDiscovery(boss); workerLog( "info", `[environment-fetch] discovery enqueued=${enqueued} skipped=${skipped}`, ); - return; + return jobDone({ users_enqueued: enqueued, users_skipped: skipped }); } const userId = payload.userId; @@ -168,7 +169,7 @@ export async function handleEnvironmentFetch( "info", `[environment-fetch] user=${userId} module disabled — skipping`, ); - return; + return jobDone({ skipped: "environment module disabled" }); } const { startDate, endDate } = resolveRange(payload); @@ -179,7 +180,11 @@ export async function handleEnvironmentFetch( "error", `[environment-fetch] user=${userId} range ${startDate}..${endDate} (${span}d) exceeds cap — skipping`, ); - return; + // A refused range is not an opt-out: the job was asked to fill days and + // filled none, so it says so instead of completing quietly on a stderr line. + return jobFailed("environment backfill range exceeds cap", undefined, { + span_days: span, + }); } try { @@ -192,6 +197,11 @@ export async function handleEnvironmentFetch( "info", `[environment-fetch] user=${userId} ${startDate}..${endDate} stored=${result.stored} skipped=${result.skipped} fetches=${result.fetches}`, ); + return jobDone({ + days_stored: result.stored, + days_skipped: result.skipped, + fetches: result.fetches, + }); } catch (err) { recordError(); workerLog("error", `[environment-fetch] user=${userId} failed`, err); diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts new file mode 100644 index 000000000..34b4c8b8e --- /dev/null +++ b/src/lib/jobs/job-outcome.ts @@ -0,0 +1,81 @@ +/** + * The value a background-job handler returns to say what it did. + * + * Before this type existed, a pg-boss handler signalled success by falling + * off the end of a function. Every one of the queue bindings in this tree + * resolved to `undefined`, and none of them was read anywhere, so a handler + * that caught an error, logged a warning and returned was indistinguishable + * from one that finished its work. pg-boss marked both `completed`; the + * operator surfaces that read `pgboss.job` saw nothing; the nightly pass + * looked healthy for as long as it kept failing. + * + * A returned `JobOutcome` closes that specific gap: completion is now + * derived from a value the handler had to construct, and `runJob` turns + * `ok: false` into a failed pg-boss job. What the type CANNOT do is make a + * handler honest — `jobDone()` after swallowing an exception compiles and + * passes every check in this repository. It converts omission (forgetting to + * signal a failure) into commission (claiming a success that did not + * happen), which is a smaller and much more reviewable class of defect. + * + * A fan-out pass is judged on the pass, not on its legs. A nightly sweep + * over every connected account is `ok: true` when the sweep ran, even if one + * account's grant was revoked: that belongs on the account's own integration + * ledger, and failing the queue would retry the whole cohort over one user's + * expired token. The per-leg failures ride out as a count in `did`, which is + * what makes them visible at all — today they are visible nowhere. + * + * `did` is the small pinned-shape bag of facts about the run — counts, + * identifiers, booleans. It rides into the failure report and, once the + * failure ledger lands, into the operator surface. Keep it to scalars that + * a dashboard can key on; it is not a place for payload data or free text. + */ + +/** A single fact about a run. Scalars only — never payload data. */ +export type JobFact = string | number | boolean; + +/** The pinned-shape bag of facts a handler reports about its run. */ +export type JobFacts = Readonly>; + +/** + * What a handler did. `ok: true` means the work named by the queue actually + * happened; `ok: false` means it did not, and `runJob` will fail the pg-boss + * job so the failure reaches the retry policy and the operator surfaces. + * + * A run that legitimately had nothing to do is `ok: true` with a zero count + * in `did` — absence of work is not failure. A run that could not do its + * work (provider down, query timed out, key missing) is `ok: false` even + * when the handler previously chose to carry on. + */ +export type JobOutcome = + | { + readonly ok: true; + readonly did: JobFacts; + } + | { + readonly ok: false; + /** Short, stable, greppable. `"nightscout fetch failed"`, not a stack. */ + readonly reason: string; + /** The thrown value, when there was one. Preserved for the stack. */ + readonly cause?: unknown; + /** Whatever the run managed before it failed. */ + readonly did?: JobFacts; + }; + +/** The run did its work. Pass the counts that make the run auditable. */ +export function jobDone(did: JobFacts = {}): JobOutcome { + return { ok: true, did }; +} + +/** + * The run did not do its work. `runJob` reports this and rethrows, so the + * pg-boss job fails and the queue's retry policy applies — check that policy + * when converting a handler that used to swallow, because a deterministic + * failure now retries instead of passing quietly. + */ +export function jobFailed( + reason: string, + cause?: unknown, + did?: JobFacts, +): JobOutcome { + return { ok: false, reason, cause, did }; +} diff --git a/src/lib/jobs/medication-intake-import.ts b/src/lib/jobs/medication-intake-import.ts index 2d0a365e9..2bf370f41 100644 --- a/src/lib/jobs/medication-intake-import.ts +++ b/src/lib/jobs/medication-intake-import.ts @@ -3,6 +3,7 @@ import { Prisma, type PrismaClient } from "@/generated/prisma/client"; import { invalidateUserMedications } from "@/lib/cache/invalidate"; import { prisma, toJson } from "@/lib/db"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { consumeImportedIntakesBatch } from "@/lib/medications/inventory/consumption"; import { queueMedicationIntakeSync } from "@/lib/notifications/medication-intake-sync"; import { @@ -711,9 +712,19 @@ async function recordWorkerFailure( export async function handleMedicationIntakeImport( job: Job, -): Promise { +): Promise { try { - await processMedicationIntakeImportJob(job.data.jobId); + const result = await processMedicationIntakeImportJob(job.data.jobId); + // A null result is a delivery that found the row already terminal or gone. + // There was nothing left to import, which is an absence of work rather + // than a failure to do it. + return result === null + ? jobDone({ finalized: false }) + : jobDone({ + finalized: true, + imported: result.imported, + skipped: result.skipped, + }); } catch (error) { await recordWorkerFailure(job, error); throw error; diff --git a/src/lib/jobs/pr-detection.ts b/src/lib/jobs/pr-detection.ts index e373e29c4..8062dc74e 100644 --- a/src/lib/jobs/pr-detection.ts +++ b/src/lib/jobs/pr-detection.ts @@ -2,7 +2,7 @@ * v1.4.25 W16c — Enqueue helper + handler glue for the pg-boss * `pr-detection` queue. Lives in `src/lib/jobs/` next to the other * job-side helpers; the worker process binds the handler in - * `reminder-worker.ts` via `boss.work(PR_DETECTION_QUEUE, ...)`. + * the maintenance registrar via `createAndWork(boss, PR_DETECTION_QUEUE, …)`. * * The Next.js side (ingest routes) imports `enqueuePrDetection` and * never touches pg-boss directly — `getGlobalBoss()` returns the diff --git a/src/lib/jobs/reaction-line.ts b/src/lib/jobs/reaction-line.ts index 413c8fcc0..db75ed241 100644 --- a/src/lib/jobs/reaction-line.ts +++ b/src/lib/jobs/reaction-line.ts @@ -29,6 +29,7 @@ import type { Job } from "pg-boss"; import { prisma } from "@/lib/db"; import { randomUUID } from "node:crypto"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { annotate } from "@/lib/logging/context"; import { withBackgroundEvent } from "@/lib/logging/background"; import { resolveProviderChain } from "@/lib/ai/provider"; @@ -677,13 +678,16 @@ export async function runReactionLine( export async function handleReactionLine( jobs: Job[], -): Promise { - await withBackgroundEvent("job.reaction_line", async (evt) => { +): Promise { + return withBackgroundEvent("job.reaction_line", async (evt) => { + let generated = 0; + let skipped = 0; for (const job of jobs) { try { const outcome = await runReactionLine(job.data); if (outcome.status === "skipped") { + skipped++; annotate({ action: { name: "arrival.reaction_line.skipped" }, meta: { reason: outcome.reason, kind: job.data.kind }, @@ -692,6 +696,7 @@ export async function handleReactionLine( continue; } + generated++; annotate({ action: { name: "arrival.reaction_line.generated" }, meta: { kind: job.data.kind, local_date: job.data.localDate }, @@ -705,5 +710,9 @@ export async function handleReactionLine( throw err; } } + // A skip is a refusal the surface is designed around, not a miss: the + // deterministic lead already stands without a line. The pass did its work + // whenever it ran to the end. + return jobDone({ total: jobs.length, generated, skipped }); }); } diff --git a/src/lib/jobs/reminder-worker.ts b/src/lib/jobs/reminder-worker.ts index 59c7036db..f8a20a384 100644 --- a/src/lib/jobs/reminder-worker.ts +++ b/src/lib/jobs/reminder-worker.ts @@ -8,8 +8,8 @@ * the domain registrars under `src/lib/jobs/reminder/`. Each registrar owns the * four facts the v1.4.37 dead-queue guards pin — the queue-name constant, its * `allQueues` membership, its `[QUEUE, CRON]` schedule tuple, and its - * `boss.work(QUEUE, …, handler)` binding — so a queue can never be declared - * without being provisioned, scheduled, and drained. + * `createAndWork(boss, QUEUE, …, handler)` binding — so a queue can never be + * declared without being provisioned, scheduled, and drained. * * Usage: Run as a standalone process or call startReminderWorker() from a * custom server setup. In dev, use: npx tsx src/lib/jobs/reminder-worker.ts diff --git a/src/lib/jobs/reminder/__tests__/poll-cohort.test.ts b/src/lib/jobs/reminder/__tests__/poll-cohort.test.ts index f0dc9123d..111c255ab 100644 --- a/src/lib/jobs/reminder/__tests__/poll-cohort.test.ts +++ b/src/lib/jobs/reminder/__tests__/poll-cohort.test.ts @@ -107,7 +107,17 @@ describe("makePollCohortHandler — marker-aware recordFailure boundary", () => recordFailure, }); - await expect(handler([job()])).resolves.toBeUndefined(); + // The recorder rejection is warned, not propagated: the pass ran, so the + // job is done and the failing user is a `users_failed` count. + await expect(handler([job()])).resolves.toEqual({ + ok: true, + did: { + total: 2, + users_synced: 1, + users_failed: 1, + measurements_imported: 1, + }, + }); // The next user still synced despite the recorder throwing. expect(syncUser).toHaveBeenCalledTimes(2); expect(syncUser).toHaveBeenLastCalledWith("next"); diff --git a/src/lib/jobs/reminder/backup-handlers.ts b/src/lib/jobs/reminder/backup-handlers.ts index 1106159ed..6d23f8a44 100644 --- a/src/lib/jobs/reminder/backup-handlers.ts +++ b/src/lib/jobs/reminder/backup-handlers.ts @@ -8,8 +8,12 @@ import { type Job } from "pg-boss"; import { recordError } from "@/lib/jobs/worker-status"; import { buildFullBackupPayload } from "@/lib/export/full-backup-payload"; import { encrypt } from "@/lib/crypto"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; -import { runOffhostBackup } from "@/lib/jobs/offhost-backup"; +import { + OffhostBackupNotConfiguredError, + runOffhostBackup, +} from "@/lib/jobs/offhost-backup"; import { getWorkerPrisma } from "./shared"; export interface DataBackupPayload { @@ -20,9 +24,11 @@ export interface OffhostBackupPayload { triggeredAt: string; } -export async function handleOffhostBackup(jobs: Job[]) { +export async function handleOffhostBackup( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.offhost_backup", async (evt) => { + return withBackgroundEvent("job.offhost_backup", async (evt) => { const p = getWorkerPrisma(); try { const report = await runOffhostBackup(p); @@ -39,16 +45,34 @@ export async function handleOffhostBackup(jobs: Job[]) { JSON.stringify(report.failures.slice(0, 10)), ); } + // Per-user upload failures ride out as `offhost_backup_failed`: the run + // itself uploaded what it could, and failing the queue over one user's + // object would re-upload the whole cohort on every retry. + return jobDone({ + offhost_backup_uploaded: report.uploaded, + offhost_backup_failed: report.failed, + offhost_backup_total_users: report.totalUsers, + offhost_backup_endpoint: report.config.endpoint, + offhost_backup_bucket: report.config.bucket, + }); } catch (err) { - // Not configured ⇒ skip silently with a warning, not an error. + // Not configured ⇒ skip silently with a warning, not an error: most + // self-hosts never set the S3 credentials, and a nightly failed job on + // every one of them buries the runs that genuinely could not upload. evt.addWarning(`offhost-backup skipped/failed: ${err}`); + if (err instanceof OffhostBackupNotConfiguredError) { + return jobDone({ offhost_backup_configured: false }); + } + return jobFailed("offhost backup failed", err); } }); } -export async function handleDataBackup(jobs: Job[]) { +export async function handleDataBackup( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.data_backup", async (evt) => { + return withBackgroundEvent("job.data_backup", async (evt) => { const prisma = getWorkerPrisma(); try { const users = await prisma.user.findMany({ @@ -56,6 +80,7 @@ export async function handleDataBackup(jobs: Job[]) { }); let backed = 0; + let usersFailed = 0; for (const user of users) { try { const { payload } = await buildFullBackupPayload(prisma, user.id, { @@ -82,6 +107,10 @@ export async function handleDataBackup(jobs: Job[]) { }); backed++; } catch (err) { + // One user's payload failing is that user's problem, not the pass's: + // it rides out as a count so the weekly run is not retried for the + // whole cohort. + usersFailed++; evt.addWarning(`Failed for user ${user.id}: ${err}`); } } @@ -90,6 +119,11 @@ export async function handleDataBackup(jobs: Job[]) { task_name: "job.data_backup", result: { backed, total: users.length }, }); + return jobDone({ + backed, + total: users.length, + users_failed: usersFailed, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/reminder/cleanup-handlers.ts b/src/lib/jobs/reminder/cleanup-handlers.ts index a4191a3c0..145ac3e30 100644 --- a/src/lib/jobs/reminder/cleanup-handlers.ts +++ b/src/lib/jobs/reminder/cleanup-handlers.ts @@ -2,10 +2,27 @@ * Retention and maintenance cleanups: rate-limit rows, idempotency keys, audit logs, OAuth states (Withings / WHOOP), mood-reminder events, push attempts, and measurement tombstones. * * Extracted from reminder-worker.ts, which owns the queue names, cron - * schedules, and boss.work registrations. + * schedules, and the queue bindings. + * + * Every pass in here used to catch its own failure, write a warning nobody + * reads, and return — so a retention purge that had not run for weeks was + * indistinguishable from one with nothing to delete. They now return + * `jobFailed`, which fails the pg-boss job and puts the queue on the operator's + * failing-jobs surface. + * + * That change has a price, and the registrars pay it: these are bulk DELETEs + * over a retention horizon, and their failure mode is deterministic. A + * statement that timed out against a large trailing edge times out again + * immediately. Under pg-boss's default policy (retryLimit 2, no delay, no + * backoff) each converted queue would run the same doomed statement three + * times in a row every night. So every cron-driven cleanup registered from + * this module is scheduled with `retryLimit: 0`: the failure is recorded once + * and the next night's tick is the retry, which is the right backoff for + * retention work that has a whole day of slack. */ import { type Job } from "pg-boss"; import { withBackgroundEvent } from "@/lib/logging/background"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { cleanupExpiredWhoopConnectTickets, cleanupExpiredWhoopOAuthStates, @@ -78,9 +95,9 @@ export interface MoodReminderCleanupPayload { export async function handleMoodReminderCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.mood_reminder_cleanup", async (evt) => { + return withBackgroundEvent("job.mood_reminder_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const cutoff = new Date(); @@ -90,8 +107,10 @@ export async function handleMoodReminderCleanup( where: { date: { lt: cutoffIso } }, }); evt.addMeta("mood_reminder_cleanup_deleted", deleted.count); + return jobDone({ deleted: deleted.count }); } catch (err) { evt.addWarning(`mood-reminder-cleanup failed: ${err}`); + return jobFailed("mood-reminder cleanup failed", err); } }); } @@ -118,9 +137,9 @@ export interface PushAttemptCleanupPayload { export async function handlePushAttemptCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.push_attempt_cleanup", async (evt) => { + return withBackgroundEvent("job.push_attempt_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const cutoff = new Date(); @@ -129,8 +148,10 @@ export async function handlePushAttemptCleanup( where: { createdAt: { lt: cutoff } }, }); evt.addMeta("push_attempt_cleanup_deleted", deleted.count); + return jobDone({ deleted: deleted.count }); } catch (err) { evt.addWarning(`push-attempt-cleanup failed: ${err}`); + return jobFailed("push-attempt cleanup failed", err); } }); } @@ -152,9 +173,9 @@ export interface ArrivalReactionCleanupPayload { export async function handleArrivalReactionCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.arrival_reaction_cleanup", async (evt) => { + return withBackgroundEvent("job.arrival_reaction_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const cutoff = new Date(); @@ -163,8 +184,10 @@ export async function handleArrivalReactionCleanup( where: { createdAt: { lt: cutoff } }, }); evt.addMeta("arrival_reaction_cleanup_deleted", deleted.count); + return jobDone({ deleted: deleted.count }); } catch (err) { evt.addWarning(`arrival-reaction-cleanup failed: ${err}`); + return jobFailed("arrival-reaction cleanup failed", err); } }); } @@ -175,9 +198,9 @@ export interface MeasurementTombstoneCleanupPayload { export async function handleMeasurementTombstoneCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent( + return withBackgroundEvent( "job.measurement_tombstone_cleanup", async (evt) => { const p = getWorkerPrisma(); @@ -208,6 +231,12 @@ export async function handleMeasurementTombstoneCleanup( "tombstone-cleanup stopped at the batch cap; a backlog remains for the next run", ); } + return jobDone({ + measurements_pruned: measurements.deleted, + mood_pruned: mood.deleted, + intakes_pruned: intakes.deleted, + drained, + }); } catch (err) { // Rethrow. A retention purge that fails silently every night looks // exactly like one that has nothing to do; pg-boss recording a failed @@ -221,32 +250,36 @@ export async function handleMeasurementTombstoneCleanup( export async function handleRateLimitCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.rate_limit_cleanup", async (evt) => { + return withBackgroundEvent("job.rate_limit_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const result = await p.$executeRaw` DELETE FROM rate_limits WHERE reset_at < NOW() `; evt.addMeta("rate_limit_cleanup_deleted", result); + return jobDone({ deleted: result }); } catch (err) { evt.addWarning(`rate-limit-cleanup failed: ${err}`); + return jobFailed("rate-limit cleanup failed", err); } }); } export async function handleIdempotencyCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.idempotency_cleanup", async (evt) => { + return withBackgroundEvent("job.idempotency_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupExpiredIdempotencyKeys(p); evt.addMeta("idempotency_cleanup_deleted", deleted); + return jobDone({ deleted }); } catch (err) { evt.addWarning(`idempotency-cleanup failed: ${err}`); + return jobFailed("idempotency-key cleanup failed", err); } }); } @@ -266,26 +299,28 @@ export async function handleIdempotencyCleanup( */ export async function handleStepUpElevationCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.step_up_elevation_cleanup", async (evt) => { + return withBackgroundEvent("job.step_up_elevation_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const { count } = await p.stepUpElevation.deleteMany({ where: { expiresAt: { lt: new Date() } }, }); evt.addMeta("step_up_elevation_cleanup_deleted", count); + return jobDone({ deleted: count }); } catch (err) { evt.addWarning(`step-up-elevation-cleanup failed: ${err}`); + return jobFailed("step-up elevation cleanup failed", err); } }); } export async function handleAuditLogCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.audit_log_cleanup", async (evt) => { + return withBackgroundEvent("job.audit_log_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const outcome = await cleanupOldAuditLogs(p); @@ -296,6 +331,7 @@ export async function handleAuditLogCleanup( "audit-log-cleanup stopped at the batch cap; a backlog remains for the next run", ); } + return jobDone({ deleted: outcome.deleted, drained: outcome.drained }); } catch (err) { // Rethrow — see the tombstone handler. Storage-limitation retention that // fails quietly is retention that is not happening. @@ -307,15 +343,17 @@ export async function handleAuditLogCleanup( export async function handleCoachMessageCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.coach_message_cleanup", async (evt) => { + return withBackgroundEvent("job.coach_message_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupOldCoachMessages(p); evt.addMeta("coach_message_cleanup_deleted", deleted); + return jobDone({ deleted }); } catch (err) { evt.addWarning(`coach-message-cleanup failed: ${err}`); + return jobFailed("coach-message cleanup failed", err); } }); } @@ -326,9 +364,9 @@ export interface McpTokenCleanupPayload { export async function handleMcpTokenCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.mcp_token_cleanup", async (evt) => { + return withBackgroundEvent("job.mcp_token_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const result = await cleanupExpiredMcpTokens(p); @@ -340,27 +378,40 @@ export async function handleMcpTokenCleanup( "mcp_token_cleanup_connections_deleted", result.connectionsDeleted, ); + return jobDone({ + access_tokens_deleted: result.accessTokensDeleted, + connections_deleted: result.connectionsDeleted, + }); } catch (err) { evt.addWarning(`mcp-token-cleanup failed: ${err}`); + return jobFailed("mcp-token cleanup failed", err); } }); } export async function handleWithingsOAuthStateCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.withings_oauth_state_cleanup", async (evt) => { - const p = getWorkerPrisma(); - try { - const deleted = await cleanupExpiredWithingsOAuthStates(p); - evt.addMeta("withings_oauth_state_cleanup_deleted", deleted); - } catch (err) { - // The OAuth flow tolerates a stale row sticking around for an - // extra day — log + carry on so the boss queue doesn't retry-loop. - evt.addWarning(`withings-oauth-state-cleanup failed: ${err}`); - } - }); + return withBackgroundEvent( + "job.withings_oauth_state_cleanup", + async (evt) => { + const p = getWorkerPrisma(); + try { + const deleted = await cleanupExpiredWithingsOAuthStates(p); + evt.addMeta("withings_oauth_state_cleanup_deleted", deleted); + return jobDone({ deleted }); + } catch (err) { + // The OAuth flow tolerates a stale row for an extra day, which used + // to be the argument for swallowing this so the queue would not + // retry-loop. Tolerable is not the same as invisible: the failure is + // now reported, and the retry-loop worry is answered where it belongs, + // by the `retryLimit: 0` this queue is scheduled with. + evt.addWarning(`withings-oauth-state-cleanup failed: ${err}`); + return jobFailed("withings oauth-state cleanup failed", err); + } + }, + ); } export interface OidcNativeHandoffCleanupPayload { @@ -369,17 +420,21 @@ export interface OidcNativeHandoffCleanupPayload { export async function handleOidcNativeHandoffCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.oidc_native_handoff_cleanup", async (evt) => { + return withBackgroundEvent("job.oidc_native_handoff_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupExpiredOidcNativeHandoffs(p); evt.addMeta("oidc_native_handoff_cleanup_deleted", deleted); + return jobDone({ deleted }); } catch (err) { - // The handoff flow tolerates a stale row for an extra day (expiry is - // enforced at read) — log + carry on so the boss queue doesn't retry-loop. + // Expiry is enforced at read, so a stale row for an extra day is + // harmless — but see the withings twin above: harmless is not a + // reason to hide the failure. `retryLimit: 0` on the schedule is what + // keeps a doomed statement from running three times a night. evt.addWarning(`oidc-native-handoff-cleanup failed: ${err}`); + return jobFailed("oidc native-handoff cleanup failed", err); } }); } @@ -390,17 +445,22 @@ export interface WhoopOAuthStateCleanupPayload { export async function handleWhoopOAuthStateCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.whoop_oauth_state_cleanup", async (evt) => { + return withBackgroundEvent("job.whoop_oauth_state_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupExpiredWhoopOAuthStates(p); evt.addMeta("whoop_oauth_state_cleanup_deleted", deleted); const ticketsDeleted = await cleanupExpiredWhoopConnectTickets(p); evt.addMeta("whoop_connect_ticket_cleanup_deleted", ticketsDeleted); + return jobDone({ + states_deleted: deleted, + connect_tickets_deleted: ticketsDeleted, + }); } catch (err) { evt.addWarning(`whoop-oauth-state-cleanup failed: ${err}`); + return jobFailed("whoop oauth-state cleanup failed", err); } }); } diff --git a/src/lib/jobs/reminder/fitbit-sync.ts b/src/lib/jobs/reminder/fitbit-sync.ts index d9a45b895..e55886ff5 100644 --- a/src/lib/jobs/reminder/fitbit-sync.ts +++ b/src/lib/jobs/reminder/fitbit-sync.ts @@ -7,6 +7,7 @@ import { type Job } from "pg-boss"; import { fireAndForget } from "@/lib/logging/fire-and-forget"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { runFitbitPollCohort } from "@/lib/fitbit/sync"; import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; @@ -23,8 +24,10 @@ export interface FitbitSyncPayload { userId?: string; } -export async function handleFitbitSync(jobs: Job[]) { - await withBackgroundEvent("job.fitbit_sync", async (evt) => { +export async function handleFitbitSync( + jobs: Job[], +): Promise { + return withBackgroundEvent("job.fitbit_sync", async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; @@ -37,16 +40,26 @@ export async function handleFitbitSync(jobs: Job[]) { }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } // Fan the cohort out with bounded concurrency + per-user error isolation: // one slow Google response can't stall the whole cohort, and a single // user's failure is warned without aborting the pass. + let usersFailed = 0; const { usersSynced, measurementsImported } = await runFitbitPollCohort( targets.map((t) => t.userId), { - onUserError: (userId, err) => - evt.addWarning(`job.fitbit_sync failed for user ${userId}: ${err}`), + onUserError: (userId, err) => { + usersFailed++; + evt.addWarning(`job.fitbit_sync failed for user ${userId}: ${err}`); + }, // v1.18.1 — a fresh reading landed; resolve the user's Vorsorge // reminders eventfully. Fire-and-forget; the cron is the net. onUserSynced: (userId, imported) => { @@ -67,6 +80,14 @@ export async function handleFitbitSync(jobs: Job[]) { measurements_imported: measurementsImported, }, }); + // The pass is the unit of success: a revoked grant belongs on that user's + // integration ledger, not in a queue failure that retries every account. + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); @@ -81,15 +102,17 @@ export interface FitbitOAuthStateCleanupPayload { export async function handleFitbitOAuthStateCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.fitbit_oauth_state_cleanup", async (evt) => { + return withBackgroundEvent("job.fitbit_oauth_state_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupExpiredFitbitOAuthStates(p); evt.addMeta("fitbit_oauth_state_cleanup_deleted", deleted); + return jobDone({ deleted }); } catch (err) { evt.addWarning(`fitbit-oauth-state-cleanup failed: ${err}`); + return jobFailed("fitbit oauth state cleanup failed", err); } }); } diff --git a/src/lib/jobs/reminder/google-health-sync.ts b/src/lib/jobs/reminder/google-health-sync.ts index 0006f5ca8..d6791814b 100644 --- a/src/lib/jobs/reminder/google-health-sync.ts +++ b/src/lib/jobs/reminder/google-health-sync.ts @@ -7,6 +7,7 @@ import { type Job } from "pg-boss"; import { fireAndForget } from "@/lib/logging/fire-and-forget"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { runGoogleHealthPollCohort } from "@/lib/google-health/sync"; import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; @@ -26,8 +27,8 @@ export interface GoogleHealthSyncPayload { export async function handleGoogleHealthSync( jobs: Job[], -) { - await withBackgroundEvent("job.google_health_sync", async (evt) => { +): Promise { + return withBackgroundEvent("job.google_health_sync", async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; @@ -40,19 +41,29 @@ export async function handleGoogleHealthSync( }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } // Fan the cohort out with bounded concurrency + per-user error isolation: // one slow Google response can't stall the whole cohort, and a single // user's failure is warned without aborting the pass. + let usersFailed = 0; const { usersSynced, measurementsImported } = await runGoogleHealthPollCohort( targets.map((t) => t.userId), { - onUserError: (userId, err) => + onUserError: (userId, err) => { + usersFailed++; evt.addWarning( `job.google_health_sync failed for user ${userId}: ${err}`, - ), + ); + }, // A fresh reading landed; resolve the user's Vorsorge reminders // eventfully. Fire-and-forget; the cron is the net. onUserSynced: (userId, imported) => { @@ -73,6 +84,14 @@ export async function handleGoogleHealthSync( measurements_imported: measurementsImported, }, }); + // The pass is the unit of success: a revoked grant belongs on that user's + // integration ledger, not in a queue failure that retries every account. + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); @@ -87,17 +106,19 @@ export interface GoogleHealthOAuthStateCleanupPayload { export async function handleGoogleHealthOAuthStateCleanup( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent( + return withBackgroundEvent( "job.google_health_oauth_state_cleanup", async (evt) => { const p = getWorkerPrisma(); try { const deleted = await cleanupExpiredGoogleHealthOAuthStates(p); evt.addMeta("google_health_oauth_state_cleanup_deleted", deleted); + return jobDone({ deleted }); } catch (err) { evt.addWarning(`google-health-oauth-state-cleanup failed: ${err}`); + return jobFailed("google health oauth state cleanup failed", err); } }, ); diff --git a/src/lib/jobs/reminder/insights-handlers.ts b/src/lib/jobs/reminder/insights-handlers.ts index 2c7117714..fd275f43c 100644 --- a/src/lib/jobs/reminder/insights-handlers.ts +++ b/src/lib/jobs/reminder/insights-handlers.ts @@ -19,6 +19,7 @@ import { type InsightStatusGeneratePayload, runInsightStatusGenerate, } from "@/lib/jobs/insight-status-generate"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { generateBloodPressureStatusForUser } from "@/lib/insights/blood-pressure-status"; import { generateWeightStatusForUser } from "@/lib/insights/weight-status"; @@ -69,6 +70,10 @@ export interface MedicationComplianceStatusPayload { * `status-cron-candidates.ts` for the full division of nightly labour). * The generators normalise `locale` themselves (de stays de, everything * else gets English prose). + * + * The pass is what succeeds or fails. One user whose generation threw is that + * user's cold card, reported as the `failed` count; failing the job would + * re-run the whole cohort over it. */ export async function runStatusCronGenerate( taskName: string, @@ -76,13 +81,15 @@ export async function runStatusCronGenerate( userId: string, options: { locale: string | null; force: boolean }, ) => Promise, -): Promise { - await withBackgroundEvent(taskName, async (evt) => { +): Promise { + return withBackgroundEvent(taskName, async (evt) => { const prisma = getWorkerPrisma(); try { const users = await findStatusCronCandidates(prisma); - if (users.length === 0) return; + if (users.length === 0) { + return jobDone({ total: 0, generated: 0, failed: 0 }); + } let generated = 0; let failed = 0; @@ -103,6 +110,7 @@ export async function runStatusCronGenerate( task_name: taskName, result: { generated, failed, total: users.length }, }); + return jobDone({ total: users.length, generated, failed }); } catch (err) { evt.setError(err); recordError(); @@ -129,13 +137,15 @@ export async function runStatusCronGenerate( * inside the batch itself). Net effect: the nightly ladder pays one call per * user instead of seven, with no queue/registry churn. */ -async function runStatusBatchCron(taskName: string): Promise { - await withBackgroundEvent(taskName, async (evt) => { +async function runStatusBatchCron(taskName: string): Promise { + return withBackgroundEvent(taskName, async (evt) => { const prisma = getWorkerPrisma(); try { recordInsightsRun(); const users = await findStatusCronCandidates(prisma); - if (users.length === 0) return; + if (users.length === 0) { + return jobDone({ total: 0, generated: 0, served: 0, failed: 0 }); + } let generated = 0; let served = 0; @@ -167,6 +177,10 @@ async function runStatusBatchCron(taskName: string): Promise { task_name: taskName, result: { generated, served, failed, total: users.length }, }); + // A user whose batch call threw already fell back to the single-card + // path inside the batch, and the six later per-metric crons cover what + // is still cold. The batch pass ran, so it reports the count. + return jobDone({ total: users.length, generated, served, failed }); } catch (err) { evt.setError(err); recordError(); @@ -176,14 +190,16 @@ async function runStatusBatchCron(taskName: string): Promise { }); } -export function handleGeneralStatusGenerate(jobs: Job[]) { +export function handleGeneralStatusGenerate( + jobs: Job[], +): Promise { void jobs; return runStatusBatchCron("job.insights.batch"); } export function handleBloodPressureStatusGenerate( jobs: Job[], -) { +): Promise { void jobs; return runStatusCronGenerate( "job.insights.blood_pressure", @@ -191,7 +207,9 @@ export function handleBloodPressureStatusGenerate( ); } -export function handleWeightStatusGenerate(jobs: Job[]) { +export function handleWeightStatusGenerate( + jobs: Job[], +): Promise { void jobs; return runStatusCronGenerate( "job.insights.weight", @@ -199,7 +217,9 @@ export function handleWeightStatusGenerate(jobs: Job[]) { ); } -export function handlePulseStatusGenerate(jobs: Job[]) { +export function handlePulseStatusGenerate( + jobs: Job[], +): Promise { void jobs; return runStatusCronGenerate( "job.insights.pulse", @@ -207,19 +227,23 @@ export function handlePulseStatusGenerate(jobs: Job[]) { ); } -export function handleBmiStatusGenerate(jobs: Job[]) { +export function handleBmiStatusGenerate( + jobs: Job[], +): Promise { void jobs; return runStatusCronGenerate("job.insights.bmi", generateBmiStatusForUser); } -export function handleMoodStatusGenerate(jobs: Job[]) { +export function handleMoodStatusGenerate( + jobs: Job[], +): Promise { void jobs; return runStatusCronGenerate("job.insights.mood", generateMoodStatusForUser); } export function handleMedicationComplianceStatusGenerate( jobs: Job[], -) { +): Promise { void jobs; return runStatusCronGenerate( "job.insights.medication_compliance", @@ -229,8 +253,8 @@ export function handleMedicationComplianceStatusGenerate( export async function handleInsightPregenerateJob( jobs: Job[], -) { - await withBackgroundEvent("job.insight_pregenerate", async (evt) => { +): Promise { + return withBackgroundEvent("job.insight_pregenerate", async (evt) => { // v1.8.7.1 — a forced single-user warm carries `{ userId, force }`; // the scheduled tick carries neither. Route each job individually so a // batch that mixes a cron tick with on-demand warms (it never does in @@ -265,13 +289,30 @@ export async function handleInsightPregenerateJob( } } - if (scheduled.length === 0) return; + // A batch of forced warms only is a complete run: every warm either landed + // or threw out of the loop above. + if (scheduled.length === 0) { + return jobDone({ forced: forced.length, scheduled: 0 }); + } try { const summary = await runInsightPregenerate(getWorkerPrisma()); evt.setBackground({ task_name: "job.insight_pregenerate", result: { ...summary }, }); + return jobDone({ + forced: forced.length, + scheduled: scheduled.length, + total: summary.total, + generated: summary.generated, + cached: summary.cached, + unchanged: summary.unchanged, + skipped: summary.skipped, + failed: summary.failed, + budget_blocked: summary.budgetBlocked, + assessments_warmed: summary.assessmentsWarmed, + metric_assessments_warmed: summary.metricAssessmentsWarmed, + }); } catch (err) { evt.addWarning( `insight-pregenerate failed: ${ @@ -296,12 +337,18 @@ export async function handleInsightPregenerateJob( */ export async function handleInsightStatusGenerate( jobs: Job[], -) { - await withBackgroundEvent("job.insight_status_generate", async (evt) => { +): Promise { + return withBackgroundEvent("job.insight_status_generate", async (evt) => { + let generated = 0; + let malformed = 0; for (const job of jobs) { - if (!job.data?.userId || !job.data?.metric) continue; + if (!job.data?.userId || !job.data?.metric) { + malformed++; + continue; + } try { await runInsightStatusGenerate(job.data); + generated++; evt.addMeta( "status_generated", `${job.data.metric}:${job.data.locale}`, @@ -321,5 +368,8 @@ export async function handleInsightStatusGenerate( throw err; } } + // Every job in the batch either generated a card or carried no metric to + // generate; a generation that failed left through the rethrow above. + return jobDone({ generated, malformed }); }); } diff --git a/src/lib/jobs/reminder/medication-maintenance.ts b/src/lib/jobs/reminder/medication-maintenance.ts index 326cb8cd3..b906eb8f6 100644 --- a/src/lib/jobs/reminder/medication-maintenance.ts +++ b/src/lib/jobs/reminder/medication-maintenance.ts @@ -10,6 +10,7 @@ import { type IntakeAutoSkipPayload, runIntakeAutoSkipPass, } from "@/lib/jobs/intake-auto-skip"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { expireStaleInUseItems } from "@/lib/medications/inventory/service"; import { getWorkerPrisma } from "./shared"; @@ -21,18 +22,21 @@ import { getWorkerPrisma } from "./shared"; */ export async function handleMedicationInventoryExpire( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.medication_inventory_expire", async (evt) => { + return withBackgroundEvent("job.medication_inventory_expire", async (evt) => { try { const count = await expireStaleInUseItems({ nowMs: Date.now() }); evt.addMeta("inventory_expired_count", count); + // A pass that found no lapsed pen did its work; zero is a result. + return jobDone({ inventory_expired_count: count }); } catch (err) { evt.addWarning( `medication-inventory-expire failed: ${ err instanceof Error ? err.message : String(err) }`, ); + return jobFailed("inventory expire pass failed", err); } }); } @@ -46,9 +50,11 @@ export async function handleMedicationInventoryExpire( * unit test can drive it with an in-memory fake Prisma; this wrapper * threads the worker's pg-boss + background-event plumbing. */ -export async function handleIntakeAutoSkip(jobs: Job[]) { +export async function handleIntakeAutoSkip( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.intake_auto_skip", async (evt) => { + return withBackgroundEvent("job.intake_auto_skip", async (evt) => { const prisma = getWorkerPrisma(); try { const result = await runIntakeAutoSkipPass(prisma, { @@ -56,12 +62,17 @@ export async function handleIntakeAutoSkip(jobs: Job[]) { }); evt.addMeta("intake_auto_skip_count", result.skippedCount); evt.addMeta("intake_auto_skip_cutoff", result.cutoff.toISOString()); + return jobDone({ + intake_auto_skip_count: result.skippedCount, + intake_auto_skip_cutoff: result.cutoff.toISOString(), + }); } catch (err) { evt.addWarning( `intake-auto-skip failed: ${ err instanceof Error ? err.message : String(err) }`, ); + return jobFailed("intake auto-skip pass failed", err); } }); } diff --git a/src/lib/jobs/reminder/medication-reminder-check.ts b/src/lib/jobs/reminder/medication-reminder-check.ts index ff64ab72d..a21f07939 100644 --- a/src/lib/jobs/reminder/medication-reminder-check.ts +++ b/src/lib/jobs/reminder/medication-reminder-check.ts @@ -8,6 +8,7 @@ import { type Job } from "pg-boss"; import { recordError, recordReminderCheck } from "@/lib/jobs/worker-status"; import { recomputeMedicationComplianceForEvent } from "@/lib/rollups/medication-compliance-rollups"; import { decrypt } from "@/lib/crypto"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { dispatchNotification } from "@/lib/notifications/dispatcher"; import { medicationDoseTag } from "@/lib/notifications/dose-tag"; @@ -137,9 +138,11 @@ export async function cleanupScheduledTelegramDeletions(): Promise { * Uses phase-based logic (GREEN/YELLOW/ORANGE/RED) to send one notification * per phase transition rather than every 15 minutes. */ -export async function handleReminderCheck(jobs: Job[]) { +export async function handleReminderCheck( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.medication_reminder", async (evt) => { + return withBackgroundEvent("job.medication_reminder", async (evt) => { const prisma = getWorkerPrisma(); try { recordReminderCheck(); @@ -176,6 +179,12 @@ export async function handleReminderCheck(jobs: Job[]) { }, }); + // The per-slot facts the tick already emits as wide-event meta, counted + // so the run's outcome carries them as scalars. + let missedDosesMinted = 0; + let notificationsDispatched = 0; + let notificationsFailed = 0; + for (const med of medications) { const userTz = med.user.timezone || "Europe/Berlin"; const { start: todayStart, end: todayEnd } = getUserTodayBounds( @@ -440,6 +449,7 @@ export async function handleReminderCheck(jobs: Job[]) { }, }); + missedDosesMinted++; evt.addMeta("missed_dose", `${med.name}:${slotTime}`); // v1.4.39 W-MED — refresh the compliance rollup so the @@ -521,7 +531,12 @@ export async function handleReminderCheck(jobs: Job[]) { url: `/medications/${med.id}`, }, }); + notificationsDispatched++; } catch (notifErr) { + // One user's channel failing must not fail the tick — the + // dedup anchor is already stamped, so the count is the only + // trace this slot's notice never left the host. + notificationsFailed++; evt.addWarning( `Notification dispatch failed for ${currentPhase} phase ${med.name}: ${notifErr}`, ); @@ -530,6 +545,13 @@ export async function handleReminderCheck(jobs: Job[]) { } } } + + return jobDone({ + medications: medications.length, + missed_doses_minted: missedDosesMinted, + notifications_dispatched: notificationsDispatched, + notifications_failed: notificationsFailed, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/reminder/mood-cycle-checks.ts b/src/lib/jobs/reminder/mood-cycle-checks.ts index afa03a835..53b783c9d 100644 --- a/src/lib/jobs/reminder/mood-cycle-checks.ts +++ b/src/lib/jobs/reminder/mood-cycle-checks.ts @@ -6,6 +6,7 @@ */ import { type Job } from "pg-boss"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { runMoodReminderTick } from "@/lib/jobs/mood-reminder"; import { runCycleReminderTick } from "@/lib/jobs/cycle-reminder"; @@ -30,9 +31,9 @@ export interface MeasurementReminderPayload { export async function handleMoodReminderCheck( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.mood_reminder", async (evt) => { + return withBackgroundEvent("job.mood_reminder", async (evt) => { const prisma = getWorkerPrisma(); try { const summary = await runMoodReminderTick(prisma, new Date()); @@ -47,6 +48,16 @@ export async function handleMoodReminderCheck( skipped_outside_window: summary.skippedOutsideWindow, }, }); + // A tick that found nobody in their local reminder hour still ran the + // sweep, so it is a success with zero counts. + return jobDone({ + candidates_scanned: summary.candidatesScanned, + in_window: summary.inWindow, + dispatched: summary.dispatched, + skipped_already_logged: summary.skippedAlreadyLogged, + skipped_already_dispatched: summary.skippedAlreadyDispatched, + skipped_outside_window: summary.skippedOutsideWindow, + }); } catch (err) { evt.setError(err); recordError(); @@ -63,9 +74,9 @@ export async function handleMoodReminderCheck( */ export async function handleCycleReminderCheck( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.cycle_reminder", async (evt) => { + return withBackgroundEvent("job.cycle_reminder", async (evt) => { const prisma = getWorkerPrisma(); try { const summary = await runCycleReminderTick(prisma, new Date()); @@ -82,6 +93,16 @@ export async function handleCycleReminderCheck( skipped_outside_window: summary.skippedOutsideWindow, }, }); + return jobDone({ + candidates_scanned: summary.candidatesScanned, + in_window: summary.inWindow, + dispatched_period_soon: summary.dispatchedPeriodSoon, + dispatched_period_confirm: summary.dispatchedPeriodConfirm, + suppressed_client_managed: summary.suppressedClientManaged, + suppressed_discreet: summary.suppressedDiscreet, + skipped_already_notified: summary.skippedAlreadyNotified, + skipped_outside_window: summary.skippedOutsideWindow, + }); } catch (err) { evt.setError(err); recordError(); @@ -97,9 +118,9 @@ export async function handleCycleReminderCheck( */ export async function handleMeasurementReminderCheck( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.measurement_reminder", async (evt) => { + return withBackgroundEvent("job.measurement_reminder", async (evt) => { const prisma = getWorkerPrisma(); try { const summary = await runMeasurementReminderTick(prisma, new Date()); @@ -116,6 +137,19 @@ export async function handleMeasurementReminderCheck( failed: summary.failed, }, }); + // `failed` counts per-reminder dispatch failures the tick isolated. The + // sweep itself ran, so those ride out as a count rather than failing the + // job and retrying the whole cohort over one user's channel. + return jobDone({ + candidates_scanned: summary.candidatesScanned, + in_window: summary.inWindow, + dispatched: summary.dispatched, + auto_resolved: summary.autoResolved, + skipped_not_due: summary.skippedNotDue, + skipped_outside_window: summary.skippedOutsideWindow, + skipped_no_channel: summary.skippedNoChannel, + failed: summary.failed, + }); } catch (err) { evt.setError(err); recordError(); @@ -133,8 +167,8 @@ export async function handleMeasurementReminderCheck( */ export async function handleReminderSatisfy( jobs: Job[], -) { - await withBackgroundEvent("job.reminder_satisfy", async (evt) => { +): Promise { + return withBackgroundEvent("job.reminder_satisfy", async (evt) => { const prisma = getWorkerPrisma(); const now = new Date(); // Distinct users in this batch — a multi-source ingest spike can enqueue @@ -160,6 +194,13 @@ export async function handleReminderSatisfy( satisfied, }, }); + // A batch whose jobs carried no usable userId resolved nothing and is + // still a success with zero counts. + return jobDone({ + users: userIds.length, + candidates_scanned: candidates, + satisfied, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/reminder/ops-handlers.ts b/src/lib/jobs/reminder/ops-handlers.ts index a03154056..b483243c5 100644 --- a/src/lib/jobs/reminder/ops-handlers.ts +++ b/src/lib/jobs/reminder/ops-handlers.ts @@ -8,6 +8,7 @@ import { type Job } from "pg-boss"; import { runGeoBackfill } from "@/lib/jobs/geo-backfill"; import { runTlsPinMonitor } from "@/lib/jobs/tls-pin-monitor"; import { type PrDetectionPayload } from "@/lib/jobs/pr-detection"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { runHostMetricTick } from "@/lib/jobs/host-metric-sampler"; import { aggregateRecommendationFeedback } from "@/lib/jobs/feedback-aggregator"; @@ -34,40 +35,45 @@ export interface TlsPinMonitorPayload { export async function handleHostMetricSample( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.host_metric_sample", async (evt) => { + return withBackgroundEvent("job.host_metric_sample", async (evt) => { const p = getWorkerPrisma(); try { const { pruned } = await runHostMetricTick(p); evt.addMeta("host_metric_pruned", pruned); + return jobDone({ host_metric_pruned: pruned }); } catch (err) { - // The chart degrades gracefully when samples are missing — log - // and move on rather than poisoning the boss queue with retries. + // The chart degrades gracefully when samples are missing, so the + // warning stays — but a tick that took no sample did not do its work. evt.addWarning(`host-metric-sample failed: ${err}`); + return jobFailed("host metric sample failed", err); } }); } export async function handleFeedbackAggregator( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.feedback_aggregator", async (evt) => { + return withBackgroundEvent("job.feedback_aggregator", async (evt) => { const p = getWorkerPrisma(); try { const summary = await aggregateRecommendationFeedback(p); + const totalRows = summary.buckets.reduce((acc, b) => acc + b.total, 0); evt.addMeta("feedback_buckets", summary.buckets.length); - evt.addMeta( - "feedback_total_rows", - summary.buckets.reduce((acc, b) => acc + b.total, 0), - ); + evt.addMeta("feedback_total_rows", totalRows); evt.addMeta("feedback_window_days", summary.windowDays); + return jobDone({ + feedback_buckets: summary.buckets.length, + feedback_total_rows: totalRows, + feedback_window_days: summary.windowDays, + }); } catch (err) { - // The admin dashboard tolerates a stale summary — log and move - // on rather than poisoning the boss queue with retries that - // would block the next cleanup window. + // The admin dashboard tolerates a stale summary, so the warning stays — + // but a pass that aggregated nothing did not do its work. evt.addWarning(`feedback-aggregator failed: ${err}`); + return jobFailed("feedback aggregation failed", err); } }); } @@ -91,18 +97,21 @@ export async function handleFeedbackAggregator( */ let geoBackfillRunning = false; -export async function handleGeoBackfill(jobs: Job[]) { +export async function handleGeoBackfill( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.geo_backfill", async (evt) => { + return withBackgroundEvent("job.geo_backfill", async (evt) => { if (geoBackfillRunning) { // Earlier pass still in flight; skip this tick. Idempotent + the // next tick after the in-flight pass completes will pick up - // anything we miss here. + // anything we miss here. Deliberately ceding to the in-flight pass + // is not a failure, so the tick reports the skip and succeeds. evt.addWarning( "geo-backfill skipped — earlier pass still in flight inside the same worker process", ); evt.addMeta("geo_backfill_skipped", true); - return; + return jobDone({ geo_backfill_skipped: true }); } geoBackfillRunning = true; const p = getWorkerPrisma(); @@ -112,38 +121,60 @@ export async function handleGeoBackfill(jobs: Job[]) { evt.addMeta("geo_backfill_located", summary.located); evt.addMeta("geo_backfill_carrier_resolved", summary.carrierResolved); evt.addMeta("geo_backfill_still_unresolved", summary.stillUnresolved); + return jobDone({ + geo_backfill_scanned: summary.scanned, + geo_backfill_located: summary.located, + geo_backfill_carrier_resolved: summary.carrierResolved, + geo_backfill_still_unresolved: summary.stillUnresolved, + }); } catch (err) { - // The admin sign-in overview tolerates a stale Standort cell — - // log and move on so a one-off resolver hiccup does not poison - // the queue and block the next pass. + // The admin sign-in overview tolerates a stale Standort cell, so the + // warning stays — but a pass that resolved nothing did not do its work. evt.addWarning(`geo-backfill failed: ${err}`); + return jobFailed("geo backfill pass failed", err); } finally { geoBackfillRunning = false; } }); } -export async function handleTlsPinMonitor(jobs: Job[]) { +export async function handleTlsPinMonitor( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.tls_pin_monitor", async (evt) => { + return withBackgroundEvent("job.tls_pin_monitor", async (evt) => { const p = getWorkerPrisma(); try { const summary = await runTlsPinMonitor(p); evt.addMeta("tls_pin_monitor_outcome", summary.outcome); evt.addMeta("tls_pin_monitor_host", summary.host); evt.addMeta("tls_pin_monitor_known_count", summary.knownPinCount); + // No configured target leaves `host` null; the empty string keeps the + // fact shape scalar, and `outcome: "skipped"` already names that case. + return jobDone({ + tls_pin_monitor_outcome: summary.outcome, + tls_pin_monitor_host: summary.host ?? "", + tls_pin_monitor_known_count: summary.knownPinCount, + }); } catch (err) { // runTlsPinMonitor swallows probe failures internally; anything that - // escapes is unexpected. Log and move on so a one-off failure does not - // poison the queue and block the next tick. + // escapes is unexpected, and a tick that never compared a pin is + // exactly the silence this monitor exists to break. evt.addWarning(`tls-pin-monitor failed: ${err}`); + return jobFailed("tls pin probe failed", err); } }); } export async function handlePrDetection( jobs: Job[], -) { +): Promise { + // Each job in the batch opens its own wide event; the outcome is reported + // once for the batch, so the per-job facts accumulate out here. + let usersScanned = 0; + let insertedAcrossJobs = 0; + let tiesAcrossJobs = 0; + let usersFailed = 0; for (const job of jobs) { await withBackgroundEvent("job.pr_detection", async (evt) => { const p = getWorkerPrisma(); @@ -170,6 +201,10 @@ export async function handlePrDetection( insertedTotal += result.inserted; tiesTotal += result.ties; } catch (err) { + // One user's detection failing is that user's problem, not the + // pass's: it rides out as a count so the cron is not retried for + // the whole cohort. + usersFailed++; evt.addWarning(`pr-detection failed for user ${userId}: ${err}`); } } @@ -178,6 +213,18 @@ export async function handlePrDetection( evt.addMeta("pr_detection_ties", tiesTotal); evt.addMeta("pr_detection_silent", silent); evt.addMeta("pr_detection_mode", payloadUserId ? "ingest" : "cron"); + + usersScanned += userIds.length; + insertedAcrossJobs += insertedTotal; + tiesAcrossJobs += tiesTotal; }); } + + return jobDone({ + pr_detection_jobs: jobs.length, + pr_detection_users: usersScanned, + pr_detection_inserted: insertedAcrossJobs, + pr_detection_ties: tiesAcrossJobs, + pr_detection_users_failed: usersFailed, + }); } diff --git a/src/lib/jobs/reminder/poll-cohort.ts b/src/lib/jobs/reminder/poll-cohort.ts index a24a12ff3..154f61af0 100644 --- a/src/lib/jobs/reminder/poll-cohort.ts +++ b/src/lib/jobs/reminder/poll-cohort.ts @@ -21,6 +21,7 @@ import { type Job } from "pg-boss"; import { fireAndForget } from "@/lib/logging/fire-and-forget"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; import { isSyncFailureRecorded } from "@/lib/integrations/status"; @@ -58,6 +59,13 @@ export interface PollCohortConfig { * handler resolves its targets from the batch payloads (single-user enqueue) or * the cohort query (cron tick), then re-syncs each sequentially with per-user * error isolation so one revoked grant / outage never starves the cohort. + * + * Outcome semantics for a fan-out pass: the JOB succeeded if the pass ran. + * One user's revoked grant is that user's problem — it belongs on their + * integration ledger, not in a queue-wide failure that retries the whole + * cohort — so per-user failures ride out as the `users_failed` count. Only a + * failure of the pass itself (cohort discovery, the wide-event scope) fails + * the job, and that path already throws. */ export function makePollCohortHandler({ taskName, @@ -65,8 +73,10 @@ export function makePollCohortHandler({ syncUser, recordFailure, }: PollCohortConfig) { - return async function handlePollCohort(jobs: Job[]) { - await withBackgroundEvent(taskName, async (evt) => { + return async function handlePollCohort( + jobs: Job[], + ): Promise { + return withBackgroundEvent(taskName, async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; @@ -77,9 +87,12 @@ export function makePollCohortHandler({ const ids = await findCohort(prisma); targets.push(...ids.map((userId) => ({ userId }))); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ total: 0, users_synced: 0, users_failed: 0 }); + } let usersSynced = 0; + let usersFailed = 0; let measurementsImported = 0; for (const { userId } of targets) { try { @@ -106,6 +119,7 @@ export function makePollCohortHandler({ ); }); } + usersFailed++; evt.addWarning(`${taskName} failed for user ${userId}: ${err}`); } } @@ -114,10 +128,17 @@ export function makePollCohortHandler({ task_name: taskName, result: { users_synced: usersSynced, + users_failed: usersFailed, total: targets.length, measurements_imported: measurementsImported, }, }); + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/reminder/register-integration-sync.ts b/src/lib/jobs/reminder/register-integration-sync.ts index 85a5e1bf2..54e753948 100644 --- a/src/lib/jobs/reminder/register-integration-sync.ts +++ b/src/lib/jobs/reminder/register-integration-sync.ts @@ -9,8 +9,8 @@ * * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `boss.work(QUEUE, …, handler)` binding drains it. The queue-wiring - * guards (`withings-queues`, `whoop-queues`, `fitbit-queues`, + * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The + * queue-wiring guards (`withings-queues`, `whoop-queues`, `fitbit-queues`, * `nightscout-queues`, `sleep-timeline-queues`) read THIS module. */ import { PgBoss } from "pg-boss"; @@ -78,8 +78,11 @@ import { type IntegrationBackfillAdmissionPayload, } from "@/lib/jobs/integration-backfill-admission"; import { workerLog } from "./shared"; +import { jobDone } from "@/lib/jobs/job-outcome"; import { createAndSchedule, + createAndWork, + cronIsTheRetry, type QueuePolicyTable, type ScheduleEntry, } from "./registrar-shared"; @@ -333,7 +336,16 @@ const schedules: ScheduleEntry[] = [ // v1.32.23 — hourly ECG catch-net (:41) so watch-only accounts pull their // strips without a webhook or a scale. [WITHINGS_ECG_SYNC_QUEUE, WITHINGS_ECG_SYNC_CRON], - [WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, WITHINGS_OAUTH_STATE_CLEANUP_CRON], + // The five OAuth-state / handoff prunes carry `cronIsTheRetry`: their + // handlers used to warn and carry on, they now fail the job, and a DELETE + // that could not run will not run any better a second later. The daily tick + // is the retry. The sync crons above keep the default retries — a provider + // outage is transient and worth retrying inside the hour. + [ + WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, + WITHINGS_OAUTH_STATE_CLEANUP_CRON, + cronIsTheRetry, + ], // v1.11.0 — WHOOP poll-fallback crons. Recovery/sleep/workout catch // dropped webhooks; cycle is the sole driver (no webhook). Staggered off // the Withings crons so the hourly ticks don't pile up on one boss poll. @@ -342,11 +354,19 @@ const schedules: ScheduleEntry[] = [ [WHOOP_WORKOUT_SYNC_QUEUE, WHOOP_WORKOUT_SYNC_CRON], [WHOOP_CYCLE_SYNC_QUEUE, WHOOP_CYCLE_SYNC_CRON], // v1.11.0 — daily 03:22 Europe/Berlin prune for expired WHOOP OAuth states. - [WHOOP_OAUTH_STATE_CLEANUP_QUEUE, WHOOP_OAUTH_STATE_CLEANUP_CRON], + [ + WHOOP_OAUTH_STATE_CLEANUP_QUEUE, + WHOOP_OAUTH_STATE_CLEANUP_CRON, + cronIsTheRetry, + ], // v1.12.0 — hourly Fitbit poll (:08, staggered off WHOOP/Withings) + the // daily 03:24 Europe/Berlin prune for expired Fitbit OAuth states. [FITBIT_SYNC_QUEUE, FITBIT_SYNC_CRON], - [FITBIT_OAUTH_STATE_CLEANUP_QUEUE, FITBIT_OAUTH_STATE_CLEANUP_CRON], + [ + FITBIT_OAUTH_STATE_CLEANUP_QUEUE, + FITBIT_OAUTH_STATE_CLEANUP_CRON, + cronIsTheRetry, + ], // v1.26.0 — hourly Google Health poll (:18, staggered off WHOOP/Fitbit/ // Nightscout/Polar/Oura) + the daily 03:26 Europe/Berlin prune for expired // Google Health OAuth states. @@ -354,6 +374,7 @@ const schedules: ScheduleEntry[] = [ [ GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON, + cronIsTheRetry, ], // v1.17.0 — hourly Nightscout CGM poll (:11, staggered off the other sync // ticks). @@ -364,7 +385,11 @@ const schedules: ScheduleEntry[] = [ // v1.28.x — hourly Strava (:17) OAuth poll. [STRAVA_SYNC_QUEUE, STRAVA_SYNC_CRON], // v1.30.x — daily 03:28 Europe/Berlin prune for expired native OIDC handoffs. - [OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, OIDC_NATIVE_HANDOFF_CLEANUP_CRON], + [ + OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, + OIDC_NATIVE_HANDOFF_CLEANUP_CRON, + cronIsTheRetry, + ], ]; /** @@ -450,19 +475,30 @@ export async function registerIntegrationSyncQueues( // reaches this shared durable queue before doing full-history work. The // common group is enforced in PostgreSQL across worker processes/nodes. - await boss.work( + await createAndWork( + boss, INTEGRATION_BACKFILL_ADMISSION_QUEUE, { localConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, groupConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, }, async (jobs) => { + // The drain has no per-job error isolation: a runner that throws fails + // the job and the admission retry policy applies. So reaching the end + // means every admitted job did its work, and the counts each runner + // already returns are what the pass reports. + let importedTotal = 0; + let removedTotal = 0; + let deletedTotal = 0; + let markersTotal = 0; + let linkedTotal = 0; for (const job of jobs) { const { kind, data } = job.data; const { userId } = data; switch (kind) { case "whoop-backfill": { const { imported } = await runWhoopBackfillForUser(userId); + importedTotal += imported; workerLog( "info", `[whoop-backfill] user=${userId} imported=${imported}`, @@ -471,6 +507,7 @@ export async function registerIntegrationSyncQueues( } case "fitbit-backfill": { const { imported } = await runFitbitBackfillForUser(userId); + importedTotal += imported; workerLog( "info", `[fitbit-backfill] user=${userId} imported=${imported}`, @@ -479,6 +516,7 @@ export async function registerIntegrationSyncQueues( } case "google-health-backfill": { const { imported } = await runGoogleHealthBackfillForUser(userId); + importedTotal += imported; workerLog( "info", `[google-health-backfill] user=${userId} imported=${imported}`, @@ -488,6 +526,7 @@ export async function registerIntegrationSyncQueues( case "google-health-sleep-repair": { const { imported } = await runGoogleHealthSleepRepairForUser(userId); + importedTotal += imported; workerLog( "info", `[google-health-sleep-repair] user=${userId} imported=${imported}`, @@ -497,6 +536,8 @@ export async function registerIntegrationSyncQueues( case "fitbit-sleep-repair": { const { imported, removed } = await runFitbitSleepRepairForUser(userId); + importedTotal += imported; + removedTotal += removed; workerLog( "info", `[fitbit-sleep-repair] user=${userId} imported=${imported} removed=${removed}`, @@ -513,6 +554,8 @@ export async function registerIntegrationSyncQueues( userId, data.provider, ); + deletedTotal += deleted; + importedTotal += imported; workerLog( "info", `[sleep-timeline-backfill] user=${userId} provider=${data.provider} deleted=${deleted} imported=${imported}`, @@ -522,6 +565,8 @@ export async function registerIntegrationSyncQueues( case "lab-biomarker-backfill": { const { markers, linked } = await runLabBiomarkerBackfillForUser(userId); + markersTotal += markers; + linkedTotal += linked; workerLog( "info", `[lab-biomarker-backfill] user=${userId} markers=${markers} linked=${linked}`, @@ -530,6 +575,7 @@ export async function registerIntegrationSyncQueues( } case "strava-backfill": { const { imported } = await runStravaBackfillForUser(userId); + importedTotal += imported; workerLog( "info", `[strava-backfill] user=${userId} imported=${imported}`, @@ -538,30 +584,43 @@ export async function registerIntegrationSyncQueues( } } } + return jobDone({ + jobs: jobs.length, + imported: importedTotal, + removed: removedTotal, + deleted: deletedTotal, + markers: markersTotal, + linked: linkedTotal, + }); }, ); - await boss.work( + await createAndWork( + boss, WITHINGS_SYNC_QUEUE, { localConcurrency: 1 }, handleWithingsFallbackSync, ); - await boss.work( + await createAndWork( + boss, WITHINGS_ACTIVITY_QUEUE, { localConcurrency: 1 }, handleWithingsActivitySync, ); - await boss.work( + await createAndWork( + boss, WITHINGS_SLEEP_QUEUE, { localConcurrency: 1 }, handleWithingsSleepSync, ); - await boss.work( + await createAndWork( + boss, WITHINGS_ECG_SYNC_QUEUE, { localConcurrency: 1 }, handleWithingsEcgSync, ); - await boss.work( + await createAndWork( + boss, WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleWithingsOAuthStateCleanup, @@ -569,29 +628,34 @@ export async function registerIntegrationSyncQueues( // v1.11.0 — WHOOP per-resource sync handlers. Webhook-driven per-user + // cron full-iteration. Serial concurrency so a backfill-heavy tick never // crowds the request pool and stays inside WHOOP's 100 req/min app cap. - await boss.work( + await createAndWork( + boss, WHOOP_RECOVERY_SYNC_QUEUE, { localConcurrency: 1 }, handleWhoopRecoverySync, ); - await boss.work( + await createAndWork( + boss, WHOOP_SLEEP_SYNC_QUEUE, { localConcurrency: 1 }, handleWhoopSleepSync, ); - await boss.work( + await createAndWork( + boss, WHOOP_WORKOUT_SYNC_QUEUE, { localConcurrency: 1 }, handleWhoopWorkoutSync, ); - await boss.work( + await createAndWork( + boss, WHOOP_CYCLE_SYNC_QUEUE, { localConcurrency: 1 }, handleWhoopCycleSync, ); // v1.11.0 — self-converging WHOOP backfill. This source queue retains the // per-connection singleton and forwards admitted work to the shared drain. - await boss.work( + await createAndWork( + boss, WHOOP_BACKFILL_QUEUE, { localConcurrency: WHOOP_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -601,29 +665,34 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); - await boss.work( + await createAndWork( + boss, WHOOP_OAUTH_STATE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleWhoopOAuthStateCleanup, ); // v1.30.x — daily sweep for the native OIDC handoff ledger. - await boss.work( + await createAndWork( + boss, OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, { localConcurrency: 1 }, handleOidcNativeHandoffCleanup, ); // v1.12.0 — Fitbit poll-sync (cron full-iteration; no webhook). Serial // concurrency so a backfill-heavy tick never crowds the request pool. - await boss.work( + await createAndWork( + boss, FITBIT_SYNC_QUEUE, { localConcurrency: 1 }, handleFitbitSync, ); // v1.12.0 — self-converging Fitbit backfill. This source queue retains the // per-connection singleton and forwards admitted work to the shared drain. - await boss.work( + await createAndWork( + boss, FITBIT_BACKFILL_QUEUE, { localConcurrency: FITBIT_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -633,12 +702,14 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); // One-shot Fitbit sleep duplicate repair. This source queue retains the // per-connection singleton and forwards the watermark-safe bounded sleep // re-read to the shared drain. - await boss.work( + await createAndWork( + boss, FITBIT_SLEEP_REPAIR_QUEUE, { localConcurrency: FITBIT_SLEEP_REPAIR_CONCURRENCY }, async (jobs) => { @@ -648,16 +719,19 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); - await boss.work( + await createAndWork( + boss, FITBIT_OAUTH_STATE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleFitbitOAuthStateCleanup, ); // v1.26.0 — Google Health poll-sync (cron full-iteration; no webhook). Serial // concurrency so a backfill-heavy tick never crowds the request pool. - await boss.work( + await createAndWork( + boss, GOOGLE_HEALTH_SYNC_QUEUE, { localConcurrency: 1 }, handleGoogleHealthSync, @@ -665,7 +739,8 @@ export async function registerIntegrationSyncQueues( // v1.26.0 — self-converging Google Health backfill. This source queue // retains the per-connection singleton and forwards admitted work to the // shared drain. - await boss.work( + await createAndWork( + boss, GOOGLE_HEALTH_BACKFILL_QUEUE, { localConcurrency: GOOGLE_HEALTH_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -675,12 +750,14 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); // v1.28.x — one-shot Google Health sleep duplicate repair. This source // queue retains the per-connection singleton and forwards the watermark-safe // full sleep-history re-read to the shared drain. - await boss.work( + await createAndWork( + boss, GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, { localConcurrency: GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY }, async (jobs) => { @@ -690,9 +767,11 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); - await boss.work( + await createAndWork( + boss, GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleGoogleHealthOAuthStateCleanup, @@ -700,7 +779,8 @@ export async function registerIntegrationSyncQueues( // v1.17.1 — one-shot sleep-timeline backfill. This source queue retains the // per-(user, provider) singleton and forwards admitted work to the shared // drain. - await boss.work( + await createAndWork( + boss, SLEEP_TIMELINE_BACKFILL_QUEUE, { localConcurrency: SLEEP_TIMELINE_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -710,11 +790,13 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); // v1.18.1 — one-shot lab-biomarker backfill. This source queue retains the // per-user singleton and forwards admitted work to the shared drain. - await boss.work( + await createAndWork( + boss, LAB_BIOMARKER_BACKFILL_QUEUE, { localConcurrency: LAB_BIOMARKER_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -724,12 +806,14 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); // v1.17.0 — Nightscout CGM poll-cohort sync. The hourly cron tick (no // `userId`) walks every configured instance; one user's unreachable host is // warned, not fatal. - await boss.work( + await createAndWork( + boss, NIGHTSCOUT_SYNC_QUEUE, { localConcurrency: 1 }, handleNightscoutSync, @@ -737,12 +821,14 @@ export async function registerIntegrationSyncQueues( // v1.17.0 (F4) — Polar + Oura OAuth poll-cohort sync. The hourly cron tick // (no `userId`) re-walks every connected user; one user's revoked grant is // warned, not fatal. - await boss.work( + await createAndWork( + boss, POLAR_SYNC_QUEUE, { localConcurrency: 1 }, handlePolarSync, ); - await boss.work( + await createAndWork( + boss, OURA_SYNC_QUEUE, { localConcurrency: 1 }, handleOuraSync, @@ -750,14 +836,16 @@ export async function registerIntegrationSyncQueues( // v1.28.x — Strava OAuth poll-cohort sync (poll-only; webhook deferred). The // hourly cron tick (no `userId`) re-walks every connected user; one user's // revoked grant is warned, not fatal. - await boss.work( + await createAndWork( + boss, STRAVA_SYNC_QUEUE, { localConcurrency: 1 }, handleStravaSync, ); // v1.28.x — self-converging Strava backfill. This source queue retains the // per-connection singleton and forwards admitted work to the shared drain. - await boss.work( + await createAndWork( + boss, STRAVA_BACKFILL_QUEUE, { localConcurrency: STRAVA_BACKFILL_CONCURRENCY }, async (jobs) => { @@ -767,6 +855,7 @@ export async function registerIntegrationSyncQueues( data: job.data, }); } + return jobDone({ jobs: jobs.length }); }, ); return allQueues; diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts index 921f1595e..1f4f327ad 100644 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ b/src/lib/jobs/reminder/register-maintenance.ts @@ -10,7 +10,7 @@ * * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `boss.work(QUEUE, …, handler)` binding drains it. The + * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The * `measurement-tombstone-cleanup`, `withings-oauth-state-cleanup`, * `intake-slot-dedup-queue`, `tls-pin-monitor-queue`, and `geo-backfill` * guards read THIS module. @@ -131,9 +131,12 @@ import { type EnvironmentFetchPayload, } from "@/lib/jobs/environment-fetch"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { workerLog, BOOT_BACKFILL_STAGGER_SECONDS } from "./shared"; import { createAndSchedule, + createAndWork, + cronIsTheRetry, type QueuePolicyTable, type ScheduleEntry, } from "./registrar-shared"; @@ -411,24 +414,35 @@ const allQueues = [ DOCUMENT_SUMMARY_CATCHUP_QUEUE, ]; +// `cronIsTheRetry` (retryLimit 0) marks the passes whose handler now returns +// `jobFailed` where it used to warn and carry on, AND whose failure repeats: +// a bulk DELETE or UPDATE that timed out will time out again on an immediate +// retry, so pg-boss's default two would run the same doomed statement three +// times a night. The queues without it are the ones whose failure is +// transient — the TLS probe, the off-host upload, the restore drill, the +// per-user environment fetch — and those still want the default retries. const schedules: ScheduleEntry[] = [ // v1.25 (W-ENV) — daily 02:10 Europe/Berlin discovery tick (empty payload) // that fans out one per-user environment fetch per opted-in account. [ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON], [DATA_BACKUP_QUEUE, DATA_BACKUP_CRON], - [RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON], - [IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON], + [RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, cronIsTheRetry], + [IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON, cronIsTheRetry], [AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON], - [STEP_UP_ELEVATION_CLEANUP_QUEUE, STEP_UP_ELEVATION_CLEANUP_CRON], + [ + STEP_UP_ELEVATION_CLEANUP_QUEUE, + STEP_UP_ELEVATION_CLEANUP_CRON, + cronIsTheRetry, + ], [OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON], [RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON], - [HOST_METRIC_QUEUE, HOST_METRIC_CRON], - [FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON], + [HOST_METRIC_QUEUE, HOST_METRIC_CRON, cronIsTheRetry], + [FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON, cronIsTheRetry], // v1.4.37 — hourly geo backfill. The helper is idempotent + capped // at 5 000 rows per pass; running it at :40 every hour catches the // long tail of audit rows that landed with the offline MMDB // missing or the online provider unreachable. - [GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON], + [GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON, cronIsTheRetry], // v1.12.2 ios-coord — every-6-hour TLS leaf SPKI probe (:07 off the // hourly sync crons). Surfaces a pinned-leaf rotation well inside the // ≥11-day re-pin window the iOS release owner needs. @@ -443,14 +457,18 @@ const schedules: ScheduleEntry[] = [ // entities. Flips IN_USE rows whose 30-day clock has blown to // EXPIRED at 03:30 Europe/Berlin (in the existing 02:xx–03:xx // maintenance window, right after idempotency-cleanup). - [MEDICATION_INVENTORY_EXPIRE_QUEUE, MEDICATION_INVENTORY_EXPIRE_CRON], + [ + MEDICATION_INVENTORY_EXPIRE_QUEUE, + MEDICATION_INVENTORY_EXPIRE_CRON, + cronIsTheRetry, + ], // v1.4.46 — hourly auto-skip for medication intakes the user never // marked. Cron `5 * * * *` slots off the top-of-the-hour // reminder-check (:00) so the hourly ticks don't pile up on the same // boss poll. - [INTAKE_AUTO_SKIP_QUEUE, INTAKE_AUTO_SKIP_CRON], + [INTAKE_AUTO_SKIP_QUEUE, INTAKE_AUTO_SKIP_CRON, cronIsTheRetry], // v0.5.4 ios-coord — daily mood-reminder dispatch-ledger retention sweep. - [MOOD_REMINDER_CLEANUP_QUEUE, MOOD_REMINDER_CLEANUP_CRON], + [MOOD_REMINDER_CLEANUP_QUEUE, MOOD_REMINDER_CLEANUP_CRON, cronIsTheRetry], // v1.15.19 — daily 03:28 Europe/Berlin duplicate dose-slot dedup // discovery. The empty cron payload (no `userId`) is the handler's // signal to run the discovery fan-out instead of a per-user pass, so @@ -458,20 +476,24 @@ const schedules: ScheduleEntry[] = [ // a day instead of waiting for the next worker reboot. [INTAKE_SLOT_DEDUP_QUEUE, INTAKE_SLOT_DEDUP_CRON], // v1.4.49 — daily 03:35 Europe/Berlin prune for push_attempts. - [PUSH_ATTEMPT_CLEANUP_QUEUE, PUSH_ATTEMPT_CLEANUP_CRON], + [PUSH_ATTEMPT_CLEANUP_QUEUE, PUSH_ATTEMPT_CLEANUP_CRON, cronIsTheRetry], // v1.31.0 — daily 04:10 Europe/Berlin prune for arrival_reactions. - [ARRIVAL_REACTION_CLEANUP_QUEUE, ARRIVAL_REACTION_CLEANUP_CRON], + [ + ARRIVAL_REACTION_CLEANUP_QUEUE, + ARRIVAL_REACTION_CLEANUP_CRON, + cronIsTheRetry, + ], // v1.7.0 — daily 03:40 Europe/Berlin prune for expired measurement // tombstones. [MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, MEASUREMENT_TOMBSTONE_CLEANUP_CRON], // v1.18.7 — daily 03:50 Europe/Berlin prune for stale Coach history. - [COACH_MESSAGE_CLEANUP_QUEUE, COACH_MESSAGE_CLEANUP_CRON], + [COACH_MESSAGE_CLEANUP_QUEUE, COACH_MESSAGE_CLEANUP_CRON, cronIsTheRetry], // v1.23 — daily 03:55 Europe/Berlin note-encryption backfill discovery. // The empty cron payload (no `userId`) is the handler's signal to fan out // one per-user job per account still holding a plaintext note. [NOTE_ENCRYPTION_BACKFILL_QUEUE, NOTE_ENCRYPTION_BACKFILL_CRON], // v1.24 — daily 04:00 Europe/Berlin prune for dead MCP connector tokens. - [MCP_TOKEN_CLEANUP_QUEUE, MCP_TOKEN_CLEANUP_CRON], + [MCP_TOKEN_CLEANUP_QUEUE, MCP_TOKEN_CLEANUP_CRON, cronIsTheRetry], // v1.25 — daily 04:05 Europe/Berlin medication-note encryption backfill // discovery. The empty cron payload (no `userId`) is the handler's signal to // fan out one per-user job per account still holding a plaintext medication @@ -479,14 +501,14 @@ const schedules: ScheduleEntry[] = [ [MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, MED_NOTES_ENCRYPTION_BACKFILL_CRON], // Document vault — daily 04:10 Europe/Berlin purge for tombstoned // documents past the 30-day undo grace. - [DOCUMENT_PURGE_QUEUE, DOCUMENT_PURGE_CRON], + [DOCUMENT_PURGE_QUEUE, DOCUMENT_PURGE_CRON, cronIsTheRetry], // v1.32.1 (issue #588) — every-15-minute orphan-ImportJob sweep. Re-runs // the same reconcile the boot path uses, so a stuck "unpacking" row // whose worker crashed/restarted without the boot-time pass catching it // (heartbeat not yet stale, pg-boss job not yet expired) still converges // to a visible `failed` state within two ticks instead of staying stuck // forever on an otherwise-healthy worker. - [IMPORT_JOB_RECONCILE_QUEUE, IMPORT_JOB_RECONCILE_CRON], + [IMPORT_JOB_RECONCILE_QUEUE, IMPORT_JOB_RECONCILE_CRON, cronIsTheRetry], ]; /** @@ -584,61 +606,76 @@ export async function registerMaintenanceQueues( ): Promise { await createAndSchedule(boss, allQueues, schedules, queuePolicies); - await boss.work( + await createAndWork( + boss, DATA_BACKUP_QUEUE, { localConcurrency: 1 }, handleDataBackup, ); - await boss.work( + await createAndWork( + boss, RATE_LIMIT_CLEANUP_QUEUE, { localConcurrency: 1 }, handleRateLimitCleanup, ); - await boss.work( + await createAndWork( + boss, IDEMPOTENCY_CLEANUP_QUEUE, { localConcurrency: 1 }, handleIdempotencyCleanup, ); - await boss.work( + await createAndWork( + boss, AUDIT_LOG_CLEANUP_QUEUE, { localConcurrency: 1 }, handleAuditLogCleanup, ); - await boss.work( + await createAndWork( + boss, STEP_UP_ELEVATION_CLEANUP_QUEUE, { localConcurrency: 1 }, handleStepUpElevationCleanup, ); - await boss.work( + await createAndWork( + boss, OFFHOST_BACKUP_QUEUE, { localConcurrency: 1 }, handleOffhostBackup, ); - // prettier-ignore - await boss.work(RESTORE_DRILL_QUEUE, { localConcurrency: 1 }, handleRestoreDrill); - await boss.work( + await createAndWork( + boss, + RESTORE_DRILL_QUEUE, + { localConcurrency: 1 }, + handleRestoreDrill, + ); + await createAndWork( + boss, HOST_METRIC_QUEUE, { localConcurrency: 1 }, handleHostMetricSample, ); - await boss.work( + await createAndWork( + boss, FEEDBACK_AGGREGATOR_QUEUE, { localConcurrency: 1 }, handleFeedbackAggregator, ); - await boss.work( + await createAndWork( + boss, GEO_BACKFILL_QUEUE, { localConcurrency: 1 }, handleGeoBackfill, ); // v1.12.2 ios-coord — TLS leaf SPKI-change monitor. Single-flight: one // short outbound TLS handshake per tick, no benefit to overlapping ticks. - await boss.work( + await createAndWork( + boss, TLS_PIN_MONITOR_QUEUE, { localConcurrency: 1 }, handleTlsPinMonitor, ); - await boss.work( + await createAndWork( + boss, MOOD_REMINDER_CLEANUP_QUEUE, { localConcurrency: 1 }, handleMoodReminderCleanup, @@ -647,7 +684,8 @@ export async function registerMaintenanceQueues( // matches every other cleanup queue; two ticks racing on the same // DELETE statement is wasted work and the second tick's payload // would be a no-op anyway. - await boss.work( + await createAndWork( + boss, PUSH_ATTEMPT_CLEANUP_QUEUE, { localConcurrency: 1 }, handlePushAttemptCleanup, @@ -655,14 +693,16 @@ export async function registerMaintenanceQueues( // v1.31.0 — daily prune of the data-arrival reaction markers. Single-flight // like every other cleanup queue: two ticks racing the same DELETE is // wasted work. - await boss.work( + await createAndWork( + boss, ARRIVAL_REACTION_CLEANUP_QUEUE, { localConcurrency: 1 }, handleArrivalReactionCleanup, ); // v1.7.0 — daily prune of expired measurement tombstones. Single-flight // like every other cleanup queue. - await boss.work( + await createAndWork( + boss, MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleMeasurementTombstoneCleanup, @@ -670,14 +710,16 @@ export async function registerMaintenanceQueues( // v1.18.7 — daily prune of the encrypted Coach conversation history. // Single-flight like every other cleanup queue; two ticks racing on the // same DELETE is wasted work and the second is a no-op. - await boss.work( + await createAndWork( + boss, COACH_MESSAGE_CLEANUP_QUEUE, { localConcurrency: 1 }, handleCoachMessageCleanup, ); // MCP Phase 3 (M2) — daily prune of dead MCP connector access tokens + // long-revoked connection anchors. Single-flight like every other cleanup. - await boss.work( + await createAndWork( + boss, MCP_TOKEN_CLEANUP_QUEUE, { localConcurrency: 1 }, handleMcpTokenCleanup, @@ -685,17 +727,20 @@ export async function registerMaintenanceQueues( // Document vault — daily purge of tombstoned documents past the 30-day // undo grace. Single-flight like every other cleanup queue; the underlying // deleteMany is idempotent so a duplicate tick is a no-op. - await boss.work( + await createAndWork( + boss, DOCUMENT_PURGE_QUEUE, { localConcurrency: 1 }, handleDocumentPurge, ); - await boss.work( + await createAndWork( + boss, PR_DETECTION_QUEUE, { localConcurrency: PR_DETECTION_CONCURRENCY }, handlePrDetection, ); - await boss.work( + await createAndWork( + boss, MEDICATION_INVENTORY_EXPIRE_QUEUE, { localConcurrency: 1 }, handleMedicationInventoryExpire, @@ -704,7 +749,8 @@ export async function registerMaintenanceQueues( // Single-flight: two ticks racing against the same row pile is wasted // work, and the underlying `updateMany` is the canonical idempotent // shape anyway. - await boss.work( + await createAndWork( + boss, INTAKE_AUTO_SKIP_QUEUE, { localConcurrency: 1 }, handleIntakeAutoSkip, @@ -712,48 +758,64 @@ export async function registerMaintenanceQueues( // v1.4.34 — Apple Health export.zip ingest worker. localConcurrency // caps at 1 because the parse loop is CPU-bound and a concurrent // second 1 GB import would race the first for RSS. - await boss.work( + await createAndWork( + boss, APPLE_HEALTH_IMPORT_V2_QUEUE, { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, async (jobs) => { // pg-boss v12 work callbacks always receive an array (batched // worker mode); for our concurrency-1 + batchSize-1 case we just // process each job sequentially. + // + // Each import is independent work, so a failed one does not abort the + // rest of the batch; the first failure is what the job reports. + let failure: JobOutcome | null = null; for (const job of jobs) { - await handleAppleHealthImport(job); + const outcome = await handleAppleHealthImport(job); + if (!outcome.ok && failure === null) failure = outcome; } + return failure ?? jobDone({ jobs: jobs.length }); }, ); // Drain pre-revision-2 backlog by moving each claimed legacy job onto the // isolated v2 queue. This preserves the ImportJob status id without letting // the current parser execute under a revision-1 mirror. - await boss.work( + await createAndWork( + boss, APPLE_HEALTH_IMPORT_LEGACY_QUEUE, { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, async (jobs) => { for (const job of jobs) { await migrateLegacyAppleHealthImport(job); } + return jobDone({ jobs: jobs.length }); }, ); // v1.32.1 (issue #588) — periodic orphan-ImportJob sweep. Single-flight: // the underlying `updateMany` is idempotent and two ticks racing the same // read-then-write pass would just be wasted work. - await boss.work( + await createAndWork( + boss, IMPORT_JOB_RECONCILE_QUEUE, { localConcurrency: 1 }, handleImportJobReconcileTick, ); - await boss.work( + await createAndWork( + boss, MEDICATION_INTAKE_IMPORT_QUEUE, { localConcurrency: MEDICATION_INTAKE_IMPORT_CONCURRENCY, includeMetadata: true, }, async (jobs) => { + // Independent per-delivery imports: run them all, report the first + // failure rather than losing it behind a later success. + let failure: JobOutcome | null = null; for (const job of jobs) { - await handleMedicationIntakeImport(job); + const outcome = await handleMedicationIntakeImport(job); + if (!outcome.ok && failure === null) failure = outcome; } + return failure ?? jobDone({ jobs: jobs.length }); }, ); @@ -768,10 +830,21 @@ export async function registerMaintenanceQueues( // above) whose payload omits `userId`. That tick runs the SAME discovery // fan-out the boot pass uses, so duplicate slots created between deploys // collapse within a day instead of waiting for the next worker reboot. - await boss.work>( + await createAndWork>( + boss, INTAKE_SLOT_DEDUP_QUEUE, { localConcurrency: INTAKE_SLOT_DEDUP_CONCURRENCY }, async (jobs) => { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + // The discovery helper returns its error instead of throwing, so a + // failed fan-out would otherwise leave no trace on the job at all. + let discoveryFailed = 0; + let slotsCollapsed = 0; + let rowsSoftDeleted = 0; + let rowsNormalised = 0; + let daysRecomputed = 0; for (const job of jobs) { const { userId } = job.data; if (!userId) { @@ -779,6 +852,9 @@ export async function registerMaintenanceQueues( // still holding duplicate-slot candidates (singletonKey-coalesced, // identical to the boot pass). const result = await enqueueBootTimeIntakeSlotDedup(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; workerLog( "info", `[intake-slot-dedup] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, @@ -787,6 +863,11 @@ export async function registerMaintenanceQueues( } try { const summary = await dedupeUserIntakeSlots(userId); + users++; + slotsCollapsed += summary.slotsCollapsed; + rowsSoftDeleted += summary.rowsSoftDeleted; + rowsNormalised += summary.rowsNormalised; + daysRecomputed += summary.daysRecomputed; workerLog( "info", `[intake-slot-dedup] user=${userId} slotsCollapsed=${summary.slotsCollapsed} rowsSoftDeleted=${summary.rowsSoftDeleted} rowsNormalised=${summary.rowsNormalised} daysRecomputed=${summary.daysRecomputed}`, @@ -797,6 +878,17 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + slots_collapsed: slotsCollapsed, + rows_soft_deleted: rowsSoftDeleted, + rows_normalised: rowsNormalised, + days_recomputed: daysRecomputed, + }); }, ); @@ -806,14 +898,26 @@ export async function registerMaintenanceQueues( // empty-userId payload is the daily discovery tick and re-fans the per-user // jobs (singletonKey-coalesced, identical to the boot pass). Serial // concurrency so the migration never crowds the request pool. - await boss.work>( + await createAndWork>( + boss, NOTE_ENCRYPTION_BACKFILL_QUEUE, { localConcurrency: NOTE_ENCRYPTION_BACKFILL_CONCURRENCY }, async (jobs) => { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + // The discovery helper returns its error instead of throwing, so a + // failed fan-out would otherwise leave no trace on the job at all. + let discoveryFailed = 0; + let measurements = 0; + let moodEntries = 0; for (const job of jobs) { const { userId } = job.data; if (!userId) { const result = await enqueueBootTimeNoteEncryptionBackfill(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; workerLog( "info", `[note-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, @@ -823,6 +927,9 @@ export async function registerMaintenanceQueues( try { const { measurementsMigrated, moodEntriesMigrated } = await runNoteEncryptionBackfillForUser(userId); + users++; + measurements += measurementsMigrated; + moodEntries += moodEntriesMigrated; workerLog( "info", `[note-encryption-backfill] user=${userId} measurements=${measurementsMigrated} moodEntries=${moodEntriesMigrated}`, @@ -837,11 +944,21 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + measurements_migrated: measurements, + mood_entries_migrated: moodEntries, + }); }, ); // v1.23 — admin-triggered encryption-key rotation (on-demand, singleton). - await boss.work( + await createAndWork( + boss, ENCRYPTION_KEY_ROTATE_QUEUE, { localConcurrency: ENCRYPTION_KEY_ROTATE_CONCURRENCY }, handleEncryptionKeyRotate, @@ -854,14 +971,27 @@ export async function registerMaintenanceQueues( // payload is the daily discovery tick and re-fans the per-user jobs // (singletonKey-coalesced, identical to the boot pass). Serial concurrency // so the migration never crowds the request pool. - await boss.work>( + await createAndWork>( + boss, MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, { localConcurrency: MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY }, async (jobs) => { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + // The discovery helper returns its error instead of throwing, so a + // failed fan-out would otherwise leave no trace on the job at all. + let discoveryFailed = 0; + let sideEffects = 0; + let doseChanges = 0; + let inventoryItems = 0; for (const job of jobs) { const { userId } = job.data; if (!userId) { const result = await enqueueBootTimeMedNotesEncryptionBackfill(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; workerLog( "info", `[med-notes-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, @@ -874,6 +1004,10 @@ export async function registerMaintenanceQueues( doseChangesMigrated, inventoryItemsMigrated, } = await runMedNotesEncryptionBackfillForUser(userId); + users++; + sideEffects += sideEffectsMigrated; + doseChanges += doseChangesMigrated; + inventoryItems += inventoryItemsMigrated; workerLog( "info", `[med-notes-encryption-backfill] user=${userId} sideEffects=${sideEffectsMigrated} doseChanges=${doseChangesMigrated} inventoryItems=${inventoryItemsMigrated}`, @@ -888,6 +1022,16 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + side_effects_migrated: sideEffects, + dose_changes_migrated: doseChanges, + inventory_items_migrated: inventoryItems, + }); }, ); @@ -895,16 +1039,25 @@ export async function registerMaintenanceQueues( // trigger endpoint sends one per-user job; this handler indexes that user's // not-yet-indexed documents (consent + budget gated, bounded + resumable). // Serial concurrency so the provider calls never crowd the request pool. - await boss.work( + await createAndWork( + boss, CONTENT_INDEX_BACKFILL_QUEUE, { localConcurrency: CONTENT_INDEX_BACKFILL_CONCURRENCY }, async (jobs) => { + let users = 0; + let skipped = 0; + let documentsIndexed = 0; for (const job of jobs) { const { userId } = job.data; - if (!userId) continue; + if (!userId) { + skipped++; + continue; + } try { const { indexed, reason } = await runContentIndexBackfillForUser(userId); + users++; + documentsIndexed += indexed; workerLog( "info", `[document-content-index-backfill] user=${userId} indexed=${indexed} reason=${reason}`, @@ -919,6 +1072,12 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + skipped, + documents_indexed: documentsIndexed, + }); }, ); @@ -926,15 +1085,22 @@ export async function registerMaintenanceQueues( // (one job per stored document); provider-first (vision) when configured + // consented, else local text-layer extraction. Serial concurrency so the // provider calls + PDF parse never crowd the request pool. - await boss.work( + await createAndWork( + boss, DOCUMENT_INDEX_QUEUE, { localConcurrency: DOCUMENT_INDEX_CONCURRENCY }, async (jobs) => { + let documents = 0; + let skipped = 0; for (const job of jobs) { const { userId, documentId } = job.data; - if (!userId || !documentId) continue; + if (!userId || !documentId) { + skipped++; + continue; + } try { await runDocumentIndex(job.data); + documents++; } catch (err) { recordError(); workerLog( @@ -945,6 +1111,7 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ jobs: jobs.length, documents, skipped }); }, ); @@ -952,15 +1119,22 @@ export async function registerMaintenanceQueues( // upload (one job per stored document); decodes + downscales the original to // a small encrypted JPEG. Pure local compute; serial concurrency so the // canvas/pdfjs decode never crowds the request pool. - await boss.work( + await createAndWork( + boss, DOCUMENT_THUMBNAIL_QUEUE, { localConcurrency: DOCUMENT_THUMBNAIL_CONCURRENCY }, async (jobs) => { + let documents = 0; + let skipped = 0; for (const job of jobs) { const { userId, documentId } = job.data; - if (!userId || !documentId) continue; + if (!userId || !documentId) { + skipped++; + continue; + } try { await runDocumentThumbnail(job.data); + documents++; } catch (err) { recordError(); workerLog( @@ -971,6 +1145,7 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ jobs: jobs.length, documents, skipped }); }, ); @@ -979,15 +1154,22 @@ export async function registerMaintenanceQueues( // the `documentsAutoAiRead` opt-in is ON (egress consent + budget gated) and // persists the summary encrypted. Serial concurrency so the provider call // never crowds the request pool. - await boss.work( + await createAndWork( + boss, DOCUMENT_SUMMARY_QUEUE, { localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY }, async (jobs) => { + let documents = 0; + let skipped = 0; for (const job of jobs) { const { userId, documentId } = job.data; - if (!userId || !documentId) continue; + if (!userId || !documentId) { + skipped++; + continue; + } try { await runDocumentSummaryJob(job.data); + documents++; } catch (err) { recordError(); workerLog( @@ -998,6 +1180,7 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ jobs: jobs.length, documents, skipped }); }, ); @@ -1007,15 +1190,24 @@ export async function registerMaintenanceQueues( // documents (the upload-time enqueue no-opped while the flag was OFF). It // only enqueues — the opt-in, egress consent and budget gates all still run // per document inside the summary job. Serial concurrency. - await boss.work( + await createAndWork( + boss, DOCUMENT_SUMMARY_CATCHUP_QUEUE, { localConcurrency: DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY }, async (jobs) => { + let users = 0; + let skipped = 0; + let summariesEnqueued = 0; for (const job of jobs) { const { userId } = job.data; - if (!userId) continue; + if (!userId) { + skipped++; + continue; + } try { - await runSummaryCatchUpForUser(userId); + const { enqueued } = await runSummaryCatchUpForUser(userId); + users++; + summariesEnqueued += enqueued; } catch (err) { recordError(); workerLog( @@ -1026,6 +1218,12 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + skipped, + summaries_enqueued: summariesEnqueued, + }); }, ); @@ -1033,15 +1231,24 @@ export async function registerMaintenanceQueues( // discovery sends one per-user job; this handler fans out per-document // thumbnail jobs for that user's not-yet-thumbnailed documents. Serial // concurrency; cheap discovery + enqueue only. - await boss.work( + await createAndWork( + boss, DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, { localConcurrency: DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY }, async (jobs) => { + let users = 0; + let skipped = 0; + let thumbnailsEnqueued = 0; for (const job of jobs) { const { userId } = job.data; - if (!userId) continue; + if (!userId) { + skipped++; + continue; + } try { const { enqueued } = await runThumbnailBackfillForUser(userId); + users++; + thumbnailsEnqueued += enqueued; workerLog( "info", `[document-thumbnail-backfill] user=${userId} enqueued=${enqueued}`, @@ -1056,6 +1263,12 @@ export async function registerMaintenanceQueues( throw err; } } + return jobDone({ + jobs: jobs.length, + users, + skipped, + thumbnails_enqueued: thumbnailsEnqueued, + }); }, ); @@ -1063,13 +1276,20 @@ export async function registerMaintenanceQueues( // payload) fans out one per-user job per opted-in account; the queue also // serves the on-demand backfill payloads from the settings surface. Serial // concurrency so the staggered outbound fetches never crowd the request pool. - await boss.work( + await createAndWork( + boss, ENVIRONMENT_FETCH_QUEUE, { localConcurrency: ENVIRONMENT_FETCH_CONCURRENCY }, async (jobs) => { + // A discovery tick and a per-user backfill share this queue and are + // independent of each other, so the pass runs to the end and reports + // the first failure it saw. + let failure: JobOutcome | null = null; for (const job of jobs) { - await handleEnvironmentFetch(boss, job.data); + const outcome = await handleEnvironmentFetch(boss, job.data); + if (!outcome.ok && failure === null) failure = outcome; } + return failure ?? jobDone({ jobs: jobs.length }); }, ); diff --git a/src/lib/jobs/reminder/register-reminders.ts b/src/lib/jobs/reminder/register-reminders.ts index 66a42de2a..581a81358 100644 --- a/src/lib/jobs/reminder/register-reminders.ts +++ b/src/lib/jobs/reminder/register-reminders.ts @@ -7,9 +7,15 @@ * * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `boss.work(QUEUE, …, handler)` binding drains it. The + * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The * `cycle-reminder-queue` and `measurement-reminder-queue` guards read THIS * module. + * + * Retry policy: every queue here keeps the pg-boss default. These ticks are + * fan-out passes whose per-user failures never fail the job, so a job-level + * failure means the pass itself could not run — a transient the default two + * retries are the right answer to. The satisfy queue carries its own + * `retryLimit` at send time (see `reminder-satisfy.ts`). */ import { PgBoss } from "pg-boss"; import { @@ -17,7 +23,11 @@ import { REMINDER_SATISFY_CONCURRENCY, type ReminderSatisfyPayload, } from "@/lib/jobs/reminder-satisfy"; -import { createAndSchedule, type ScheduleEntry } from "./registrar-shared"; +import { + createAndSchedule, + createAndWork, + type ScheduleEntry, +} from "./registrar-shared"; import { ReminderCheckPayload, handleReminderCheck, @@ -121,7 +131,8 @@ export async function registerReminderQueues( ): Promise { await createAndSchedule(boss, allQueues, schedules); - await boss.work( + await createAndWork( + boss, QUEUE_NAME, { localConcurrency: 1 }, handleReminderCheck, @@ -130,7 +141,8 @@ export async function registerReminderQueues( // two reminder ticks from interleaving against the same user row; // the dedup ledger would still save us, but skipping the race here // avoids spurious P2002 errors in the wide-event log. - await boss.work( + await createAndWork( + boss, MOOD_REMINDER_QUEUE, { localConcurrency: 1 }, handleMoodReminderCheck, @@ -138,7 +150,8 @@ export async function registerReminderQueues( // v1.15 — single-flight cycle-reminder worker. localConcurrency=1 keeps // two ticks from racing the fire-and-forget `push_attempts` ledger that // anchors the per-day idempotency, exactly like the mood-reminder worker. - await boss.work( + await createAndWork( + boss, CYCLE_REMINDER_QUEUE, { localConcurrency: 1 }, handleCycleReminderCheck, @@ -147,7 +160,8 @@ export async function registerReminderQueues( // localConcurrency=1 keeps two ticks from racing the `nextDueAt` // advance that anchors the per-cycle idempotency, exactly like the // mood / cycle reminder workers. - await boss.work( + await createAndWork( + boss, MEASUREMENT_REMINDER_QUEUE, { localConcurrency: 1 }, handleMeasurementReminderCheck, @@ -155,7 +169,8 @@ export async function registerReminderQueues( // v1.18.1 — eventful Vorsorge satisfaction. Resolves a user's reminders // against their just-landed measurement / lab. Read-heavy on the user's // own data; the same small concurrency budget as PR detection. - await boss.work( + await createAndWork( + boss, REMINDER_SATISFY_QUEUE, { localConcurrency: REMINDER_SATISFY_CONCURRENCY }, handleReminderSatisfy, diff --git a/src/lib/jobs/reminder/register-rollup.ts b/src/lib/jobs/reminder/register-rollup.ts index 1b9a0ae01..eca1bf4ff 100644 --- a/src/lib/jobs/reminder/register-rollup.ts +++ b/src/lib/jobs/reminder/register-rollup.ts @@ -10,7 +10,7 @@ * * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `boss.work(QUEUE, …, handler)` binding drains it. The + * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The * `drain-cumulative-queue`, `mean-consolidation-queue`, and * `stress-strain-retention-queue` (dense-retention facet) guards read THIS * module. @@ -104,9 +104,12 @@ import { } from "./shared"; import { createAndSchedule, + createAndWork, + cronIsTheRetry, type QueuePolicyTable, type ScheduleEntry, } from "./registrar-shared"; +import { jobDone, jobFailed } from "@/lib/jobs/job-outcome"; // v1.4.37 W7c — nightly drain of per-sample APPLE_HEALTH cumulative rows. // Collapses each user × cumulative-type × calendar-day bucket into one // `stats:…` row so the list view stops painting hundreds of step chunks per @@ -170,7 +173,7 @@ const schedules: ScheduleEntry[] = [ // v1.4.37 W7c — nightly fold of per-sample APPLE_HEALTH cumulative // rows into one row per day per type. Slots between the // audit-log cleanup (03:15) and the feedback aggregator (04:00). - [DRAIN_CUMULATIVE_QUEUE, DRAIN_CUMULATIVE_CRON], + [DRAIN_CUMULATIVE_QUEUE, DRAIN_CUMULATIVE_CRON, cronIsTheRetry], ]; /** @@ -274,7 +277,8 @@ export async function registerRollupQueues( // the DAY bucket is already recomputed synchronously by the hook // itself. Concurrency-2 keeps two recomputes in flight without // crowding the dashboard request pool. - await boss.work( + await createAndWork( + boss, ROLLUP_RECOMPUTE_QUEUE, { localConcurrency: ROLLUP_RECOMPUTE_CONCURRENCY }, async (jobs) => { @@ -287,6 +291,10 @@ export async function registerRollupQueues( to: new Date(payload.to), }); } + // No per-bucket isolation here on purpose: a recompute that throws + // fails the job, so reaching this line means every bucket in the batch + // was folded. + return jobDone({ buckets_recomputed: jobs.length }); }, ); @@ -295,18 +303,25 @@ export async function registerRollupQueues( // full `recomputeUserRollups` against the default 5-year window // across every granularity. Serial concurrency so the populator // never crowds the dashboard request pool. - await boss.work( + await createAndWork( + boss, ROLLUP_FULL_BACKFILL_QUEUE, { localConcurrency: ROLLUP_FULL_BACKFILL_CONCURRENCY }, async (jobs) => { + let rowsUpsertedTotal = 0; for (const job of jobs) { const { userId } = job.data; const { rowsUpserted, durationMs } = await recomputeUserRollups(userId); + rowsUpsertedTotal += rowsUpserted; workerLog( "info", `[rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, ); } + return jobDone({ + users: jobs.length, + rows_upserted: rowsUpsertedTotal, + }); }, ); @@ -315,19 +330,29 @@ export async function registerRollupQueues( // step rows; this handler collapses them into one daily-total row per // calendar day and soft-deletes the originals. Serial concurrency so // the populator never crowds the dashboard request pool. - await boss.work( + await createAndWork( + boss, STEP_CONSOLIDATION_QUEUE, { localConcurrency: STEP_CONSOLIDATION_CONCURRENCY }, async (jobs) => { + let daysTotal = 0; + let legacyRowsTotal = 0; for (const job of jobs) { const { userId } = job.data; const { daysConsolidated, legacyRowsSoftDeleted } = await runStepConsolidationForUser(userId); + daysTotal += daysConsolidated; + legacyRowsTotal += legacyRowsSoftDeleted; workerLog( "info", `[step-consolidation] user=${userId} days=${daysConsolidated} legacyRowsSoftDeleted=${legacyRowsSoftDeleted}`, ); } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + legacy_rows_soft_deleted: legacyRowsTotal, + }); }, ); @@ -335,18 +360,39 @@ export async function registerRollupQueues( // GOOGLE_HEALTH / FITBIT daily-total step rows the pre-fix consolidation // swept and removes the shadow MANUAL totals it minted. Serial // concurrency so the repair never crowds the dashboard request pool. - await boss.work( + await createAndWork( + boss, STEP_CONSOLIDATION_REPAIR_QUEUE, { localConcurrency: STEP_CONSOLIDATION_REPAIR_CONCURRENCY }, async (jobs) => { + let rowsResurrected = 0; + let wedgeSkipped = 0; + let manualMintsRemoved = 0; + let daysRecomputed = 0; + let failures = 0; for (const job of jobs) { const { userId } = job.data; const summary = await runStepConsolidationRepairForUser(userId); + rowsResurrected += summary.rowsResurrected; + wedgeSkipped += summary.wedgeSkipped; + manualMintsRemoved += summary.manualMintsRemoved; + daysRecomputed += summary.daysRecomputed; + // The repair isolates its own per-day failures and reports them as a + // count, so they ride out as a fact rather than failing the job. + failures += summary.failures; workerLog( "info", `[step-consolidation-repair] user=${userId} resurrected=${summary.rowsResurrected} wedgeSkipped=${summary.wedgeSkipped} manualMintsRemoved=${summary.manualMintsRemoved} daysRecomputed=${summary.daysRecomputed} failures=${summary.failures}`, ); } + return jobDone({ + users: jobs.length, + rows_resurrected: rowsResurrected, + wedge_skipped: wedgeSkipped, + manual_mints_removed: manualMintsRemoved, + days_recomputed: daysRecomputed, + failures, + }); }, ); @@ -356,18 +402,28 @@ export async function registerRollupQueues( // it and re-runs detection silently so the honest re-derived best takes // its place. Serial concurrency so the repair never crowds the // dashboard request pool. - await boss.work( + await createAndWork( + boss, CUMULATIVE_PR_REDERIVE_QUEUE, { localConcurrency: CUMULATIVE_PR_REDERIVE_CONCURRENCY }, async (jobs) => { + let rowsDeleted = 0; + let rowsReinserted = 0; for (const job of jobs) { const { userId } = job.data; const summary = await runCumulativePrRederivationForUser(userId); + rowsDeleted += summary.rowsDeleted; + rowsReinserted += summary.rowsReinserted; workerLog( "info", `[cumulative-pr-rederive] user=${userId} rowsDeleted=${summary.rowsDeleted} rowsReinserted=${summary.rowsReinserted}`, ); } + return jobDone({ + users: jobs.length, + rows_deleted: rowsDeleted, + rows_reinserted: rowsReinserted, + }); }, ); @@ -376,19 +432,29 @@ export async function registerRollupQueues( // mean-type rows; this handler collapses each completed day to its // mean and soft-deletes the originals. Serial concurrency so the // populator never crowds the dashboard request pool. - await boss.work( + await createAndWork( + boss, MEAN_CONSOLIDATION_QUEUE, { localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY }, async (jobs) => { + let daysTotal = 0; + let perSampleRowsTotal = 0; for (const job of jobs) { const { userId } = job.data; const { daysConsolidated, perSampleRowsSoftDeleted } = await runMeanConsolidationForUser(userId); + daysTotal += daysConsolidated; + perSampleRowsTotal += perSampleRowsSoftDeleted; workerLog( "info", `[mean-consolidation] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted}`, ); } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + per_sample_rows_soft_deleted: perSampleRowsTotal, + }); }, ); @@ -399,10 +465,14 @@ export async function registerRollupQueues( // soft-deletes the originals, keeping the in-window intra-day shape intact // for the Stress engine. Serial concurrency so the backfill never crowds // the request pool. - await boss.work( + await createAndWork( + boss, DENSE_INTRADAY_RETENTION_QUEUE, { localConcurrency: DENSE_INTRADAY_RETENTION_CONCURRENCY }, async (jobs) => { + let daysTotal = 0; + let perSampleRowsTotal = 0; + let derivedRestingRowsTotal = 0; for (const job of jobs) { const { userId } = job.data; try { @@ -411,6 +481,9 @@ export async function registerRollupQueues( perSampleRowsSoftDeleted, derivedRestingRowsUpserted, } = await runDenseIntradayRetentionForUser(userId); + daysTotal += daysConsolidated; + perSampleRowsTotal += perSampleRowsSoftDeleted; + derivedRestingRowsTotal += derivedRestingRowsUpserted; workerLog( "info", `[dense-intraday-retention] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted} derivedRestingRowsUpserted=${derivedRestingRowsUpserted}`, @@ -425,6 +498,12 @@ export async function registerRollupQueues( throw err; } } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + per_sample_rows_soft_deleted: perSampleRowsTotal, + derived_resting_rows_upserted: derivedRestingRowsTotal, + }); }, ); @@ -435,10 +514,15 @@ export async function registerRollupQueues( // Self-converging: a rebuilt day drops out of the discovery pairing, so // the pass runs once per install. Serial concurrency so the rebuild never // crowds the request pool. - await boss.work( + await createAndWork( + boss, DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, { localConcurrency: DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY }, async (jobs) => { + let daysRebuiltTotal = 0; + let hourlyRowsTotal = 0; + let dailyRowsRetiredTotal = 0; + let daysSkippedTotal = 0; for (const job of jobs) { const { userId } = job.data; try { @@ -448,6 +532,10 @@ export async function registerRollupQueues( dailyRowsRetired, daysSkippedNoTombstones, } = await runDenseIntradayHourlyRebuildForUser(userId); + daysRebuiltTotal += daysRebuilt; + hourlyRowsTotal += hourlyRowsUpserted; + dailyRowsRetiredTotal += dailyRowsRetired; + daysSkippedTotal += daysSkippedNoTombstones; workerLog( "info", `[dense-intraday-hourly-rebuild] user=${userId} daysRebuilt=${daysRebuilt} hourlyRowsUpserted=${hourlyRowsUpserted} dailyRowsRetired=${dailyRowsRetired} daysSkippedNoTombstones=${daysSkippedNoTombstones}`, @@ -462,6 +550,13 @@ export async function registerRollupQueues( throw err; } } + return jobDone({ + users: jobs.length, + days_rebuilt: daysRebuiltTotal, + hourly_rows_upserted: hourlyRowsTotal, + daily_rows_retired: dailyRowsRetiredTotal, + days_skipped_no_tombstones: daysSkippedTotal, + }); }, ); @@ -471,7 +566,8 @@ export async function registerRollupQueues( // No current read path consumes these buckets — they exist so a // future cross-granularity reader can ship without a backfill // step. Concurrency-2 mirrors the measurement-rollup worker. - await boss.work( + await createAndWork( + boss, MOOD_ROLLUP_RECOMPUTE_QUEUE, { localConcurrency: MOOD_ROLLUP_RECOMPUTE_CONCURRENCY }, async (jobs) => { @@ -483,6 +579,7 @@ export async function registerRollupQueues( to: new Date(payload.to), }); } + return jobDone({ buckets_recomputed: jobs.length }); }, ); @@ -491,10 +588,12 @@ export async function registerRollupQueues( // but zero rollup rows; this handler folds the full 5-year window // across every granularity. Concurrency-1 so the populator never // crowds the request pool. - await boss.work( + await createAndWork( + boss, MOOD_ROLLUP_FULL_BACKFILL_QUEUE, { localConcurrency: MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY }, async (jobs) => { + let rowsUpsertedTotal = 0; for (const job of jobs) { const { userId } = job.data; try { @@ -508,6 +607,7 @@ export async function registerRollupQueues( userId, { replace: true }, ); + rowsUpsertedTotal += rowsUpserted; workerLog( "info", `[mood-rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, @@ -522,6 +622,10 @@ export async function registerRollupQueues( throw err; } } + return jobDone({ + users: jobs.length, + rows_upserted: rowsUpsertedTotal, + }); }, ); @@ -530,15 +634,18 @@ export async function registerRollupQueues( // but zero rollup coverage; this handler folds the trailing 90-day // window per account. Concurrency-1 so the populator never crowds // the request pool. - await boss.work( + await createAndWork( + boss, MEDICATION_COMPLIANCE_BACKFILL_QUEUE, { localConcurrency: MEDICATION_COMPLIANCE_BACKFILL_CONCURRENCY }, async (jobs) => { + let rowsUpsertedTotal = 0; for (const job of jobs) { const { userId } = job.data; try { const { rowsUpserted, durationMs } = await recomputeUserMedicationCompliance(userId); + rowsUpsertedTotal += rowsUpserted; workerLog( "info", `[medication-compliance-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, @@ -553,6 +660,10 @@ export async function registerRollupQueues( throw err; } } + return jobDone({ + users: jobs.length, + rows_upserted: rowsUpsertedTotal, + }); }, ); @@ -563,10 +674,26 @@ export async function registerRollupQueues( // Concurrency-1 so the drain never crowds the dashboard request pool // and a long backfill on the maintainer's account (300 k+ measurement rows) // stays a single sequential walk. - await boss.work( + await createAndWork( + boss, DRAIN_CUMULATIVE_QUEUE, { localConcurrency: 1 }, async (jobs) => { + // The three drains below are independent passes sharing one nightly + // tick, so one failing pass must not stop the other two from running. + // The first failure is held back and reported once the tick is over — + // that keeps the existing order and still fails the job, which is what + // the swallowed errors never did. + let firstFailure: { reason: string; cause: unknown } | undefined; + let passesFailed = 0; + let bucketsCollapsed = 0; + let perSampleRowsDeleted = 0; + let meanDaysConsolidated = 0; + let meanRowsSoftDeleted = 0; + let denseDaysConsolidated = 0; + let denseRowsSoftDeleted = 0; + let denseSkipped = false; + for (const job of jobs) { try { const summary = await drainPerSampleCumulative(getWorkerPrisma(), { @@ -574,6 +701,8 @@ export async function registerRollupQueues( cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS, log: (line) => workerLog("info", line), }); + bucketsCollapsed += summary.totals.bucketsCollapsed; + perSampleRowsDeleted += summary.totals.perSampleRowsDeleted; workerLog( "info", `[drain-cumulative] triggeredAt=${job.data.triggeredAt} usersScanned=${summary.totals.usersScanned} bucketsCollapsed=${summary.totals.bucketsCollapsed} perSampleRowsDeleted=${summary.totals.perSampleRowsDeleted} dailyRowsUpserted=${summary.totals.dailyRowsUpserted}`, @@ -581,6 +710,8 @@ export async function registerRollupQueues( } catch (err) { recordError(); await reportWorkerError("drain-cumulative", err); + passesFailed++; + firstFailure ??= { reason: "cumulative drain failed", cause: err }; } // v1.8.5 — fold the daily-MEAN drain onto the same nightly tick as @@ -600,6 +731,8 @@ export async function registerRollupQueues( cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS, log: (line) => workerLog("info", line), }); + meanDaysConsolidated += meanSummary.totals.daysConsolidated; + meanRowsSoftDeleted += meanSummary.totals.perSampleRowsSoftDeleted; workerLog( "info", `[mean-consolidation] triggeredAt=${job.data.triggeredAt} usersScanned=${meanSummary.totals.usersScanned} daysConsolidated=${meanSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${meanSummary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${meanSummary.totals.dailyRowsUpserted} daysFailed=${meanSummary.totals.daysFailed}`, @@ -609,6 +742,8 @@ export async function registerRollupQueues( await reportWorkerError("mean-consolidation", err, { tick: "nightly", }); + passesFailed++; + firstFailure ??= { reason: "mean consolidation failed", cause: err }; } // v1.10.0 WX-E — fold the dense intra-day retention drain onto the @@ -624,6 +759,7 @@ export async function registerRollupQueues( // shipped. try { if (!DENSE_INTRADAY_RETENTION_ENABLED) { + denseSkipped = true; workerLog( "info", "[dense-intraday-retention] disabled via DENSE_INTRADAY_RETENTION_ENABLED — skipping the nightly walk (operator kill-switch)", @@ -636,6 +772,9 @@ export async function registerRollupQueues( log: (line) => workerLog("info", line), }, ); + denseDaysConsolidated += denseSummary.totals.daysConsolidated; + denseRowsSoftDeleted += + denseSummary.totals.perSampleRowsSoftDeleted; workerLog( "info", `[dense-intraday-retention] triggeredAt=${job.data.triggeredAt} usersScanned=${denseSummary.totals.usersScanned} daysConsolidated=${denseSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${denseSummary.totals.perSampleRowsSoftDeleted} hourlyRowsUpserted=${denseSummary.totals.hourlyRowsUpserted} dailyRowsRetired=${denseSummary.totals.dailyRowsRetired}`, @@ -646,8 +785,26 @@ export async function registerRollupQueues( await reportWorkerError("dense-intraday-retention", err, { tick: "nightly", }); + passesFailed++; + firstFailure ??= { reason: "dense retention failed", cause: err }; } } + + const did = { + passes_failed: passesFailed, + buckets_collapsed: bucketsCollapsed, + per_sample_rows_deleted: perSampleRowsDeleted, + mean_days_consolidated: meanDaysConsolidated, + mean_rows_soft_deleted: meanRowsSoftDeleted, + dense_days_consolidated: denseDaysConsolidated, + dense_rows_soft_deleted: denseRowsSoftDeleted, + dense_skipped: denseSkipped, + }; + + if (firstFailure) { + return jobFailed(firstFailure.reason, firstFailure.cause, did); + } + return jobDone(did); }, ); diff --git a/src/lib/jobs/reminder/register-status.ts b/src/lib/jobs/reminder/register-status.ts index 5e2efa814..9ebca0bc2 100644 --- a/src/lib/jobs/reminder/register-status.ts +++ b/src/lib/jobs/reminder/register-status.ts @@ -10,8 +10,8 @@ * * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `boss.work(QUEUE, …, handler)` binding drains it. The status-wiring - * guards (`status-cron-candidates`, `insight-pregenerate`, + * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The + * status-wiring guards (`status-cron-candidates`, `insight-pregenerate`, * `insight-status-generate`, `recovery-score-queue`, * `stress-strain-retention-queue`, `period-narrative-warm`, * `coach-memory-refresh-queue`) read THIS module. @@ -101,9 +101,11 @@ import { type InsightStatusGeneratePayload, } from "@/lib/jobs/insight-status-generate"; import { withBackgroundEvent } from "@/lib/logging/background"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { getWorkerPrisma, workerLog } from "./shared"; import { createAndSchedule, + createAndWork, insightRetryOptions, type QueuePolicyTable, type ScheduleEntry, @@ -363,37 +365,44 @@ export async function registerStatusQueues( ): Promise { await createAndSchedule(boss, allQueues, schedules, queuePolicies); - await boss.work( + await createAndWork( + boss, GENERAL_STATUS_QUEUE, { localConcurrency: 1 }, handleGeneralStatusGenerate, ); - await boss.work( + await createAndWork( + boss, BLOOD_PRESSURE_STATUS_QUEUE, { localConcurrency: 1 }, handleBloodPressureStatusGenerate, ); - await boss.work( + await createAndWork( + boss, WEIGHT_STATUS_QUEUE, { localConcurrency: 1 }, handleWeightStatusGenerate, ); - await boss.work( + await createAndWork( + boss, PULSE_STATUS_QUEUE, { localConcurrency: 1 }, handlePulseStatusGenerate, ); - await boss.work( + await createAndWork( + boss, BMI_STATUS_QUEUE, { localConcurrency: 1 }, handleBmiStatusGenerate, ); - await boss.work( + await createAndWork( + boss, MOOD_STATUS_QUEUE, { localConcurrency: 1 }, handleMoodStatusGenerate, ); - await boss.work( + await createAndWork( + boss, MEDICATION_COMPLIANCE_STATUS_QUEUE, { localConcurrency: 1 }, handleMedicationComplianceStatusGenerate, @@ -409,7 +418,8 @@ export async function registerStatusQueues( // (the cohort walk is itself sequential per user, and the content-hash // gate + per-user budget gate inside the handler cover the rare // double-tick overlap the old single slot serialised away). - await boss.work( + await createAndWork( + boss, INSIGHT_PREGENERATE_QUEUE, { localConcurrency: 2 }, handleInsightPregenerateJob, @@ -420,17 +430,23 @@ export async function registerStatusQueues( // night's samples into one job, and the handler's marker re-check makes a // rare double-fire a no-op, so one slot is enough and bounds the provider // concurrency the same way the nightly path does. - await boss.work( + await createAndWork( + boss, MORNING_DIGEST_REFRESH_QUEUE, { localConcurrency: 1 }, - async (jobs) => { - await withBackgroundEvent("job.morning_digest_refresh", async (evt) => { + async (jobs): Promise => { + return withBackgroundEvent("job.morning_digest_refresh", async (evt) => { + // The counts the batch already decides on: one refresh per job, and + // the finalised subset that reaches the morning push seam. + let refreshed = 0; + let finalised = 0; for (const job of jobs) { try { const result = await runMorningDigestRefresh( getWorkerPrisma(), job.data, ); + refreshed++; evt.addMeta( "morning_refresh", `${result.status}:${result.comprehensive ?? "none"}`, @@ -442,6 +458,7 @@ export async function registerStatusQueues( // there, so a user who never opted in or is outside the window is a // clean no-op. The fallback cron covers the no-sleep-arrived case. if (result.status === "finalised") { + finalised++; const pushResult = await maybeDispatchDailyBriefing( getWorkerPrisma(), job.data.userId, @@ -457,6 +474,7 @@ export async function registerStatusQueues( throw err; } } + return jobDone({ refreshed, finalised }); }); }, ); @@ -464,7 +482,8 @@ export async function registerStatusQueues( // read-only status route on a cold card. Low concurrency so a first // visit that cold-misses several cards can't saturate the Prisma pool // or fan out an unbounded number of concurrent provider calls. - await boss.work( + await createAndWork( + boss, INSIGHT_STATUS_GENERATE_QUEUE, { localConcurrency: INSIGHT_STATUS_GENERATE_CONCURRENCY }, handleInsightStatusGenerate, @@ -474,56 +493,84 @@ export async function registerStatusQueues( // eligible user and upserts one `COMPUTED RECOVERY_SCORE` row per scored // day (idempotent — a re-fire overwrites in place). Single-flight so two // ticks never double-walk the cohort. - await boss.work(RECOVERY_SCORE_QUEUE, { localConcurrency: 1 }, async () => { - try { - const summary = await runRecoveryScore(getWorkerPrisma()); - workerLog( - "info", - `[recovery-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - } catch (err) { - recordError(); - workerLog("error", "[recovery-score] pass failed", err); - throw err; - } - }); + await createAndWork( + boss, + RECOVERY_SCORE_QUEUE, + { localConcurrency: 1 }, + async (): Promise => { + try { + const summary = await runRecoveryScore(getWorkerPrisma()); + workerLog( + "info", + `[recovery-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, + ); + return jobDone({ + considered: summary.considered, + stored: summary.stored, + insufficient: summary.insufficient, + errored: summary.errored, + }); + } catch (err) { + recordError(); + workerLog("error", "[recovery-score] pass failed", err); + throw err; + } + }, + ); // v1.15.20 — proactive Coach nudge. Single-flight; the push-attempts // ledger caps a user at one nudge per rolling week, so an overlapping // tick would only waste reads. Deterministic triggers, no AI call. - await boss.work(COACH_NUDGE_QUEUE, { localConcurrency: 1 }, async () => { - await withBackgroundEvent("job.coach_nudge", async (evt) => { - try { - const summary = await runCoachNudgeTick(getWorkerPrisma(), new Date()); - evt.setBackground({ - task_name: "job.coach_nudge", - result: { + await createAndWork( + boss, + COACH_NUDGE_QUEUE, + { localConcurrency: 1 }, + async (): Promise => { + return withBackgroundEvent("job.coach_nudge", async (evt) => { + try { + const summary = await runCoachNudgeTick( + getWorkerPrisma(), + new Date(), + ); + evt.setBackground({ + task_name: "job.coach_nudge", + result: { + candidates_scanned: summary.candidatesScanned, + dispatched: summary.dispatched, + persisted: summary.persisted, + skipped_opted_out: summary.skippedOptedOut, + skipped_no_provider: summary.skippedNoProvider, + skipped_recent_nudge: summary.skippedRecentNudge, + skipped_recent_engagement: summary.skippedRecentEngagement, + skipped_no_trigger: summary.skippedNoTrigger, + skipped_no_channel: summary.skippedNoChannel, + failed: summary.failed, + }, + }); + // A per-user failure never fails this pass — it rides out as the + // `failed` count, exactly as it does in the wide event. + return jobDone({ candidates_scanned: summary.candidatesScanned, dispatched: summary.dispatched, persisted: summary.persisted, - skipped_opted_out: summary.skippedOptedOut, - skipped_no_provider: summary.skippedNoProvider, - skipped_recent_nudge: summary.skippedRecentNudge, - skipped_recent_engagement: summary.skippedRecentEngagement, - skipped_no_trigger: summary.skippedNoTrigger, - skipped_no_channel: summary.skippedNoChannel, failed: summary.failed, - }, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }); + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); + }, + ); // v1.16.11 — medication low-stock pass. Single-flight; the // per-medication stamp makes a re-fire idempotent (already-notified // crossings skip), so an overlapping tick would only waste reads. - await boss.work( + await createAndWork( + boss, MEDICATION_LOW_STOCK_QUEUE, { localConcurrency: 1 }, - async () => { - await withBackgroundEvent("job.medication_low_stock", async (evt) => { + async (): Promise => { + return withBackgroundEvent("job.medication_low_stock", async (evt) => { try { const summary = await runMedicationLowStockTick( getWorkerPrisma(), @@ -544,6 +591,15 @@ export async function registerStatusQueues( failed: summary.failed, }, }); + // A per-medication failure never fails this pass — it rides out as + // the `failed` count. + return jobDone({ + users_scanned: summary.usersScanned, + medications_evaluated: summary.medicationsEvaluated, + notified: summary.notified, + rearmed: summary.rearmed, + failed: summary.failed, + }); } catch (err) { evt.setError(err); recordError(); @@ -556,7 +612,8 @@ export async function registerStatusQueues( // to claim a durable unique row, so two slots would only race each other for // a claim exactly one of them can win. It makes no provider call — a // module-graph test pins that — so one slot costs nothing in throughput. - await boss.work( + await createAndWork( + boss, DATA_ARRIVAL_QUEUE, { localConcurrency: 1 }, handleDataArrival, @@ -564,7 +621,8 @@ export async function registerStatusQueues( // v1.31.0 — the arrival reaction line. Single-flight: this is the spine's // ONE provider call, and a second concurrent slot would only race the same // durable `generatedAt` claim exactly one of them can win. - await boss.work( + await createAndWork( + boss, REACTION_LINE_QUEUE, { localConcurrency: 1 }, handleReactionLine, @@ -573,7 +631,8 @@ export async function registerStatusQueues( // path the spine deliberately does not walk, so it stays off the request pool // and cannot fan out concurrent completions when several workouts land in one // sync. - await boss.work( + await createAndWork( + boss, WORKOUT_INSIGHT_GENERATE_QUEUE, { localConcurrency: WORKOUT_INSIGHT_GENERATE_CONCURRENCY }, handleWorkoutInsightGenerate, @@ -583,84 +642,131 @@ export async function registerStatusQueues( // is suppressed), and the tick only loads a digest for opted-in users at // their exact local fallback hour. No provider call — it reads the cached // digest and dispatches the deterministic line + cached lead. - await boss.work(DAILY_BRIEFING_QUEUE, { localConcurrency: 1 }, async () => { - await withBackgroundEvent("job.daily_briefing", async (evt) => { - try { - const summary = await runDailyBriefingTick( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.daily_briefing", - result: { + await createAndWork( + boss, + DAILY_BRIEFING_QUEUE, + { localConcurrency: 1 }, + async (): Promise => { + return withBackgroundEvent("job.daily_briefing", async (evt) => { + try { + const summary = await runDailyBriefingTick( + getWorkerPrisma(), + new Date(), + ); + evt.setBackground({ + task_name: "job.daily_briefing", + result: { + candidates_scanned: summary.candidatesScanned, + in_slot: summary.inSlot, + sent: summary.sent, + suppressed_frequency: summary.suppressedFrequency, + no_digest: summary.noDigest, + opted_out: summary.optedOut, + module_off: summary.moduleOff, + no_channel: summary.noChannel, + outside_window: summary.outsideWindow, + failed: summary.failed, + }, + }); + // Most ticks land outside every user's slot; that is the tick doing + // its job, so a zero `sent` is a success with zero counts. + return jobDone({ candidates_scanned: summary.candidatesScanned, in_slot: summary.inSlot, sent: summary.sent, - suppressed_frequency: summary.suppressedFrequency, - no_digest: summary.noDigest, - opted_out: summary.optedOut, - module_off: summary.moduleOff, - no_channel: summary.noChannel, - outside_window: summary.outsideWindow, failed: summary.failed, - }, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }); + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); + }, + ); // v1.10.0 — computed scores (WX-E). Nightly Stress-score (HRV-derived // proxy) compute + store. Single-flight so two ticks never double-walk // the cohort. The runner iterates every eligible user and upserts one // `COMPUTED STRESS_SCORE` row per scored day (idempotent — a re-fire // overwrites in place). - await boss.work(STRESS_SCORE_QUEUE, { localConcurrency: 1 }, async () => { - try { - const summary = await runStressScore(getWorkerPrisma()); - workerLog( - "info", - `[stress-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - } catch (err) { - recordError(); - workerLog("error", "[stress-score] pass failed", err); - throw err; - } - }); + await createAndWork( + boss, + STRESS_SCORE_QUEUE, + { localConcurrency: 1 }, + async (): Promise => { + try { + const summary = await runStressScore(getWorkerPrisma()); + workerLog( + "info", + `[stress-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, + ); + return jobDone({ + considered: summary.considered, + stored: summary.stored, + insufficient: summary.insufficient, + errored: summary.errored, + }); + } catch (err) { + recordError(); + workerLog("error", "[stress-score] pass failed", err); + throw err; + } + }, + ); // v1.10.0 — computed scores (WX-E). Nightly Strain-score (Banister TRIMP // cardio-load) compute + store. Single-flight; upserts one `COMPUTED // STRAIN_SCORE` row per scored day (idempotent). - await boss.work(STRAIN_SCORE_QUEUE, { localConcurrency: 1 }, async () => { - try { - const summary = await runStrainScore(getWorkerPrisma()); - workerLog( - "info", - `[strain-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - } catch (err) { - recordError(); - workerLog("error", "[strain-score] pass failed", err); - throw err; - } - }); + await createAndWork( + boss, + STRAIN_SCORE_QUEUE, + { localConcurrency: 1 }, + async (): Promise => { + try { + const summary = await runStrainScore(getWorkerPrisma()); + workerLog( + "info", + `[strain-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, + ); + return jobDone({ + considered: summary.considered, + stored: summary.stored, + insufficient: summary.insufficient, + errored: summary.errored, + }); + } catch (err) { + recordError(); + workerLog("error", "[strain-score] pass failed", err); + throw err; + } + }, + ); // v1.11.0 — period-narrative warm. A scheduled tick (no `userId`) runs the // boundary-gated nightly fan-out; a `userId` payload runs a single-user warm // enqueued by the read-only GET on a cold/stale read. Single-flight so two // ticks never double-walk the cohort; the per-user budget gate covers the // fan-out and the enqueue `singletonKey` covers the single-user path. - await boss.work( + await createAndWork( + boss, PERIOD_NARRATIVE_QUEUE, { localConcurrency: 1 }, - async (jobs) => { + async (jobs): Promise => { + // Two modes on one queue, so the outcome counts them separately: a + // non-boundary night is a scheduled pass that generated nothing, which + // is a success with zero counts, not an idle queue. + let singleUser = 0; + let scheduled = 0; + let generated = 0; + let failed = 0; for (const job of jobs) { try { if (job.data?.userId) { await warmOneNarrative(job.data); + singleUser++; } else { const summary = await runPeriodNarrativeWarm(getWorkerPrisma()); + scheduled++; + generated += summary.generated; + failed += summary.failed; workerLog( "info", `[period-narrative] periods=${summary.periods.join(",") || "none"} total=${summary.total} generated=${summary.generated} cached=${summary.cached} skipped=${summary.skipped} insufficient=${summary.insufficient} failed=${summary.failed} budget=${summary.budgetBlocked}`, @@ -674,26 +780,45 @@ export async function registerStatusQueues( throw err; } } + return jobDone({ + single_user: singleUser, + scheduled, + generated, + failed, + }); }, ); // v1.11.1 — combined Coach memory refresh: rolling conversation summary + // durable fact extraction for one long conversation. localConcurrency 1 so a // burst of long-conversation turns can't fan out concurrent provider calls; // each step is budget-gated inside runStatusCompletion and fault-isolated. - await boss.work( + await createAndWork( + boss, COACH_MEMORY_REFRESH_QUEUE, { localConcurrency: 1 }, - async (jobs) => { + async (jobs): Promise => { + // An incomplete payload is skipped rather than failed, so it has to be + // countable — otherwise a queue full of them reads as a clean run. + let refreshed = 0; + let skippedIncomplete = 0; for (const job of jobs) { - if (!job.data?.conversationId || !job.data?.userId) continue; + if (!job.data?.conversationId || !job.data?.userId) { + skippedIncomplete++; + continue; + } try { await runCoachMemoryRefresh(job.data); + refreshed++; } catch (err) { recordError(); workerLog("error", "[coach-memory-refresh] failed", err); throw err; } } + return jobDone({ + refreshed, + skipped_incomplete: skippedIncomplete, + }); }, ); @@ -701,11 +826,12 @@ export async function registerStatusQueues( // reminder to `due` and minting plan-review reminders are both idempotent // across re-fires (an already-due reminder is not re-flipped; a plan whose // reviewDate was cleared is not re-minted). No provider call, no push. - await boss.work( + await createAndWork( + boss, COACH_REMINDER_SWEEP_QUEUE, { localConcurrency: 1 }, - async () => { - await withBackgroundEvent("job.coach_reminder_sweep", async (evt) => { + async (): Promise => { + return withBackgroundEvent("job.coach_reminder_sweep", async (evt) => { try { const summary = await runCoachReminderSweep( getWorkerPrisma(), @@ -719,6 +845,11 @@ export async function registerStatusQueues( errored: summary.errored, }, }); + return jobDone({ + reminders_due: summary.remindersDue, + plan_reviews_minted: summary.planReviewsMinted, + errored: summary.errored, + }); } catch (err) { evt.setError(err); recordError(); @@ -732,11 +863,12 @@ export async function registerStatusQueues( // reminders the sweep minted and writes the grounded before/after read-back // into the plan's encrypted outcome column, flipping it to `reviewed`. // Idempotent (a reviewed plan is not re-processed). No provider call, no push. - await boss.work( + await createAndWork( + boss, COACH_PLAN_REVIEW_QUEUE, { localConcurrency: 1 }, - async () => { - await withBackgroundEvent("job.coach_plan_review", async (evt) => { + async (): Promise => { + return withBackgroundEvent("job.coach_plan_review", async (evt) => { try { const summary = await runCoachPlanReviewTick( getWorkerPrisma(), @@ -750,6 +882,11 @@ export async function registerStatusQueues( errored: summary.errored, }, }); + return jobDone({ + reviewed: summary.reviewed, + insufficient: summary.insufficient, + errored: summary.errored, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/reminder/registrar-shared.ts b/src/lib/jobs/reminder/registrar-shared.ts index ceb84ea2b..c05f1ea87 100644 --- a/src/lib/jobs/reminder/registrar-shared.ts +++ b/src/lib/jobs/reminder/registrar-shared.ts @@ -16,13 +16,18 @@ * assert a single aggregate `allQueues` (defence in depth: the per-registrar * arrays AND the boot-level union both have to agree). */ -import { PgBoss } from "pg-boss"; +import { PgBoss, type WorkOptions } from "pg-boss"; import { prisma } from "@/lib/db"; import { workerLog } from "./shared"; import { withBackgroundEvent } from "@/lib/logging/background"; import { annotate } from "@/lib/logging/context"; +import { + runJob, + type JobHandler, + type JobWithMetadataHandler, +} from "@/lib/jobs/run-job"; /** * Cron schedule tuple: `[queueName, cronExpression, sendOptions?]`. The @@ -162,6 +167,32 @@ export const insightRetryOptions = { retryBackoff: true, } as const; +/** + * Retry policy for a cron-driven pass whose failure mode is deterministic. + * + * Handlers now return a `JobOutcome`, and the retention cleanups that used to + * warn-and-return fail their job instead. That is the point — but it walks + * into pg-boss's queue default of `retryLimit: 2` with no delay and no + * backoff, so a bulk DELETE that timed out against a large trailing edge + * would run twice more, immediately, against the same rows. Three doomed + * sequential scans a night is a worse outcome than the silence it replaced. + * + * For this class the next cron tick IS the retry, and it arrives with the + * whole gap as backoff. So the schedule declares zero retries: the failure is + * recorded once, reaches the operator once, and the pass tries again on its + * own cadence. + * + * It rides on the SCHEDULE rather than on `createQueue`, deliberately. + * pg-boss's `create_queue()` ends in ON CONFLICT DO NOTHING, so a queue-level + * default would be inert on every already-running instance — the same trap + * `reconcileQueuePolicies` above exists to work around. Send options apply to + * every job the cron mints, new database or old. + * + * Use it only where the failure genuinely repeats. A transient failure (a TLS + * probe, an S3 fetch, a provider call) wants the default retries, not this. + */ +export const cronIsTheRetry = { retryLimit: 0 } as const; + /** * Create every queue in `queues`, then schedule every cron in `schedules`. * Centralised so each registrar provisions before it schedules in the exact @@ -198,3 +229,41 @@ export async function createAndSchedule( ); } } + +/** + * Bind a handler to a queue. The one legal way to reach `boss.work` in this + * tree — the `healthlog/job-handler-outcome` lint rule refuses a bare + * `boss.work(` anywhere else. + * + * The point of routing every binding through here is the handler type: + * `JobHandler` resolves to `Promise`, so a handler cannot be + * bound unless it returns a value declaring what it did. `runJob` then fails + * the pg-boss job on `ok: false` instead of completing it. Falling off the + * end of a handler, which used to read as success, is now a type error. + * + * Argument order matches `boss.work(queue, options, handler)` so a binding + * reads the same as it did before, with `boss` moved to the front. + */ +export async function createAndWork( + boss: PgBoss, + queue: string, + options: WorkOptions & { includeMetadata: true }, + handler: JobWithMetadataHandler, +): Promise; +export async function createAndWork( + boss: PgBoss, + queue: string, + options: WorkOptions, + handler: JobHandler, +): Promise; +export async function createAndWork( + boss: PgBoss, + queue: string, + options: WorkOptions, + handler: JobHandler | JobWithMetadataHandler, +): Promise { + // `JobWithMetadata` extends `Job`, so the metadata handler accepts + // everything the plain one does; the overloads above are what keeps the + // call sites honest about which shape they asked pg-boss for. + await boss.work(queue, options, runJob(queue, handler as JobHandler)); +} diff --git a/src/lib/jobs/reminder/whoop-sync.ts b/src/lib/jobs/reminder/whoop-sync.ts index 10f07df56..830ab4ebc 100644 --- a/src/lib/jobs/reminder/whoop-sync.ts +++ b/src/lib/jobs/reminder/whoop-sync.ts @@ -7,6 +7,7 @@ import { type Job } from "pg-boss"; import { fireAndForget } from "@/lib/logging/fire-and-forget"; import { recordError } from "@/lib/jobs/worker-status"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { syncUserRecovery, @@ -66,6 +67,10 @@ type ByIdSync = (userId: string, resourceId: string) => Promise; * - A cron tick (no `userId` on any job) walks every connection's collection. * * One user's parked-at-reauth state never starves the rest of the cohort. + * + * The pass is what the outcome judges: a per-user failure is that user's + * problem, recorded on their integration ledger and counted as `users_failed`, + * not a queue-wide failure that would retry the whole cohort. */ export async function runWhoopResourceSync( taskName: string, @@ -73,8 +78,8 @@ export async function runWhoopResourceSync( jobs: Job[], syncFn: CollectionSync, byIdFn?: ByIdSync, -): Promise { - await withBackgroundEvent(taskName, async (evt) => { +): Promise { + return withBackgroundEvent(taskName, async (evt) => { const prisma = getWorkerPrisma(); try { // (userId, optional resourceId) work items from the webhook payloads. @@ -94,9 +99,17 @@ export async function runWhoopResourceSync( }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } let usersSynced = 0; + let usersFailed = 0; let measurementsImported = 0; for (const { userId, resourceId } of targets) { try { @@ -123,6 +136,7 @@ export async function runWhoopResourceSync( if (!isSyncFailureRecorded(err)) { await recordWhoopSyncFailure(userId, err, leg); } + usersFailed++; evt.addWarning(`${taskName} failed for user ${userId}: ${err}`); } } @@ -135,6 +149,12 @@ export async function runWhoopResourceSync( measurements_imported: measurementsImported, }, }); + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); @@ -143,7 +163,9 @@ export async function runWhoopResourceSync( }); } -export function handleWhoopRecoverySync(jobs: Job[]) { +export function handleWhoopRecoverySync( + jobs: Job[], +): Promise { return runWhoopResourceSync( "job.whoop_recovery_sync", "recovery", @@ -153,7 +175,9 @@ export function handleWhoopRecoverySync(jobs: Job[]) { ); } -export function handleWhoopSleepSync(jobs: Job[]) { +export function handleWhoopSleepSync( + jobs: Job[], +): Promise { return runWhoopResourceSync( "job.whoop_sleep_sync", "sleep", @@ -163,7 +187,9 @@ export function handleWhoopSleepSync(jobs: Job[]) { ); } -export function handleWhoopWorkoutSync(jobs: Job[]) { +export function handleWhoopWorkoutSync( + jobs: Job[], +): Promise { return runWhoopResourceSync( "job.whoop_workout_sync", "workout", @@ -173,7 +199,9 @@ export function handleWhoopWorkoutSync(jobs: Job[]) { ); } -export function handleWhoopCycleSync(jobs: Job[]) { +export function handleWhoopCycleSync( + jobs: Job[], +): Promise { // Cycle has no webhook (poll-only), so no fetch-by-id path. return runWhoopResourceSync( "job.whoop_cycle_sync", diff --git a/src/lib/jobs/reminder/withings-sync.ts b/src/lib/jobs/reminder/withings-sync.ts index 96be0b354..153283b58 100644 --- a/src/lib/jobs/reminder/withings-sync.ts +++ b/src/lib/jobs/reminder/withings-sync.ts @@ -7,6 +7,7 @@ import { type Job } from "pg-boss"; import { fireAndForget } from "@/lib/logging/fire-and-forget"; import { recordError, recordWithingsSync } from "@/lib/jobs/worker-status"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; import { retryDueWithingsWebhookSubscriptions, @@ -50,15 +51,20 @@ export interface WithingsSleepSyncPayload { */ export async function handleWithingsFallbackSync( jobs: Job[], -) { +): Promise { void jobs; - await withBackgroundEvent("job.withings_sync", async (evt) => { + return withBackgroundEvent("job.withings_sync", async (evt) => { const prisma = getWorkerPrisma(); try { recordWithingsSync(); + // The subscription repair is an auxiliary leg, not the pass this queue is + // named for. Failing the job on it would skip the fallback sync that every + // user relies on when a webhook goes missing, so it rides out as a fact. + let subscriptionRepairFailed = false; try { await retryDueWithingsWebhookSubscriptions(); } catch { + subscriptionRepairFailed = true; evt.addWarning( "Withings subscription repair failed; continuing fallback sync", ); @@ -68,10 +74,17 @@ export async function handleWithingsFallbackSync( }); if (connections.length === 0) { - return; + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + subscription_repair_failed: subscriptionRepairFailed, + }); } let usersSynced = 0; + let usersFailed = 0; let measurementsImported = 0; for (const connection of connections) { @@ -87,6 +100,7 @@ export async function handleWithingsFallbackSync( }); } } catch { + usersFailed++; evt.addWarning(`Fallback sync failed for user ${connection.userId}`); } } @@ -99,6 +113,13 @@ export async function handleWithingsFallbackSync( measurements_imported: measurementsImported, }, }); + return jobDone({ + total: connections.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + subscription_repair_failed: subscriptionRepairFailed, + }); } catch (err) { evt.setError(err); recordError(); @@ -127,8 +148,8 @@ export async function handleWithingsFallbackSync( */ export async function handleWithingsEcgSync( jobs: Job[], -) { - await withBackgroundEvent("job.withings_ecg_sync", async (evt) => { +): Promise { + return withBackgroundEvent("job.withings_ecg_sync", async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ @@ -152,9 +173,17 @@ export async function handleWithingsEcgSync( }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + recordings_imported: 0, + }); + } let usersSynced = 0; + let usersFailed = 0; let recordingsImported = 0; for (const { userId, startdate, enddate } of targets) { try { @@ -165,6 +194,7 @@ export async function handleWithingsEcgSync( recordingsImported += await syncUserEcg(userId, options); usersSynced++; } catch (err) { + usersFailed++; evt.addWarning(`Withings ECG sync failed for user ${userId}: ${err}`); } } @@ -177,6 +207,12 @@ export async function handleWithingsEcgSync( recordings_imported: recordingsImported, }, }); + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + recordings_imported: recordingsImported, + }); } catch (err) { evt.setError(err); recordError(); @@ -203,8 +239,8 @@ export async function handleWithingsEcgSync( */ export async function handleWithingsActivitySync( jobs: Job[], -) { - await withBackgroundEvent("job.withings_activity_sync", async (evt) => { +): Promise { + return withBackgroundEvent("job.withings_activity_sync", async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; @@ -220,9 +256,17 @@ export async function handleWithingsActivitySync( }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } let usersSynced = 0; + let usersFailed = 0; let measurementsImported = 0; for (const { userId } of targets) { try { @@ -235,6 +279,7 @@ export async function handleWithingsActivitySync( }); } } catch { + usersFailed++; evt.addWarning(`Withings activity sync failed for user ${userId}`); } } @@ -247,6 +292,12 @@ export async function handleWithingsActivitySync( measurements_imported: measurementsImported, }, }); + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); @@ -262,8 +313,8 @@ export async function handleWithingsActivitySync( */ export async function handleWithingsSleepSync( jobs: Job[], -) { - await withBackgroundEvent("job.withings_sleep_sync", async (evt) => { +): Promise { + return withBackgroundEvent("job.withings_sleep_sync", async (evt) => { const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; @@ -278,9 +329,17 @@ export async function handleWithingsSleepSync( }); targets.push(...connections); } - if (targets.length === 0) return; + if (targets.length === 0) { + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } let usersSynced = 0; + let usersFailed = 0; let measurementsImported = 0; for (const { userId } of targets) { try { @@ -293,6 +352,7 @@ export async function handleWithingsSleepSync( }); } } catch { + usersFailed++; evt.addWarning(`Withings sleep sync failed for user ${userId}`); } } @@ -305,6 +365,12 @@ export async function handleWithingsSleepSync( measurements_imported: measurementsImported, }, }); + return jobDone({ + total: targets.length, + users_synced: usersSynced, + users_failed: usersFailed, + measurements_imported: measurementsImported, + }); } catch (err) { evt.setError(err); recordError(); diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index c37b6fc73..5497174ce 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -30,6 +30,7 @@ import { type S3Like, } from "@/lib/jobs/offhost-backup"; import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; export const RESTORE_DRILL_QUEUE = "data-restore-drill"; @@ -149,9 +150,11 @@ export async function runRestoreDrill( * page); every other failure goes through `reportWorkerError` so the * operator hears about a rotting backup chain. */ -export async function handleRestoreDrill(jobs: Job[]): Promise { +export async function handleRestoreDrill( + jobs: Job[], +): Promise { void jobs; - await withBackgroundEvent("job.restore_drill", async (evt) => { + return withBackgroundEvent("job.restore_drill", async (evt) => { try { const report = await runRestoreDrill(); evt.addMeta("restore_drill_object_key", report.objectKey); @@ -181,13 +184,31 @@ export async function handleRestoreDrill(jobs: Job[]): Promise { { objectKey: report.objectKey, ageDays: report.ageDays }, ); } + // A stale chain is a verdict about the uploader, not a failure of the + // drill: the artefact was fetched, decrypted and parsed, and the page + // above already reached the operator. Failing the job here would retry a + // reading that cannot change until the next upload lands. + return jobDone({ + restore_drill_object_key: report.objectKey, + restore_drill_age_days: report.ageDays, + restore_drill_ciphertext_bytes: report.ciphertextBytes, + restore_drill_plaintext_bytes: report.plaintextBytes, + restore_drill_measurements: report.recordCounts.measurements, + restore_drill_medications: report.recordCounts.medications, + restore_drill_intake_events: report.recordCounts.intakeEvents, + restore_drill_mood_entries: report.recordCounts.moodEntries, + restore_drill_stale: report.stale, + }); } catch (err) { if (err instanceof OffhostBackupNotConfiguredError) { evt.addWarning(`restore-drill skipped: ${err.message}`); - return; + // Most self-hosters never set the S3 vars. Nothing to verify is not a + // failed verification, and a monthly failed job would be noise. + return jobDone({ skipped: "offhost backup not configured" }); } evt.addWarning(`restore-drill failed: ${err}`); await reportWorkerError(RESTORE_DRILL_QUEUE, err); + return jobFailed("restore drill failed", err); } }); } diff --git a/src/lib/jobs/run-job.ts b/src/lib/jobs/run-job.ts new file mode 100644 index 000000000..05a180084 --- /dev/null +++ b/src/lib/jobs/run-job.ts @@ -0,0 +1,94 @@ +/** + * The wrapper that turns a returned `JobOutcome` into a pg-boss result. + * + * `boss.work(queue, opts, handler)` completes a job when the handler's + * promise resolves, whatever it resolves to. Wrapping every handler in + * `runJob` moves that decision from "the function returned" to "the function + * said it succeeded": an `ok: false` outcome is reported and rethrown, so + * pg-boss records a failed job, the queue's retry policy applies, and the + * failed-job readers in `job-failures.ts` surface it to the operator. + * + * This changes runtime behaviour, not only visibility. Handlers that used to + * catch, warn and return now fail their job. For a queue whose failure mode + * is deterministic — a DELETE that times out will time out again — retrying + * is wasted load, so the retry policy of every converted queue has to be a + * decision rather than a default. + * + * The wrapper deliberately does not open a wide event of its own. Handlers + * own their `withBackgroundEvent` scope and annotate inside it; a second + * event here would double every worker line for no added fact. + */ +import type { Job, JobWithMetadata } from "pg-boss"; + +import type { JobFact, JobFacts, JobOutcome } from "./job-outcome"; +import { reportWorkerError } from "./report-worker-error"; + +/** A converted handler: it says what it did instead of just finishing. */ +export type JobHandler = (jobs: Job[]) => Promise; + +/** The same, for a queue bound with `includeMetadata: true`. */ +export type JobWithMetadataHandler = ( + jobs: JobWithMetadata[], +) => Promise; + +/** + * The error a failed outcome becomes. Carries the queue and the handler's + * own reason so the pg-boss `output` column, the GlitchTip event and the + * container log all name the same thing. + */ +export class JobFailure extends Error { + readonly queue: string; + readonly reason: string; + + constructor(queue: string, reason: string, cause?: unknown) { + super(`${queue}: ${reason}`, cause === undefined ? undefined : { cause }); + this.name = "JobFailure"; + this.queue = queue; + this.reason = reason; + // Carry the thrown error's stack. Without this the only stack the + // operator ever sees points at `runJob`, which is the same for all 101 + // queues and says nothing about where the work actually broke. + if (cause instanceof Error && cause.stack) { + this.stack = `${this.name}: ${this.message}\ncaused by ${cause.stack}`; + } + } +} + +function factMeta(did: JobFacts | undefined): Record { + const meta: Record = {}; + for (const [key, value] of Object.entries(did ?? {})) meta[key] = value; + return meta; +} + +/** + * Wrap a handler for `boss.work`. Resolves on `ok: true`; reports and + * rethrows on `ok: false`. + * + * A handler that throws is left alone — the throw already fails the job, and + * re-wrapping it would only bury the original stack. `runJob` exists for the + * case the throw never happens. + */ +export function runJob( + queue: string, + handler: (jobs: J[]) => Promise, +): (jobs: J[]) => Promise { + return async (jobs: J[]): Promise => { + const outcome = await handler(jobs); + if (outcome.ok) return; + + const failure = new JobFailure(queue, outcome.reason, outcome.cause); + const causeMessage = + outcome.cause instanceof Error + ? outcome.cause.message + : typeof outcome.cause === "string" + ? outcome.cause + : undefined; + + await reportWorkerError(queue, failure, { + ...factMeta(outcome.did), + ...(causeMessage === undefined ? {} : { cause: causeMessage }), + }); + + throw failure; + }; +} diff --git a/src/lib/jobs/workout-insight-generate.ts b/src/lib/jobs/workout-insight-generate.ts index 4a690d088..a5854f285 100644 --- a/src/lib/jobs/workout-insight-generate.ts +++ b/src/lib/jobs/workout-insight-generate.ts @@ -48,6 +48,7 @@ import { pickCanonicalWorkoutRows } from "@/lib/measurements/pick-canonical-work import { defaultLocale, locales, type Locale } from "@/lib/i18n/config"; import { finalizeStatusSummary } from "@/lib/insights/status-shared"; import { runStatusCompletion } from "@/lib/insights/status-provider"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { annotate } from "@/lib/logging/context"; import { withBackgroundEvent } from "@/lib/logging/background"; import { resolveModuleMap } from "@/lib/modules/gate"; @@ -524,10 +525,14 @@ export async function runWorkoutInsightGenerate( export async function handleWorkoutInsightGenerate( jobs: Job[], -): Promise { - await withBackgroundEvent("job.workout_insight_generate", async (evt) => { +): Promise { + return withBackgroundEvent("job.workout_insight_generate", async (evt) => { + let generated = 0; + let skipped = 0; for (const job of jobs) { const outcome = await runWorkoutInsightGenerate(job.data); + if (outcome.status === "skipped") skipped++; + else generated++; evt.addMeta( "workout_insight", outcome.status === "skipped" @@ -535,5 +540,9 @@ export async function handleWorkoutInsightGenerate( : `generated:${outcome.providerType}`, ); } + // Every gate in front of the provider returns `skipped`, so a pass that + // wrote no paragraph at all is still a pass that ran. Only a transient + // fault escapes `runWorkoutInsightGenerate`, and that one throws. + return jobDone({ total: jobs.length, generated, skipped }); }); }