merge: s1-jobspine into release/v1.34.0

This commit is contained in:
Marc-André Bombeck committed 2026-07-27 23:41:33 +02:00
commit 924babe2ea
69 files changed
+2300 -532

No files matched your search

@@ -0,0 +1,67 @@
/**
* Unit tests for the `healthlog/job-handler-outcome` rule — the bare
* `boss.work(` ban, the one exempt module that owns `createAndWork`, the
* test-file exemption, and the scope limit to `src/lib/jobs/`.
*
* The `invalid` cases below also stand as the rule's own documented blind
* spots: an aliased or renamed receiver is not detected, and neither is a
* handler that returns `{ ok: true }` after swallowing. Those sit in `valid`
* on purpose, so a future reader can see what this rule does not claim.
*/
import { describe, it } from "vitest";
import { RuleTester } from "eslint";
import rule from "../job-handler-outcome.js";
RuleTester.describe = describe;
RuleTester.it = it;
RuleTester.itOnly = it.only;
const ruleTester = new RuleTester({
languageOptions: { ecmaVersion: 2023, sourceType: "module" },
});
ruleTester.run("job-handler-outcome", rule, {
valid: [
{
code: "await createAndWork(boss, QUEUE, { localConcurrency: 1 }, handleX);",
filename: "/repo/src/lib/jobs/reminder/register-status.ts",
},
{
// The wrapper's own call — this module is the one legal caller.
code: "await boss.work(queue, options, runJob(queue, handler));",
filename: "/repo/src/lib/jobs/reminder/registrar-shared.ts",
},
{
// Tests mock `boss.work` and assert against it directly.
code: 'boss.work("q", {}, async () => {});',
filename: "/repo/src/lib/jobs/__tests__/whoop-queues.test.ts",
},
{
// Out of scope: not the worker tree.
code: 'boss.work("q", {}, async () => {});',
filename: "/repo/src/lib/integrations/status.ts",
},
{
// Documented blind spot: an aliased receiver is not detected.
code: 'const b = boss; b.work("q", {}, async () => {});',
filename: "/repo/src/lib/jobs/reminder/register-rollup.ts",
},
{
// Documented blind spot: this rule cannot see a swallowed failure.
code: "await createAndWork(boss, QUEUE, {}, async () => ({ ok: true, did: {} }));",
filename: "/repo/src/lib/jobs/reminder/register-rollup.ts",
},
],
invalid: [
{
code: 'await boss.work("scratch", async () => {});',
filename: "/repo/src/lib/jobs/reminder/register-maintenance.ts",
errors: [{ messageId: "bareBossWork" }],
},
{
code: "await boss.work(QUEUE, { localConcurrency: 1 }, handleX);",
filename: "/repo/src/lib/jobs/worker-status.ts",
errors: [{ messageId: "bareBossWork" }],
},
],
});
+4
View File
@@ -8,6 +8,8 @@
* - `no-raw-palette-color` — ban raw Tailwind palette utilities in app UI.
* - `spacing-scale` — no pt-/pb- overrides on gap-based Card slots,
* and no off-scale `5` step on a bg-card shell.
* - `job-handler-outcome` — pg-boss bindings route through createAndWork,
* whose handler returns a JobOutcome.
*/
"use strict";
@@ -17,6 +19,7 @@ const safeFetchRequired = require("./safe-fetch-required.js");
const apiFetchRequired = require("./api-fetch-required.js");
const noRawPaletteColor = require("./no-raw-palette-color.js");
const spacingScale = require("./spacing-scale.js");
const jobHandlerOutcome = require("./job-handler-outcome.js");
module.exports = {
rules: {
@@ -30,5 +33,6 @@ module.exports = {
// named rule (now error-level too; the staged warn phase ended with
// the semantic sweep). See the rule header.
"no-dracula-utility": noRawPaletteColor,
"job-handler-outcome": jobHandlerOutcome,
},
};
@@ -0,0 +1,107 @@
/**
* @fileoverview ESLint rule — a pg-boss queue binding must go through
* `createAndWork`, never a bare `boss.work(`.
*
* `boss.work(queue, options, handler)` completes a job whenever the handler's
* promise resolves, whatever it resolves to. Every binding in this tree used
* to be written that way, so a handler that caught an error, logged a warning
* and returned looked exactly like one that did its work. `createAndWork`
* (`src/lib/jobs/reminder/registrar-shared.ts`) types its handler as
* `(jobs) => Promise<JobOutcome>` and wraps it in `runJob`, which fails the
* pg-boss job on `ok: false`. Routing every binding through it is what makes
* "the handler returned nothing" a compile error rather than a silent pass.
*
* WHAT THIS RULE PROVES, AND WHAT IT DOES NOT
*
* It proves that no queue in this tree is bound by a path that accepts a
* void handler. That is all it proves. It does NOT detect a handler that
* returns `{ ok: true }` after swallowing an exception — that compiles, it
* passes this rule, and it is exactly as invisible as the defect the outcome
* type was written for. The rule converts an omission (forgetting to signal
* failure) into a commission (claiming a success that did not happen). The
* second is a smaller class and a reviewable one; it is not zero.
*
* Two further evasions are known and unhandled: aliasing the method
* (`const w = boss.work.bind(boss)`, `const { work } = boss`) and calling
* `work` on an object not named `boss`. Both are syntactic dodges a reviewer
* would see in the diff; neither is defended against here.
*
* Scope: `src/lib/jobs/`, excluding the registrar module that owns the one
* legal `boss.work` call and excluding test files, which mock `boss.work`
* and assert against it directly.
*
* @see src/lib/jobs/run-job.ts
* @see src/lib/jobs/job-outcome.ts
* @see src/lib/jobs/reminder/registrar-shared.ts
*/
"use strict";
// The one module allowed to call `boss.work` — `createAndWork` lives here
// and is the wrapper every other binding has to use.
const EXEMPT_FILES = ["src/lib/jobs/reminder/registrar-shared.ts"];
// The worker tree. A `boss.work` outside it would not be a queue binding.
const ENFORCED_ROOTS = ["src/lib/jobs/"];
function toPosix(filename) {
return filename.replace(/\\/g, "/");
}
function isTestFile(posix) {
return (
/\.test\.[cm]?[jt]sx?$/.test(posix) ||
/\.spec\.[cm]?[jt]sx?$/.test(posix) ||
posix.includes("/__tests__/") ||
posix.includes("/__mocks__/")
);
}
function isEnforced(filename) {
const posix = toPosix(filename);
if (!ENFORCED_ROOTS.some((root) => posix.includes(root))) return false;
if (EXEMPT_FILES.some((f) => posix.includes(f))) return false;
if (isTestFile(posix)) return false;
return true;
}
/** @type {import("eslint").Rule.RuleModule} */
const jobHandlerOutcomeRule = {
meta: {
type: "problem",
docs: {
description:
"pg-boss queue bindings must go through createAndWork so the handler returns a JobOutcome.",
},
schema: [],
messages: {
bareBossWork:
'boss.work() completes a job whenever the handler resolves, so a swallowed failure reads as success. Bind the queue with createAndWork(boss, QUEUE, options, handler) from "./registrar-shared" — its handler type is (jobs) => Promise<JobOutcome> and runJob fails the job on ok:false.',
},
},
create(context) {
const filename = context.filename ?? context.getFilename?.();
if (!filename || !isEnforced(filename)) {
return {};
}
return {
CallExpression(node) {
const callee = node.callee;
if (
callee.type !== "MemberExpression" ||
callee.computed ||
callee.property.type !== "Identifier" ||
callee.property.name !== "work" ||
callee.object.type !== "Identifier" ||
callee.object.name !== "boss"
) {
return;
}
context.report({ node, messageId: "bareBossWork" });
},
};
},
};
module.exports = jobHandlerOutcomeRule;
+7
View File
@@ -75,6 +75,13 @@ const eslintConfig = defineConfig([
// trip it. Density lives on the Card itself (`gap-2 py-3 md:py-4`) or on
// a swept `p-4 md:p-6` shell. See the rule header.
"healthlog/spacing-scale": "error",
// Every pg-boss queue binding under src/lib/jobs goes through
// `createAndWork`, whose handler type is `(jobs) => Promise<JobOutcome>`.
// A bare `boss.work(` accepts a handler that resolves to nothing, which
// is how a swallowed failure came to read as a completed job. The rule
// catches the omission only — a handler that returns `{ ok: true }`
// after swallowing still passes, and its header says so.
"healthlog/job-handler-outcome": "error",
},
},
// v1.28.17 — every recharts-rendering component funnels through the
@@ -73,12 +73,16 @@ describe("periodic reconcile — wiring", () => {
/const schedules[\s\S]*?=\s*\[([\s\S]*?)\];/,
);
expect(schedules).not.toBeNull();
// The third element pins the retry policy. It is load-bearing now that
// the handler reports a failed outcome instead of swallowing: without
// `cronIsTheRetry` a database outage would run the same doomed sweep
// three times per tick, and the next tick is already the retry.
expect(schedules![1]).toMatch(
/\[IMPORT_JOB_RECONCILE_QUEUE,\s*IMPORT_JOB_RECONCILE_CRON\]/,
/\[IMPORT_JOB_RECONCILE_QUEUE,\s*IMPORT_JOB_RECONCILE_CRON,\s*cronIsTheRetry\]/,
);
expect(maintenanceSource).toMatch(
/boss\.work\(\s*IMPORT_JOB_RECONCILE_QUEUE[\s\S]{0,120}handleImportJobReconcileTick/,
/createAndWork\(\s*boss,\s*IMPORT_JOB_RECONCILE_QUEUE[\s\S]{0,120}handleImportJobReconcileTick/,
);
});
});
@@ -163,24 +167,30 @@ describe("reconcileOrphanImportJobs", () => {
});
describe("handleImportJobReconcileTick", () => {
it("delegates to reconcileOrphanImportJobs and resolves on success", async () => {
it("delegates to reconcileOrphanImportJobs and reports a done outcome", async () => {
mocks.findMany.mockResolvedValue([]);
await expect(
handleImportJobReconcileTick([] as never),
).resolves.toBeUndefined();
await expect(handleImportJobReconcileTick([] as never)).resolves.toEqual({
ok: true,
did: {},
});
expect(mocks.findMany).toHaveBeenCalledTimes(1);
});
it("swallows a reconcile failure instead of throwing", async () => {
mocks.findMany.mockRejectedValue(new Error("db unavailable"));
it("reports a failed outcome when the reconcile pass throws", async () => {
const cause = new Error("db unavailable");
mocks.findMany.mockRejectedValue(cause);
// A background sweep this cheap must never spam pg-boss's
// retry/backoff machinery — it just re-runs on the next 15-minute
// tick. The handler must resolve even when the underlying reconcile
// pass throws.
await expect(
handleImportJobReconcileTick([] as never),
).resolves.toBeUndefined();
// The handler used to swallow this and resolve, which left a sweep
// that had not run for hours indistinguishable from one that found
// nothing to do. It now names the failure, `runJob` rethrows it, and
// pg-boss records a failed job. The queue carries `cronIsTheRetry`
// (retryLimit 0) because the next 15-minute tick is the retry — a
// database that is down stays down for an immediate re-run.
await expect(handleImportJobReconcileTick([] as never)).resolves.toEqual({
ok: false,
reason: "apple health import reconcile failed",
cause,
});
});
});
@@ -87,11 +87,14 @@ describe("apple health import — parser revision boundary", () => {
expect(allQueues).not.toBeNull();
expect(allQueues![1]).toMatch(/\bAPPLE_HEALTH_IMPORT_V2_QUEUE\b/);
expect(allQueues![1]).toMatch(/\bAPPLE_HEALTH_IMPORT_LEGACY_QUEUE\b/);
// The v2 window is the wider of the two: its binding wraps the batch in
// a loop that keeps the first failed outcome, so the handler call sits
// further from the queue name than the legacy bridge's does.
expect(maintenanceSource).toMatch(
/boss\.work[\s\S]{0,200}APPLE_HEALTH_IMPORT_V2_QUEUE[\s\S]{0,400}handleAppleHealthImport/,
/createAndWork[\s\S]{0,200}APPLE_HEALTH_IMPORT_V2_QUEUE[\s\S]{0,600}handleAppleHealthImport/,
);
expect(maintenanceSource).toMatch(
/boss\.work[\s\S]{0,200}APPLE_HEALTH_IMPORT_LEGACY_QUEUE[\s\S]{0,400}migrateLegacyAppleHealthImport/,
/createAndWork[\s\S]{0,200}APPLE_HEALTH_IMPORT_LEGACY_QUEUE[\s\S]{0,400}migrateLegacyAppleHealthImport/,
);
});
@@ -4,7 +4,7 @@
*
* Same source-text-grep approach as the period-narrative / stress-strain
* guards: assert the queue is imported, registered in `allQueues`, and wired
* to a `boss.work` handler that runs the refresh pipeline — without booting
* to a `createAndWork` handler that runs the refresh pipeline — without booting
* pg-boss + Prisma. An unregistered queue silently never runs (the recurring
* past bug this guards against; v1.4.37 W10 caught exactly this class).
*/
@@ -41,9 +41,9 @@ describe("reminder-worker — coach-memory-refresh wiring", () => {
expect(allQueuesBlock()).toMatch(/\bCOACH_MEMORY_REFRESH_QUEUE\b/);
});
it("registers a boss.work handler that runs the refresh pipeline", () => {
it("registers a createAndWork handler that runs the refresh pipeline", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}COACH_MEMORY_REFRESH_QUEUE[\s\S]{0,400}runCoachMemoryRefresh/,
/createAndWork[\s\S]{0,200}COACH_MEMORY_REFRESH_QUEUE[\s\S]{0,600}runCoachMemoryRefresh/,
);
});
});
@@ -2,7 +2,7 @@
* v1.18.7 — coach-message-cleanup queue registration guard.
*
* Same source-text-grep approach as the other queue-wiring guards: assert the
* queue is registered in `allQueues`, scheduled, and wired to a `boss.work`
* queue is registered in `allQueues`, scheduled, and wired to a `createAndWork`
* handler — without booting pg-boss + Prisma. An unregistered queue silently
* never drains (the recurring v1.4.37 dead-queue bug), which would leave the
* encrypted Coach history growing unbounded with the retention job dark.
@@ -41,13 +41,13 @@ describe("reminder-worker — coach-message-cleanup wiring", () => {
it("schedules the coach-message-cleanup cron", () => {
expect(workerSource).toMatch(
/\[COACH_MESSAGE_CLEANUP_QUEUE,\s*COACH_MESSAGE_CLEANUP_CRON\]/,
/\[COACH_MESSAGE_CLEANUP_QUEUE,\s*COACH_MESSAGE_CLEANUP_CRON,\s*cronIsTheRetry\]/,
);
});
it("registers a boss.work handler for the queue", () => {
it("registers a createAndWork handler for the queue", () => {
expect(workerSource).toMatch(
/boss\.work[\s\S]{0,200}COACH_MESSAGE_CLEANUP_QUEUE[\s\S]{0,200}handleCoachMessageCleanup/,
/createAndWork[\s\S]{0,200}COACH_MESSAGE_CLEANUP_QUEUE[\s\S]{0,200}handleCoachMessageCleanup/,
);
});
@@ -3,7 +3,7 @@
* registration guard.
*
* Same source-text-grep approach as the step-consolidation-repair guard:
* assert the queue is registered in `allQueues`, a `boss.work` handler is
* assert the queue is registered in `allQueues`, a `createAndWork` handler is
* wired against it, and the boot-discovery enqueue helper fires — without
* booting pg-boss + Prisma. An unregistered queue silently never drains
* (the v1.4.37 dead-queue class).
@@ -34,9 +34,9 @@ describe("reminder-worker — cumulative-pr-rederive wiring", () => {
expect(allQueuesMatch![1]).toMatch(/\bCUMULATIVE_PR_REDERIVE_QUEUE\b/);
});
it("registers a boss.work handler against the queue", () => {
it("registers a createAndWork handler against the queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}CUMULATIVE_PR_REDERIVE_QUEUE[\s\S]{0,400}runCumulativePrRederivationForUser/,
/createAndWork[\s\S]{0,200}CUMULATIVE_PR_REDERIVE_QUEUE[\s\S]{0,400}runCumulativePrRederivationForUser/,
);
});
@@ -5,7 +5,7 @@
* consolidation registration guards: assert the cycle-reminder queue is
* declared, listed in `allQueues` (the v1.4.37 dead-queue lesson — an
* unregistered queue is never provisioned and the schedule silently
* no-ops), scheduled on its cron, and wired to a `boss.work` handler that
* no-ops), scheduled on its cron, and wired to a `createAndWork` handler that
* delegates to `runCycleReminderTick`. No pg-boss / Prisma boot.
*/
import { readFileSync } from "node:fs";
@@ -57,9 +57,9 @@ describe("reminder-worker — cycle-reminder wiring", () => {
expect(source).toMatch(/\[CYCLE_REMINDER_QUEUE,\s*CYCLE_REMINDER_CRON\]/);
});
it("registers a boss.work handler that runs the cycle-reminder tick", () => {
it("registers a createAndWork handler that runs the cycle-reminder tick", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}CYCLE_REMINDER_QUEUE[\s\S]{0,200}handleCycleReminderCheck/,
/createAndWork[\s\S]{0,200}CYCLE_REMINDER_QUEUE[\s\S]{0,200}handleCycleReminderCheck/,
);
expect(source).toMatch(
/handleCycleReminderCheck[\s\S]{0,400}runCycleReminderTick/,
@@ -98,9 +98,9 @@ describe("reminder-worker — daily-briefing wiring", () => {
);
});
it("wires a boss.work handler that runs the tick", () => {
it("wires a createAndWork handler that runs the tick", () => {
expect(registrar).toMatch(
/boss\.work[\s\S]{0,200}DAILY_BRIEFING_QUEUE[\s\S]{0,400}runDailyBriefingTick/,
/createAndWork[\s\S]{0,200}DAILY_BRIEFING_QUEUE[\s\S]{0,400}runDailyBriefingTick/,
);
});
@@ -39,9 +39,9 @@ describe("data-arrival queue wiring", () => {
expect(allQueues![1]).toMatch(/\bDATA_ARRIVAL_QUEUE\b/);
});
it("binds a boss.work handler that drains it", () => {
it("binds a createAndWork handler that drains it", () => {
expect(statusRegistrar).toMatch(
/boss\.work[\s\S]{0,200}DATA_ARRIVAL_QUEUE[\s\S]{0,200}handleDataArrival/,
/createAndWork[\s\S]{0,200}DATA_ARRIVAL_QUEUE[\s\S]{0,200}handleDataArrival/,
);
});
@@ -61,7 +61,11 @@ describe("data-arrival queue wiring", () => {
});
it("has no cron schedule — it is send-only, driven by ingest", () => {
expect(statusRegistrar).not.toMatch(/\[DATA_ARRIVAL_QUEUE,\s*\w+_CRON\]/);
// Tolerates the optional third schedule element, so a cron sneaking in
// with send options still trips this.
expect(statusRegistrar).not.toMatch(
/\[\s*DATA_ARRIVAL_QUEUE,\s*\w+_CRON\s*[,\]]/,
);
});
it("registers the retention job that prunes the markers", () => {
@@ -71,10 +75,10 @@ describe("data-arrival queue wiring", () => {
expect(allQueues).not.toBeNull();
expect(allQueues![1]).toMatch(/\bARRIVAL_REACTION_CLEANUP_QUEUE\b/);
expect(maintenanceSource).toMatch(
/\[ARRIVAL_REACTION_CLEANUP_QUEUE,\s*ARRIVAL_REACTION_CLEANUP_CRON\]/,
/\[\s*ARRIVAL_REACTION_CLEANUP_QUEUE,\s*ARRIVAL_REACTION_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/,
);
expect(maintenanceSource).toMatch(
/boss\.work[\s\S]{0,200}ARRIVAL_REACTION_CLEANUP_QUEUE[\s\S]{0,200}handleArrivalReactionCleanup/,
/createAndWork[\s\S]{0,200}ARRIVAL_REACTION_CLEANUP_QUEUE[\s\S]{0,200}handleArrivalReactionCleanup/,
);
});
@@ -50,15 +50,15 @@ describe("reminder-worker — drainPerSampleCumulative nightly schedule", () =>
expect(source).toMatch(/\bDRAIN_CUMULATIVE_CUTOFF_HOURS\b/);
});
it("registers a boss.work handler against the drain queue", () => {
it("registers a createAndWork handler against the drain queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,400}drainPerSampleCumulative/,
/createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,900}drainPerSampleCumulative/,
);
});
it("schedules the drain cron via boss.schedule (allQueues + schedules)", () => {
expect(source).toMatch(
/\[DRAIN_CUMULATIVE_QUEUE,\s*DRAIN_CUMULATIVE_CRON\]/,
/\[DRAIN_CUMULATIVE_QUEUE,\s*DRAIN_CUMULATIVE_CRON,\s*cronIsTheRetry\]/,
);
});
+6 -6
View File
@@ -58,22 +58,22 @@ describe("reminder-worker — Fitbit sync queues", () => {
it("schedules the hourly poll cron + the daily oauth-state sweep", () => {
expect(source).toMatch(/\[FITBIT_SYNC_QUEUE,\s*FITBIT_SYNC_CRON\]/);
expect(source).toMatch(
/\[FITBIT_OAUTH_STATE_CLEANUP_QUEUE,\s*FITBIT_OAUTH_STATE_CLEANUP_CRON\]/,
/\[\s*FITBIT_OAUTH_STATE_CLEANUP_QUEUE,\s*FITBIT_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/,
);
});
it("registers a boss.work handler for every Fitbit queue", () => {
it("registers a createAndWork handler for every Fitbit queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,160}FITBIT_SYNC_QUEUE[\s\S]{0,160}handleFitbitSync/,
/createAndWork[\s\S]{0,160}FITBIT_SYNC_QUEUE[\s\S]{0,160}handleFitbitSync/,
);
expect(source).toMatch(
/boss\.work[\s\S]{0,200}FITBIT_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/,
/createAndWork[\s\S]{0,200}FITBIT_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/,
);
expect(source).toMatch(
/boss\.work[\s\S]{0,200}FITBIT_SLEEP_REPAIR_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/,
/createAndWork[\s\S]{0,200}FITBIT_SLEEP_REPAIR_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/,
);
expect(source).toMatch(
/boss\.work[\s\S]{0,160}FITBIT_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,160}handleFitbitOAuthStateCleanup/,
/createAndWork[\s\S]{0,160}FITBIT_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,160}handleFitbitOAuthStateCleanup/,
);
});
@@ -61,19 +61,19 @@ describe("reminder-worker — Google Health sync queues", () => {
/\[GOOGLE_HEALTH_SYNC_QUEUE,\s*GOOGLE_HEALTH_SYNC_CRON\]/,
);
expect(source).toMatch(
/\[\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE,\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON,?\s*\]/,
/\[\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE,\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/,
);
});
it("registers a boss.work handler for every Google Health queue", () => {
it("registers a createAndWork handler for every Google Health queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}GOOGLE_HEALTH_SYNC_QUEUE[\s\S]{0,200}handleGoogleHealthSync/,
/createAndWork[\s\S]{0,200}GOOGLE_HEALTH_SYNC_QUEUE[\s\S]{0,200}handleGoogleHealthSync/,
);
expect(source).toMatch(
/boss\.work[\s\S]{0,320}GOOGLE_HEALTH_BACKFILL_QUEUE[\s\S]{0,320}enqueueIntegrationBackfillAdmission/,
/createAndWork[\s\S]{0,320}GOOGLE_HEALTH_BACKFILL_QUEUE[\s\S]{0,320}enqueueIntegrationBackfillAdmission/,
);
expect(source).toMatch(
/boss\.work[\s\S]{0,200}GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,200}handleGoogleHealthOAuthStateCleanup/,
/createAndWork[\s\S]{0,200}GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,200}handleGoogleHealthOAuthStateCleanup/,
);
});
@@ -1181,9 +1181,9 @@ describe("queue registration", () => {
);
});
it("registers a boss.work handler for the queue", () => {
it("registers a createAndWork handler for the queue", () => {
expect(workerSrc).toMatch(
/boss\.work[\s\S]{0,80}INSIGHT_PREGENERATE_QUEUE/,
/createAndWork[\s\S]{0,80}INSIGHT_PREGENERATE_QUEUE/,
);
});
@@ -6,7 +6,7 @@
* - an unknown metric is annotated and skipped, not thrown;
* - the enqueue helper no-ops without a global boss instance and de-dupes
* with a singletonKey when one is present;
* - the queue is registered in `allQueues` AND wired to a `boss.work`
* - the queue is registered in `allQueues` AND wired to a `createAndWork`
* handler in reminder-worker.ts (the v1.4.37 W10 unregistered-queue
* catch — a queue built but never registered silently never drains).
*/
@@ -171,9 +171,9 @@ describe("queue registration", () => {
expect(match![1]).toMatch(/\bINSIGHT_STATUS_GENERATE_QUEUE\b/);
});
it("registers a boss.work handler for the queue", () => {
it("registers a createAndWork handler for the queue", () => {
expect(workerSrc).toMatch(
/boss\.work[\s\S]{0,120}INSIGHT_STATUS_GENERATE_QUEUE/,
/createAndWork[\s\S]{0,120}INSIGHT_STATUS_GENERATE_QUEUE/,
);
});
@@ -3,7 +3,7 @@
*
* Same source-text-grep approach as the drain + step-consolidation +
* mean-consolidation regression guards: assert the queue is registered
* in `allQueues`, a `boss.work` handler is wired against it, and the
* in `allQueues`, a `createAndWork` handler is wired against it, and the
* boot-discovery enqueue helper is called — without booting pg-boss +
* Prisma. An unregistered queue silently never drains (the past incident
* this guard exists to prevent).
@@ -42,12 +42,12 @@ describe("reminder-worker — intake-slot-dedup wiring", () => {
expect(allQueuesMatch![1]).toMatch(/\bINTAKE_SLOT_DEDUP_QUEUE\b/);
});
it("registers a boss.work handler against the intake-slot-dedup queue", () => {
it("registers a createAndWork handler against the intake-slot-dedup queue", () => {
// v1.15.19 — span widened from {0,400}: the handler gained the daily
// cron-tick dispatch branch (no `userId` → discovery fan-out) between
// the queue name and the per-user dedup call.
expect(source).toMatch(
/boss\.work[\s\S]{0,200}INTAKE_SLOT_DEDUP_QUEUE[\s\S]{0,1200}dedupeUserIntakeSlots/,
/createAndWork[\s\S]{0,200}INTAKE_SLOT_DEDUP_QUEUE[\s\S]{0,1400}dedupeUserIntakeSlots/,
);
});
@@ -2,7 +2,7 @@
* MCP Phase 3 (M2) — mcp-token-cleanup queue registration guard.
*
* Source-text-grep guard like the other queue-wiring checks: assert the queue
* is in `allQueues`, scheduled, and wired to a `boss.work` handler. An
* is in `allQueues`, scheduled, and wired to a `createAndWork` handler. An
* unregistered queue silently never drains (the recurring v1.4.37 dead-queue
* bug), which would leave expired MCP connector access tokens growing unbounded.
*/
@@ -40,13 +40,13 @@ describe("reminder-worker — mcp-token-cleanup wiring", () => {
it("schedules the mcp-token-cleanup cron", () => {
expect(workerSource).toMatch(
/\[MCP_TOKEN_CLEANUP_QUEUE,\s*MCP_TOKEN_CLEANUP_CRON\]/,
/\[MCP_TOKEN_CLEANUP_QUEUE,\s*MCP_TOKEN_CLEANUP_CRON,\s*cronIsTheRetry\]/,
);
});
it("registers a boss.work handler for the queue", () => {
it("registers a createAndWork handler for the queue", () => {
expect(workerSource).toMatch(
/boss\.work[\s\S]{0,200}MCP_TOKEN_CLEANUP_QUEUE[\s\S]{0,200}handleMcpTokenCleanup/,
/createAndWork[\s\S]{0,200}MCP_TOKEN_CLEANUP_QUEUE[\s\S]{0,200}handleMcpTokenCleanup/,
);
});
@@ -3,7 +3,7 @@
*
* Same source-text-grep approach as the drain + step-consolidation
* regression guards: assert the queue is registered in `allQueues`, a
* `boss.work` handler is wired against it, and the boot-discovery
* `createAndWork` handler is wired against it, and the boot-discovery
* enqueue helper is called — without booting pg-boss + Prisma. An
* unregistered queue silently never drains.
*/
@@ -34,9 +34,9 @@ describe("reminder-worker — mean-consolidation wiring", () => {
expect(allQueuesMatch![1]).toMatch(/\bMEAN_CONSOLIDATION_QUEUE\b/);
});
it("registers a boss.work handler against the mean-consolidation queue", () => {
it("registers a createAndWork handler against the mean-consolidation queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}MEAN_CONSOLIDATION_QUEUE[\s\S]{0,400}runMeanConsolidationForUser/,
/createAndWork[\s\S]{0,200}MEAN_CONSOLIDATION_QUEUE[\s\S]{0,400}runMeanConsolidationForUser/,
);
});
@@ -5,7 +5,7 @@
* guards: assert the Vorsorge-reminder queue is declared, listed in
* `allQueues` (the v1.4.37 dead-queue lesson — an unregistered queue is
* never provisioned and the schedule silently no-ops), scheduled on its
* cron, and wired to a `boss.work` handler that delegates to
* cron, and wired to a `createAndWork` handler that delegates to
* `runMeasurementReminderTick`. No pg-boss / Prisma boot.
*/
import { readFileSync } from "node:fs";
@@ -56,9 +56,9 @@ describe("reminder-worker — measurement-reminder wiring", () => {
);
});
it("registers a boss.work handler that runs the tick", () => {
it("registers a createAndWork handler that runs the tick", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}MEASUREMENT_REMINDER_QUEUE[\s\S]{0,200}handleMeasurementReminderCheck/,
/createAndWork[\s\S]{0,200}MEASUREMENT_REMINDER_QUEUE[\s\S]{0,200}handleMeasurementReminderCheck/,
);
expect(source).toMatch(
/handleMeasurementReminderCheck[\s\S]{0,400}runMeasurementReminderTick/,
@@ -80,12 +80,12 @@ describe("reminder-worker — eventful reminder-satisfy wiring (v1.18.1)", () =>
expect(allQueuesMatch![1]).toMatch(/\bREMINDER_SATISFY_QUEUE\b/);
});
it("binds a boss.work handler for the satisfy queue", () => {
it("binds a createAndWork handler for the satisfy queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}REMINDER_SATISFY_QUEUE[\s\S]{0,200}handleReminderSatisfy/,
/createAndWork[\s\S]{0,200}REMINDER_SATISFY_QUEUE[\s\S]{0,200}handleReminderSatisfy/,
);
expect(source).toMatch(
/handleReminderSatisfy[\s\S]{0,600}runReminderSatisfyForUser/,
/handleReminderSatisfy[\s\S]{0,800}runReminderSatisfyForUser/,
);
});
});
@@ -7,7 +7,7 @@
* recently-tombstoned rows untouched.
* 2. Source-grep wiring (same approach as the drain-cumulative guard):
* the queue is registered in `allQueues`, scheduled, and bound to a
* `boss.work` handler — a missing `allQueues` entry silently no-ops
* `createAndWork` handler — a missing `allQueues` entry silently no-ops
* the schedule under pg-boss v12.
*/
import { readFileSync } from "node:fs";
@@ -114,9 +114,9 @@ describe("reminder-worker — measurement-tombstone-cleanup schedule", () => {
);
});
it("binds a boss.work handler to the queue", () => {
it("binds a createAndWork handler to the queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE[\s\S]{0,200}handleMeasurementTombstoneCleanup/,
/createAndWork[\s\S]{0,200}MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE[\s\S]{0,200}handleMeasurementTombstoneCleanup/,
);
});
});
@@ -50,9 +50,9 @@ describe("reminder-worker — Nightscout sync queue", () => {
expect(source).toMatch(/\[NIGHTSCOUT_SYNC_QUEUE,\s*NIGHTSCOUT_SYNC_CRON\]/);
});
it("registers a boss.work handler for the queue", () => {
it("registers a createAndWork handler for the queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,160}NIGHTSCOUT_SYNC_QUEUE[\s\S]{0,160}handleNightscoutSync/,
/createAndWork[\s\S]{0,160}NIGHTSCOUT_SYNC_QUEUE[\s\S]{0,160}handleNightscoutSync/,
);
});
});
@@ -85,12 +85,12 @@ describe("reminder-worker — Polar / Oura sync queues", () => {
expect(source).toMatch(/\[OURA_SYNC_QUEUE,\s*OURA_SYNC_CRON\]/);
});
it("registers a boss.work handler for each queue", () => {
it("registers a createAndWork handler for each queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,160}POLAR_SYNC_QUEUE[\s\S]{0,160}handlePolarSync/,
/createAndWork[\s\S]{0,160}POLAR_SYNC_QUEUE[\s\S]{0,160}handlePolarSync/,
);
expect(source).toMatch(
/boss\.work[\s\S]{0,160}OURA_SYNC_QUEUE[\s\S]{0,160}handleOuraSync/,
/createAndWork[\s\S]{0,160}OURA_SYNC_QUEUE[\s\S]{0,160}handleOuraSync/,
);
});
});
@@ -196,8 +196,10 @@ describe("queue registration", () => {
);
});
it("registers a boss.work handler for the queue", () => {
expect(workerSrc).toMatch(/boss\.work[\s\S]{0,120}PERIOD_NARRATIVE_QUEUE/);
it("registers a createAndWork handler for the queue", () => {
expect(workerSrc).toMatch(
/createAndWork[\s\S]{0,120}PERIOD_NARRATIVE_QUEUE/,
);
});
it("exposes a sane queue name + nightly cron", () => {
@@ -3,7 +3,7 @@
*
* Same source-text-grep approach as the other queue-wiring guards: assert
* the queue is imported, registered in `allQueues`, scheduled, and wired to
* a `boss.work` handler — without booting pg-boss + Prisma. An unregistered
* a `createAndWork` handler — without booting pg-boss + Prisma. An unregistered
* queue silently never drains (the recurring past bug this guards against).
*
* Also pins the iOS-write rejection of the server-owned COMPUTED source on
@@ -57,9 +57,9 @@ describe("reminder-worker — recovery-score wiring", () => {
);
});
it("registers a boss.work handler that runs the recovery-score pass", () => {
it("registers a createAndWork handler that runs the recovery-score pass", () => {
expect(workerSource).toMatch(
/boss\.work[\s\S]{0,200}RECOVERY_SCORE_QUEUE[\s\S]{0,400}runRecoveryScore/,
/createAndWork[\s\S]{0,200}RECOVERY_SCORE_QUEUE[\s\S]{0,400}runRecoveryScore/,
);
});
});
+112
View File
@@ -0,0 +1,112 @@
/**
* Unit tests for the job-outcome spine.
*
* The whole point of `runJob` is that an `ok: false` outcome reaches pg-boss
* as a failed job. That is a behaviour claim, not a typing one, and it is the
* claim the retention cleanups now depend on — so it is pinned here: the
* failure is reported once with the handler's own facts, it rethrows, and a
* handler that throws on its own is passed through untouched rather than
* rewrapped into a stack that points at this file.
*/
import { describe, it, expect, vi, beforeEach } from "vitest";
import type { Job } from "pg-boss";
const reportWorkerError = vi.fn();
vi.mock("../report-worker-error", () => ({
reportWorkerError: (...a: unknown[]) => reportWorkerError(...a),
}));
import { jobDone, jobFailed } from "../job-outcome";
import { JobFailure, runJob } from "../run-job";
const noJobs: Job<object>[] = [];
beforeEach(() => {
vi.clearAllMocks();
reportWorkerError.mockResolvedValue(undefined);
});
describe("runJob", () => {
it("resolves and reports nothing when the handler says it did the work", async () => {
const wrapped = runJob("rate-limit-cleanup", async () =>
jobDone({ deleted: 12 }),
);
await expect(wrapped(noJobs)).resolves.toBeUndefined();
expect(reportWorkerError).not.toHaveBeenCalled();
});
it("treats a run with nothing to do as a success", async () => {
const wrapped = runJob("mood-reminder-cleanup", async () =>
jobDone({ deleted: 0 }),
);
await expect(wrapped(noJobs)).resolves.toBeUndefined();
expect(reportWorkerError).not.toHaveBeenCalled();
});
it("rethrows a failed outcome so pg-boss records a failed job", async () => {
const cause = new Error("statement timeout");
const wrapped = runJob("push-attempt-cleanup", async () =>
jobFailed("push-attempt cleanup failed", cause, { deleted: 0 }),
);
await expect(wrapped(noJobs)).rejects.toBeInstanceOf(JobFailure);
await expect(wrapped(noJobs)).rejects.toThrow(
"push-attempt-cleanup: push-attempt cleanup failed",
);
});
it("reports the failure with the handler's own facts and the cause", async () => {
const cause = new Error("statement timeout");
const wrapped = runJob("push-attempt-cleanup", async () =>
jobFailed("push-attempt cleanup failed", cause, { deleted: 0 }),
);
await expect(wrapped(noJobs)).rejects.toThrow();
expect(reportWorkerError).toHaveBeenCalledTimes(1);
const [queue, error, meta] = reportWorkerError.mock.calls[0];
expect(queue).toBe("push-attempt-cleanup");
expect(error).toBeInstanceOf(JobFailure);
expect(meta).toEqual({ deleted: 0, cause: "statement timeout" });
});
it("carries the thrown error's stack instead of its own frame", async () => {
const cause = new Error("statement timeout");
cause.stack = "Error: statement timeout\n at deleteMany (prisma.ts:1:1)";
const wrapped = runJob("mcp-token-cleanup", async () =>
jobFailed("mcp-token cleanup failed", cause),
);
const failure = await wrapped(noJobs).catch((err: unknown) => err);
expect(failure).toBeInstanceOf(JobFailure);
expect((failure as JobFailure).stack).toContain("at deleteMany");
expect((failure as JobFailure).cause).toBe(cause);
});
it("passes a throwing handler through untouched", async () => {
const thrown = new Error("cohort discovery failed");
const wrapped = runJob("withings-sync", async () => {
throw thrown;
});
await expect(wrapped(noJobs)).rejects.toBe(thrown);
// The throw already fails the job. Reporting it here would double every
// handler that reports for itself, and rewrapping would bury the stack.
expect(reportWorkerError).not.toHaveBeenCalled();
});
it("hands the batch to the handler unchanged", async () => {
const handler = vi.fn(async () => jobDone({ jobs: 2 }));
const jobs = [
{ id: "a", data: { userId: "u1" } },
{ id: "b", data: { userId: "u2" } },
] as unknown as Job<{ userId: string }>[];
await runJob("data-arrival", handler)(jobs);
expect(handler).toHaveBeenCalledWith(jobs);
});
});
@@ -3,7 +3,7 @@
*
* The reminder-worker imports heavy infrastructure, so we read the source as
* text and assert the queue constant is declared, registered in `allQueues`,
* wired to a `boss.work` handler, and enqueued at boot — the fast guard that
* wired to a `createAndWork` handler, and enqueued at boot — the fast guard that
* catches the v1.4.37 dead-queue class (a queue declared but never registered,
* which silently never drains).
*/
@@ -30,9 +30,9 @@ describe("reminder-worker — sleep-timeline backfill queue", () => {
expect(block![1]!).toContain("SLEEP_TIMELINE_BACKFILL_QUEUE");
});
it("wires a boss.work handler through the shared admission queue", () => {
it("wires a createAndWork handler through the shared admission queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,260}SLEEP_TIMELINE_BACKFILL_QUEUE[\s\S]{0,300}enqueueIntegrationBackfillAdmission/,
/createAndWork[\s\S]{0,260}SLEEP_TIMELINE_BACKFILL_QUEUE[\s\S]{0,300}enqueueIntegrationBackfillAdmission/,
);
});
@@ -196,8 +196,8 @@ describe("mood-status queue registration (dead-queue guard)", () => {
expect(workerSrc).toMatch(/MOOD_STATUS_CRON\s*=\s*"30 2 \* \* \*"/);
});
it("registers a boss.work handler for the queue", () => {
expect(workerSrc).toMatch(/boss\.work[\s\S]{0,120}MOOD_STATUS_QUEUE/);
it("registers a createAndWork handler for the queue", () => {
expect(workerSrc).toMatch(/createAndWork[\s\S]{0,120}MOOD_STATUS_QUEUE/);
});
it("drives all seven status crons through the shared discovery", () => {
@@ -4,7 +4,7 @@
*
* Same source-text-grep approach as the drain + step-consolidation +
* mean-consolidation guards: assert the repair queue is registered in
* `allQueues`, a `boss.work` handler is wired against it, and the
* `allQueues`, a `createAndWork` handler is wired against it, and the
* boot-discovery enqueue helper fires — without booting pg-boss + Prisma.
* An unregistered queue silently never drains (the dead-queue class).
*/
@@ -36,9 +36,9 @@ describe("reminder-worker — step-consolidation-repair wiring", () => {
expect(allQueuesMatch![1]).toMatch(/\bSTEP_CONSOLIDATION_REPAIR_QUEUE\b/);
});
it("registers a boss.work handler against the repair queue", () => {
it("registers a createAndWork handler against the repair queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}STEP_CONSOLIDATION_REPAIR_QUEUE[\s\S]{0,400}runStepConsolidationRepairForUser/,
/createAndWork[\s\S]{0,200}STEP_CONSOLIDATION_REPAIR_QUEUE[\s\S]{0,400}runStepConsolidationRepairForUser/,
);
});
@@ -5,7 +5,7 @@
*
* Same source-text-grep approach as the WX-C recovery-score guard: assert
* each queue is imported, registered in `allQueues`, scheduled (the two
* cron-driven score jobs), and wired to a `boss.work` handler — without
* cron-driven score jobs), and wired to a `createAndWork` handler — without
* booting pg-boss + Prisma. An unregistered queue silently never drains
* (the recurring past bug this guards against).
*/
@@ -67,9 +67,9 @@ describe("reminder-worker — stress-score wiring", () => {
expect(source).toMatch(/\[STRESS_SCORE_QUEUE,\s*STRESS_SCORE_CRON\]/);
});
it("registers a boss.work handler that runs the stress-score pass", () => {
it("registers a createAndWork handler that runs the stress-score pass", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}STRESS_SCORE_QUEUE[\s\S]{0,400}runStressScore/,
/createAndWork[\s\S]{0,200}STRESS_SCORE_QUEUE[\s\S]{0,400}runStressScore/,
);
});
});
@@ -90,9 +90,9 @@ describe("reminder-worker — strain-score wiring", () => {
expect(source).toMatch(/\[STRAIN_SCORE_QUEUE,\s*STRAIN_SCORE_CRON\]/);
});
it("registers a boss.work handler that runs the strain-score pass", () => {
it("registers a createAndWork handler that runs the strain-score pass", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,200}STRAIN_SCORE_QUEUE[\s\S]{0,400}runStrainScore/,
/createAndWork[\s\S]{0,200}STRAIN_SCORE_QUEUE[\s\S]{0,400}runStrainScore/,
);
});
});
@@ -111,9 +111,9 @@ describe("reminder-worker — dense intra-day retention wiring", () => {
expect(allQueuesBlock()).toMatch(/\bDENSE_INTRADAY_RETENTION_QUEUE\b/);
});
it("registers a boss.work handler against the dense-intraday-retention queue", () => {
it("registers a createAndWork handler against the dense-intraday-retention queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,250}DENSE_INTRADAY_RETENTION_QUEUE[\s\S]{0,500}runDenseIntradayRetentionForUser/,
/createAndWork[\s\S]{0,250}DENSE_INTRADAY_RETENTION_QUEUE[\s\S]{0,500}runDenseIntradayRetentionForUser/,
);
});
@@ -163,9 +163,9 @@ describe("reminder-worker — dense hourly-rebuild wiring", () => {
expect(allQueuesBlock()).toMatch(/\bDENSE_INTRADAY_HOURLY_REBUILD_QUEUE\b/);
});
it("registers a boss.work handler against the hourly-rebuild queue", () => {
it("registers a createAndWork handler against the hourly-rebuild queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,250}DENSE_INTRADAY_HOURLY_REBUILD_QUEUE[\s\S]{0,500}runDenseIntradayHourlyRebuildForUser/,
/createAndWork[\s\S]{0,250}DENSE_INTRADAY_HOURLY_REBUILD_QUEUE[\s\S]{0,500}runDenseIntradayHourlyRebuildForUser/,
);
});
@@ -3,7 +3,7 @@
*
* Same source-text-grep approach as the other queue-wiring guards: assert
* the queue is imported, registered in `allQueues`, scheduled, and wired to
* a `boss.work` handler — without booting pg-boss + Prisma. An unregistered
* a `createAndWork` handler — without booting pg-boss + Prisma. An unregistered
* queue silently never drains (the recurring v1.4.37 dead-queue bug), which
* would leave the pinned-leaf-rotation alarm dark.
*/
@@ -50,9 +50,9 @@ describe("reminder-worker — tls-pin-monitor wiring", () => {
);
});
it("registers a boss.work handler for the queue", () => {
it("registers a createAndWork handler for the queue", () => {
expect(workerSource).toMatch(
/boss\.work[\s\S]{0,200}TLS_PIN_MONITOR_QUEUE[\s\S]{0,200}handleTlsPinMonitor/,
/createAndWork[\s\S]{0,200}TLS_PIN_MONITOR_QUEUE[\s\S]{0,200}handleTlsPinMonitor/,
);
});
+7 -7
View File
@@ -77,25 +77,25 @@ describe("reminder-worker — WHOOP sync queues", () => {
/\[WHOOP_CYCLE_SYNC_QUEUE,\s*WHOOP_CYCLE_SYNC_CRON\]/,
);
expect(source).toMatch(
/\[WHOOP_OAUTH_STATE_CLEANUP_QUEUE,\s*WHOOP_OAUTH_STATE_CLEANUP_CRON\]/,
/\[\s*WHOOP_OAUTH_STATE_CLEANUP_QUEUE,\s*WHOOP_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/,
);
});
it("registers a boss.work handler for every WHOOP sync queue", () => {
it("registers a createAndWork handler for every WHOOP sync queue", () => {
expect(source).toMatch(
/boss\.work[\s\S]{0,160}WHOOP_RECOVERY_SYNC_QUEUE[\s\S]{0,160}handleWhoopRecoverySync/,
/createAndWork[\s\S]{0,160}WHOOP_RECOVERY_SYNC_QUEUE[\s\S]{0,160}handleWhoopRecoverySync/,
);
expect(source).toMatch(
/boss\.work[\s\S]{0,160}WHOOP_SLEEP_SYNC_QUEUE[\s\S]{0,160}handleWhoopSleepSync/,
/createAndWork[\s\S]{0,160}WHOOP_SLEEP_SYNC_QUEUE[\s\S]{0,160}handleWhoopSleepSync/,
);
expect(source).toMatch(
/boss\.work[\s\S]{0,160}WHOOP_WORKOUT_SYNC_QUEUE[\s\S]{0,160}handleWhoopWorkoutSync/,
/createAndWork[\s\S]{0,160}WHOOP_WORKOUT_SYNC_QUEUE[\s\S]{0,160}handleWhoopWorkoutSync/,
);
expect(source).toMatch(
/boss\.work[\s\S]{0,160}WHOOP_CYCLE_SYNC_QUEUE[\s\S]{0,160}handleWhoopCycleSync/,
/createAndWork[\s\S]{0,160}WHOOP_CYCLE_SYNC_QUEUE[\s\S]{0,160}handleWhoopCycleSync/,
);
expect(source).toMatch(
/boss\.work[\s\S]{0,200}WHOOP_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/,
/createAndWork[\s\S]{0,200}WHOOP_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/,
);
});
@@ -128,7 +128,18 @@ describe("runWhoopResourceSync dispatch", () => {
findManyMock.mockResolvedValue([{ userId: "a" }, { userId: "b" }]);
syncUserWorkout.mockRejectedValueOnce(new Error("boom for a"));
await expect(handleWhoopWorkoutSync([job({})])).resolves.toBeUndefined();
// The pass is what the outcome judges: the sweep ran, so the job is
// done. The revoked-grant user rides out as `users_failed` rather than
// failing the queue and retrying the whole cohort over one account.
await expect(handleWhoopWorkoutSync([job({})])).resolves.toEqual({
ok: true,
did: {
total: 2,
users_synced: 1,
users_failed: 1,
measurements_imported: 0,
},
});
// Both users were attempted despite the first throwing.
expect(syncUserWorkout).toHaveBeenCalledTimes(2);
});
@@ -93,7 +93,15 @@ describe("handleWithingsEcgSync", () => {
enddate: 1715000060,
});
await expect(handleWithingsEcgSync([queued])).resolves.toBeUndefined();
await expect(handleWithingsEcgSync([queued])).resolves.toEqual({
ok: true,
did: {
total: 1,
users_synced: 1,
users_failed: 0,
recordings_imported: 1,
},
});
expect(ecgFindMany).not.toHaveBeenCalled();
expect(syncUserEcg).toHaveBeenCalledTimes(1);
@@ -120,14 +128,24 @@ describe("handleWithingsEcgSync", () => {
it("one user's failure never starves the rest of the cohort", async () => {
ecgFindMany.mockResolvedValue([{ userId: "a" }, { userId: "b" }]);
syncUserEcg.mockResolvedValue(0);
syncUserEcg.mockRejectedValueOnce(new Error("boom for a"));
// Per-user catch-and-warn (mirrors the activity / sleep handlers): the
// handler resolves, both users attempted, the leaf records its own ledger
// failure so honesty is preserved without rejecting the whole batch.
// pass ran, so the job is done and the failing user rides out as a
// `users_failed` count. The leaf records its own ledger failure, so
// honesty is preserved without rejecting the whole batch.
await expect(
handleWithingsEcgSync([job({} as WithingsEcgSyncPayload)]),
).resolves.toBeUndefined();
).resolves.toEqual({
ok: true,
did: {
total: 2,
users_synced: 1,
users_failed: 1,
recordings_imported: 0,
},
});
expect(syncUserEcg).toHaveBeenCalledTimes(2);
});
});
@@ -143,7 +161,7 @@ describe("withings ECG queue registration", () => {
/\[WITHINGS_ECG_SYNC_QUEUE\]:\s*\{[\s\S]*?policy:\s*"short"/,
);
expect(registrarSource).toMatch(
/boss\.work<WithingsEcgSyncPayload>\([\s\S]*?WITHINGS_ECG_SYNC_QUEUE[\s\S]*?handleWithingsEcgSync/,
/createAndWork<WithingsEcgSyncPayload>\(\s*boss,\s*WITHINGS_ECG_SYNC_QUEUE[\s\S]{0,120}handleWithingsEcgSync/,
);
});
});
@@ -50,14 +50,14 @@ describe("reminder-worker — Withings activity + sleep v2 queues", () => {
it("registers the activity-sync handler against pg-boss", () => {
expect(source).toMatch(/handleWithingsActivitySync\b/);
expect(source).toMatch(
/boss\.work[\s\S]{0,200}WITHINGS_ACTIVITY_QUEUE[\s\S]{0,200}handleWithingsActivitySync/,
/createAndWork[\s\S]{0,200}WITHINGS_ACTIVITY_QUEUE[\s\S]{0,200}handleWithingsActivitySync/,
);
});
it("registers the sleep-sync handler against pg-boss", () => {
expect(source).toMatch(/handleWithingsSleepSync\b/);
expect(source).toMatch(
/boss\.work[\s\S]{0,200}WITHINGS_SLEEP_QUEUE[\s\S]{0,200}handleWithingsSleepSync/,
/createAndWork[\s\S]{0,200}WITHINGS_SLEEP_QUEUE[\s\S]{0,200}handleWithingsSleepSync/,
);
});
@@ -78,7 +78,7 @@ describe("reminder-worker — Withings activity + sleep v2 queues", () => {
});
it("schedules the ECG catch-net cron at :41 so watch-only accounts pull strips", () => {
// The queue is already in allQueues + boss.work-bound; W-1 adds the missing
// The queue is already in allQueues + createAndWork-bound; W-1 adds the missing
// schedule entry so a payload-less cron tick walks the connection cohort.
expect(source).toMatch(
/WITHINGS_ECG_SYNC_CRON\s*=\s*["']41 \* \* \* \*["']/,
@@ -72,7 +72,16 @@ describe("handleWithingsFallbackSync subscription repair", () => {
data: { triggeredAt: "2026-07-21T10:00:00.000Z" },
} as Job<WithingsSyncPayload>;
await handleWithingsFallbackSync([queued]);
await expect(handleWithingsFallbackSync([queued])).resolves.toEqual({
ok: true,
did: {
total: 0,
users_synced: 0,
users_failed: 0,
measurements_imported: 0,
subscription_repair_failed: false,
},
});
expect(retryDueWithingsWebhookSubscriptions).toHaveBeenCalledTimes(1);
expect(syncUserMeasurements).not.toHaveBeenCalled();
@@ -89,7 +98,19 @@ describe("handleWithingsFallbackSync subscription repair", () => {
data: { triggeredAt: "2026-07-21T11:00:00.000Z" },
} as Job<WithingsSyncPayload>;
await expect(handleWithingsFallbackSync([queued])).resolves.toBeUndefined();
// The repair is an auxiliary leg, not the pass this queue is named for:
// the fallback sync still ran for everyone, so the job is done and the
// repair failure rides out as a fact on the outcome.
await expect(handleWithingsFallbackSync([queued])).resolves.toEqual({
ok: true,
did: {
total: 2,
users_synced: 2,
users_failed: 0,
measurements_imported: 0,
subscription_repair_failed: true,
},
});
expect(findMany).toHaveBeenCalledWith({ select: { userId: true } });
expect(syncUserMeasurements).toHaveBeenNthCalledWith(
@@ -32,9 +32,9 @@ describe("workout-insight-generate queue wiring", () => {
expect(allQueues![1]).toMatch(/\bWORKOUT_INSIGHT_GENERATE_QUEUE\b/);
});
it("binds a boss.work handler that drains it", () => {
it("binds a createAndWork handler that drains it", () => {
expect(statusRegistrar).toMatch(
/boss\.work[\s\S]{0,300}WORKOUT_INSIGHT_GENERATE_QUEUE[\s\S]{0,300}handleWorkoutInsightGenerate/,
/createAndWork[\s\S]{0,300}WORKOUT_INSIGHT_GENERATE_QUEUE[\s\S]{0,300}handleWorkoutInsightGenerate/,
);
});
@@ -50,8 +50,10 @@ describe("workout-insight-generate queue wiring", () => {
it("has no cron schedule — it is dispatched by arrival only", () => {
// A cron here would be a standing invitation to regenerate, which is the
// saturation surface the design refuses.
// Tolerates the optional third schedule element, so a cron sneaking in
// with send options still trips this.
expect(statusRegistrar).not.toMatch(
/\[WORKOUT_INSIGHT_GENERATE_QUEUE,\s*\w+_CRON\]/,
/\[\s*WORKOUT_INSIGHT_GENERATE_QUEUE,\s*\w+_CRON\s*[,\]]/,
);
});
+26 -8
View File
@@ -31,6 +31,7 @@ import type { Job } from "pg-boss";
import { extractExportXml } from "@/lib/import/unzip-export-xml";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import {
streamParseExportXml,
type ImportJobProgress,
@@ -66,23 +67,26 @@ export const IMPORT_JOB_RECONCILE_CRON = "*/15 * * * *";
* the same `reconcileOrphanImportJobs()` the boot path calls — the
* heartbeat + live pg-boss-job check already make it safe to re-run
* on a healthy worker (a live import's fresh heartbeat and `active`
* pg-boss state both keep its row untouched). Errors are logged, not
* rethrown — a reconcile miss must never spam pg-boss's retry/backoff
* machinery for a background sweep this cheap to just re-run in 15
* minutes.
* pg-boss state both keep its row untouched). A sweep that throws now
* reports a failed outcome instead of only leaving a warning behind:
* this pass exists to end a silent stall, and a stall in the pass
* itself was the one condition nothing surfaced. The next tick is 15
* minutes away regardless, so nothing rests on the retry.
*/
export async function handleImportJobReconcileTick(
jobs: Job<object>[],
): Promise<void> {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent(
return withBackgroundEvent(
"job.apple_health_import_reconcile",
async (evt) => {
try {
await reconcileOrphanImportJobs();
} catch (err) {
evt.addWarning(`apple-health-import-reconcile failed: ${err}`);
return jobFailed("apple health import reconcile failed", err);
}
return jobDone();
},
);
}
@@ -230,7 +234,7 @@ async function writeProgress(
*/
export async function handleAppleHealthImport(
job: Job<AppleHealthImportPayload>,
): Promise<void> {
): Promise<JobOutcome> {
const { userId, uploadPath, uploadBytes, triggeredByAdminId } = job.data;
const prisma = getWorkerPrisma();
@@ -280,7 +284,7 @@ export async function handleAppleHealthImport(
` — row is already terminal (${importJob.status}); the staged upload` +
" was consumed by the first run and a re-run could only mask its outcome",
);
return;
return jobDone({ skipped: "already_terminal" });
}
// Extracted-XML path, hoisted so the failure path can clean it up —
@@ -364,6 +368,12 @@ export async function handleAppleHealthImport(
// the end, scoped to the user's full measurement span, so
// post-import reads of the analytics + comprehensive surfaces
// hit the warm rollup table on first paint.
//
// The miss rides out as a fact rather than as a failed job: the
// import is committed and its mirror row already reads `done`, so
// failing here would report a completed import as failed and
// contradict the row the status poll surfaces to the operator.
let rollupFailed = false;
try {
const span = await prisma.measurement.aggregate({
where: { userId },
@@ -382,6 +392,7 @@ export async function handleAppleHealthImport(
} catch (rollupErr) {
// Rollup failure is non-fatal — the next read falls through to
// live aggregation. Log but don't poison the import.
rollupFailed = true;
console.warn(
`[apple-health-import] Rollup recompute failed for user ${userId}`,
rollupErr,
@@ -392,6 +403,13 @@ export async function handleAppleHealthImport(
// periodically swept on the host.
safeUnlink(unzip.xmlPath);
safeUnlink(uploadPath);
return jobDone({
records_read: result.totals.recordsRead,
rows_upserted: result.totals.rowsUpserted,
duration_ms: result.totals.durationMs,
rollup_failed: rollupFailed,
});
} catch (err) {
// v1.28.33 (issue #486) — a missing staging file is an operational
// condition, not a parse failure: `/tmp` is wiped on a container
+11 -2
View File
@@ -30,6 +30,7 @@
import type { Job } from "pg-boss";
import { invalidateUserDashboardSnapshot } from "@/lib/cache/invalidate";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { annotate } from "@/lib/logging/context";
import { withBackgroundEvent } from "@/lib/logging/background";
@@ -248,14 +249,17 @@ export async function runDataArrival(
export async function handleDataArrival(
jobs: Job<DataArrival>[],
): Promise<void> {
await withBackgroundEvent("job.data_arrival", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.data_arrival", async (evt) => {
let processed = 0;
let skipped = 0;
for (const job of jobs) {
const arrival = job.data;
try {
const outcome = await runDataArrival(getWorkerPrisma(), arrival);
if (outcome.status === "skipped") {
skipped++;
annotate({
action: { name: `arrival.${arrival.kind}.skipped` },
meta: { reason: outcome.reason, source: arrival.source },
@@ -264,6 +268,7 @@ export async function handleDataArrival(
continue;
}
processed++;
annotate({
action: { name: `arrival.${arrival.kind}.processed` },
meta: {
@@ -285,5 +290,9 @@ export async function handleDataArrival(
throw err;
}
}
// Both statuses are successes — a refused arrival is a decision the spine
// made, not work it failed to do. The counts are what make the split
// between them readable at all.
return jobDone({ total: jobs.length, processed, skipped });
});
}
+9 -2
View File
@@ -11,6 +11,7 @@
*/
import { type Job } from "pg-boss";
import { withBackgroundEvent } from "@/lib/logging/background";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { getWorkerPrisma } from "@/lib/jobs/reminder/shared";
import type { PrismaClient } from "@/generated/prisma/client";
@@ -43,16 +44,22 @@ export async function purgeExpiredDocumentTombstones(
return count;
}
export async function handleDocumentPurge(jobs: Job<DocumentPurgePayload>[]) {
export async function handleDocumentPurge(
jobs: Job<DocumentPurgePayload>[],
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.document_tombstone_purge", async (evt) => {
return withBackgroundEvent("job.document_tombstone_purge", async (evt) => {
const prisma = getWorkerPrisma();
try {
const purged = await purgeExpiredDocumentTombstones(prisma);
evt.setAction({ name: "documents.vault.purge" });
evt.addMeta("document_purge_deleted", purged);
// A night with no expired tombstone purges nothing and is still a run
// that did its work — the zero is the fact, not an absence of one.
return jobDone({ document_purge_deleted: purged });
} catch (err) {
evt.addWarning(`document-tombstone-purge failed: ${err}`);
return jobFailed("document tombstone purge failed", err);
}
});
}
+12 -2
View File
@@ -24,6 +24,7 @@
*/
import { type Job } from "pg-boss";
import { withBackgroundEvent } from "@/lib/logging/background";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { auditLog } from "@/lib/auth/audit";
import {
rotateCorpus,
@@ -60,8 +61,8 @@ export async function runEncryptionKeyRotation(): Promise<{
export async function handleEncryptionKeyRotate(
jobs: Job<EncryptionKeyRotatePayload>[],
) {
await withBackgroundEvent("job.encryption_key_rotate", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.encryption_key_rotate", async (evt) => {
const requestedBy = jobs[0]?.data?.requestedByUserId ?? null;
try {
const result = await runEncryptionKeyRotation();
@@ -78,6 +79,15 @@ export async function handleEncryptionKeyRotate(
errors: result.totalErrors,
},
});
// Per-row errors are the fail-closed skip of a row under a key the
// deployment no longer configures. The pass still ran, so they ride out
// as a count rather than failing (and retrying) the whole corpus.
return jobDone({
rotate_active_key_id: result.activeKeyId,
rotate_scanned: result.totalScanned,
rotate_rotated: result.totalRotated,
rotate_errors: result.totalErrors,
});
} catch (err) {
evt.addWarning(`encryption-key-rotate failed: ${err}`);
await auditLog("admin.encryption.rotate.failed", {
+14 -4
View File
@@ -33,6 +33,7 @@ import {
utcDayKey,
} from "@/lib/environment/service";
import { recordError } from "@/lib/jobs/worker-status";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { workerLog } from "./reminder/shared";
export const ENVIRONMENT_FETCH_QUEUE = "environment-fetch";
@@ -152,14 +153,14 @@ export async function enqueueEnvironmentFetchDiscovery(
export async function handleEnvironmentFetch(
boss: PgBoss,
payload: EnvironmentFetchPayload,
): Promise<void> {
): Promise<JobOutcome> {
if (!payload.userId) {
const { enqueued, skipped } = await enqueueEnvironmentFetchDiscovery(boss);
workerLog(
"info",
`[environment-fetch] discovery enqueued=${enqueued} skipped=${skipped}`,
);
return;
return jobDone({ users_enqueued: enqueued, users_skipped: skipped });
}
const userId = payload.userId;
@@ -168,7 +169,7 @@ export async function handleEnvironmentFetch(
"info",
`[environment-fetch] user=${userId} module disabled — skipping`,
);
return;
return jobDone({ skipped: "environment module disabled" });
}
const { startDate, endDate } = resolveRange(payload);
@@ -179,7 +180,11 @@ export async function handleEnvironmentFetch(
"error",
`[environment-fetch] user=${userId} range ${startDate}..${endDate} (${span}d) exceeds cap — skipping`,
);
return;
// A refused range is not an opt-out: the job was asked to fill days and
// filled none, so it says so instead of completing quietly on a stderr line.
return jobFailed("environment backfill range exceeds cap", undefined, {
span_days: span,
});
}
try {
@@ -192,6 +197,11 @@ export async function handleEnvironmentFetch(
"info",
`[environment-fetch] user=${userId} ${startDate}..${endDate} stored=${result.stored} skipped=${result.skipped} fetches=${result.fetches}`,
);
return jobDone({
days_stored: result.stored,
days_skipped: result.skipped,
fetches: result.fetches,
});
} catch (err) {
recordError();
workerLog("error", `[environment-fetch] user=${userId} failed`, err);
+81
View File
@@ -0,0 +1,81 @@
/**
* The value a background-job handler returns to say what it did.
*
* Before this type existed, a pg-boss handler signalled success by falling
* off the end of a function. Every one of the queue bindings in this tree
* resolved to `undefined`, and none of them was read anywhere, so a handler
* that caught an error, logged a warning and returned was indistinguishable
* from one that finished its work. pg-boss marked both `completed`; the
* operator surfaces that read `pgboss.job` saw nothing; the nightly pass
* looked healthy for as long as it kept failing.
*
* A returned `JobOutcome` closes that specific gap: completion is now
* derived from a value the handler had to construct, and `runJob` turns
* `ok: false` into a failed pg-boss job. What the type CANNOT do is make a
* handler honest — `jobDone()` after swallowing an exception compiles and
* passes every check in this repository. It converts omission (forgetting to
* signal a failure) into commission (claiming a success that did not
* happen), which is a smaller and much more reviewable class of defect.
*
* A fan-out pass is judged on the pass, not on its legs. A nightly sweep
* over every connected account is `ok: true` when the sweep ran, even if one
* account's grant was revoked: that belongs on the account's own integration
* ledger, and failing the queue would retry the whole cohort over one user's
* expired token. The per-leg failures ride out as a count in `did`, which is
* what makes them visible at all — today they are visible nowhere.
*
* `did` is the small pinned-shape bag of facts about the run — counts,
* identifiers, booleans. It rides into the failure report and, once the
* failure ledger lands, into the operator surface. Keep it to scalars that
* a dashboard can key on; it is not a place for payload data or free text.
*/
/** A single fact about a run. Scalars only — never payload data. */
export type JobFact = string | number | boolean;
/** The pinned-shape bag of facts a handler reports about its run. */
export type JobFacts = Readonly<Record<string, JobFact>>;
/**
* What a handler did. `ok: true` means the work named by the queue actually
* happened; `ok: false` means it did not, and `runJob` will fail the pg-boss
* job so the failure reaches the retry policy and the operator surfaces.
*
* A run that legitimately had nothing to do is `ok: true` with a zero count
* in `did` — absence of work is not failure. A run that could not do its
* work (provider down, query timed out, key missing) is `ok: false` even
* when the handler previously chose to carry on.
*/
export type JobOutcome =
| {
readonly ok: true;
readonly did: JobFacts;
}
| {
readonly ok: false;
/** Short, stable, greppable. `"nightscout fetch failed"`, not a stack. */
readonly reason: string;
/** The thrown value, when there was one. Preserved for the stack. */
readonly cause?: unknown;
/** Whatever the run managed before it failed. */
readonly did?: JobFacts;
};
/** The run did its work. Pass the counts that make the run auditable. */
export function jobDone(did: JobFacts = {}): JobOutcome {
return { ok: true, did };
}
/**
* The run did not do its work. `runJob` reports this and rethrows, so the
* pg-boss job fails and the queue's retry policy applies — check that policy
* when converting a handler that used to swallow, because a deterministic
* failure now retries instead of passing quietly.
*/
export function jobFailed(
reason: string,
cause?: unknown,
did?: JobFacts,
): JobOutcome {
return { ok: false, reason, cause, did };
}
+13 -2
View File
@@ -3,6 +3,7 @@ import { Prisma, type PrismaClient } from "@/generated/prisma/client";
import { invalidateUserMedications } from "@/lib/cache/invalidate";
import { prisma, toJson } from "@/lib/db";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { consumeImportedIntakesBatch } from "@/lib/medications/inventory/consumption";
import { queueMedicationIntakeSync } from "@/lib/notifications/medication-intake-sync";
import {
@@ -711,9 +712,19 @@ async function recordWorkerFailure(
export async function handleMedicationIntakeImport(
job: Job<MedicationIntakeImportQueuePayload>,
): Promise<void> {
): Promise<JobOutcome> {
try {
await processMedicationIntakeImportJob(job.data.jobId);
const result = await processMedicationIntakeImportJob(job.data.jobId);
// A null result is a delivery that found the row already terminal or gone.
// There was nothing left to import, which is an absence of work rather
// than a failure to do it.
return result === null
? jobDone({ finalized: false })
: jobDone({
finalized: true,
imported: result.imported,
skipped: result.skipped,
});
} catch (error) {
await recordWorkerFailure(job, error);
throw error;
+1 -1
View File
@@ -2,7 +2,7 @@
* v1.4.25 W16c — Enqueue helper + handler glue for the pg-boss
* `pr-detection` queue. Lives in `src/lib/jobs/` next to the other
* job-side helpers; the worker process binds the handler in
* `reminder-worker.ts` via `boss.work(PR_DETECTION_QUEUE, ...)`.
* the maintenance registrar via `createAndWork(boss, PR_DETECTION_QUEUE, …)`.
*
* The Next.js side (ingest routes) imports `enqueuePrDetection` and
* never touches pg-boss directly — `getGlobalBoss()` returns the
+11 -2
View File
@@ -29,6 +29,7 @@ import type { Job } from "pg-boss";
import { prisma } from "@/lib/db";
import { randomUUID } from "node:crypto";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { annotate } from "@/lib/logging/context";
import { withBackgroundEvent } from "@/lib/logging/background";
import { resolveProviderChain } from "@/lib/ai/provider";
@@ -677,13 +678,16 @@ export async function runReactionLine(
export async function handleReactionLine(
jobs: Job<ReactionLineJob>[],
): Promise<void> {
await withBackgroundEvent("job.reaction_line", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.reaction_line", async (evt) => {
let generated = 0;
let skipped = 0;
for (const job of jobs) {
try {
const outcome = await runReactionLine(job.data);
if (outcome.status === "skipped") {
skipped++;
annotate({
action: { name: "arrival.reaction_line.skipped" },
meta: { reason: outcome.reason, kind: job.data.kind },
@@ -692,6 +696,7 @@ export async function handleReactionLine(
continue;
}
generated++;
annotate({
action: { name: "arrival.reaction_line.generated" },
meta: { kind: job.data.kind, local_date: job.data.localDate },
@@ -705,5 +710,9 @@ export async function handleReactionLine(
throw err;
}
}
// A skip is a refusal the surface is designed around, not a miss: the
// deterministic lead already stands without a line. The pass did its work
// whenever it ran to the end.
return jobDone({ total: jobs.length, generated, skipped });
});
}
+2 -2
View File
@@ -8,8 +8,8 @@
* the domain registrars under `src/lib/jobs/reminder/`. Each registrar owns the
* four facts the v1.4.37 dead-queue guards pin — the queue-name constant, its
* `allQueues` membership, its `[QUEUE, CRON]` schedule tuple, and its
* `boss.work(QUEUE, …, handler)` binding — so a queue can never be declared
* without being provisioned, scheduled, and drained.
* `createAndWork(boss, QUEUE, …, handler)` binding — so a queue can never be
* declared without being provisioned, scheduled, and drained.
*
* Usage: Run as a standalone process or call startReminderWorker() from a
* custom server setup. In dev, use: npx tsx src/lib/jobs/reminder-worker.ts
@@ -107,7 +107,17 @@ describe("makePollCohortHandler — marker-aware recordFailure boundary", () =>
recordFailure,
});
await expect(handler([job()])).resolves.toBeUndefined();
// The recorder rejection is warned, not propagated: the pass ran, so the
// job is done and the failing user is a `users_failed` count.
await expect(handler([job()])).resolves.toEqual({
ok: true,
did: {
total: 2,
users_synced: 1,
users_failed: 1,
measurements_imported: 1,
},
});
// The next user still synced despite the recorder throwing.
expect(syncUser).toHaveBeenCalledTimes(2);
expect(syncUser).toHaveBeenLastCalledWith("next");
+40 -6
View File
@@ -8,8 +8,12 @@ import { type Job } from "pg-boss";
import { recordError } from "@/lib/jobs/worker-status";
import { buildFullBackupPayload } from "@/lib/export/full-backup-payload";
import { encrypt } from "@/lib/crypto";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import { runOffhostBackup } from "@/lib/jobs/offhost-backup";
import {
OffhostBackupNotConfiguredError,
runOffhostBackup,
} from "@/lib/jobs/offhost-backup";
import { getWorkerPrisma } from "./shared";
export interface DataBackupPayload {
@@ -20,9 +24,11 @@ export interface OffhostBackupPayload {
triggeredAt: string;
}
export async function handleOffhostBackup(jobs: Job<OffhostBackupPayload>[]) {
export async function handleOffhostBackup(
jobs: Job<OffhostBackupPayload>[],
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.offhost_backup", async (evt) => {
return withBackgroundEvent("job.offhost_backup", async (evt) => {
const p = getWorkerPrisma();
try {
const report = await runOffhostBackup(p);
@@ -39,16 +45,34 @@ export async function handleOffhostBackup(jobs: Job<OffhostBackupPayload>[]) {
JSON.stringify(report.failures.slice(0, 10)),
);
}
// Per-user upload failures ride out as `offhost_backup_failed`: the run
// itself uploaded what it could, and failing the queue over one user's
// object would re-upload the whole cohort on every retry.
return jobDone({
offhost_backup_uploaded: report.uploaded,
offhost_backup_failed: report.failed,
offhost_backup_total_users: report.totalUsers,
offhost_backup_endpoint: report.config.endpoint,
offhost_backup_bucket: report.config.bucket,
});
} catch (err) {
// Not configured ⇒ skip silently with a warning, not an error.
// Not configured ⇒ skip silently with a warning, not an error: most
// self-hosts never set the S3 credentials, and a nightly failed job on
// every one of them buries the runs that genuinely could not upload.
evt.addWarning(`offhost-backup skipped/failed: ${err}`);
if (err instanceof OffhostBackupNotConfiguredError) {
return jobDone({ offhost_backup_configured: false });
}
return jobFailed("offhost backup failed", err);
}
});
}
export async function handleDataBackup(jobs: Job<DataBackupPayload>[]) {
export async function handleDataBackup(
jobs: Job<DataBackupPayload>[],
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.data_backup", async (evt) => {
return withBackgroundEvent("job.data_backup", async (evt) => {
const prisma = getWorkerPrisma();
try {
const users = await prisma.user.findMany({
@@ -56,6 +80,7 @@ export async function handleDataBackup(jobs: Job<DataBackupPayload>[]) {
});
let backed = 0;
let usersFailed = 0;
for (const user of users) {
try {
const { payload } = await buildFullBackupPayload(prisma, user.id, {
@@ -82,6 +107,10 @@ export async function handleDataBackup(jobs: Job<DataBackupPayload>[]) {
});
backed++;
} catch (err) {
// One user's payload failing is that user's problem, not the pass's:
// it rides out as a count so the weekly run is not retried for the
// whole cohort.
usersFailed++;
evt.addWarning(`Failed for user ${user.id}: ${err}`);
}
}
@@ -90,6 +119,11 @@ export async function handleDataBackup(jobs: Job<DataBackupPayload>[]) {
task_name: "job.data_backup",
result: { backed, total: users.length },
});
return jobDone({
backed,
total: users.length,
users_failed: usersFailed,
});
} catch (err) {
evt.setError(err);
recordError();
+99 -39
View File
@@ -2,10 +2,27 @@
* Retention and maintenance cleanups: rate-limit rows, idempotency keys, audit logs, OAuth states (Withings / WHOOP), mood-reminder events, push attempts, and measurement tombstones.
*
* Extracted from reminder-worker.ts, which owns the queue names, cron
* schedules, and boss.work registrations.
* schedules, and the queue bindings.
*
* Every pass in here used to catch its own failure, write a warning nobody
* reads, and return — so a retention purge that had not run for weeks was
* indistinguishable from one with nothing to delete. They now return
* `jobFailed`, which fails the pg-boss job and puts the queue on the operator's
* failing-jobs surface.
*
* That change has a price, and the registrars pay it: these are bulk DELETEs
* over a retention horizon, and their failure mode is deterministic. A
* statement that timed out against a large trailing edge times out again
* immediately. Under pg-boss's default policy (retryLimit 2, no delay, no
* backoff) each converted queue would run the same doomed statement three
* times in a row every night. So every cron-driven cleanup registered from
* this module is scheduled with `retryLimit: 0`: the failure is recorded once
* and the next night's tick is the retry, which is the right backoff for
* retention work that has a whole day of slack.
*/
import { type Job } from "pg-boss";
import { withBackgroundEvent } from "@/lib/logging/background";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import {
cleanupExpiredWhoopConnectTickets,
cleanupExpiredWhoopOAuthStates,
@@ -78,9 +95,9 @@ export interface MoodReminderCleanupPayload {
export async function handleMoodReminderCleanup(
jobs: Job<MoodReminderCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.mood_reminder_cleanup", async (evt) => {
return withBackgroundEvent("job.mood_reminder_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const cutoff = new Date();
@@ -90,8 +107,10 @@ export async function handleMoodReminderCleanup(
where: { date: { lt: cutoffIso } },
});
evt.addMeta("mood_reminder_cleanup_deleted", deleted.count);
return jobDone({ deleted: deleted.count });
} catch (err) {
evt.addWarning(`mood-reminder-cleanup failed: ${err}`);
return jobFailed("mood-reminder cleanup failed", err);
}
});
}
@@ -118,9 +137,9 @@ export interface PushAttemptCleanupPayload {
export async function handlePushAttemptCleanup(
jobs: Job<PushAttemptCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.push_attempt_cleanup", async (evt) => {
return withBackgroundEvent("job.push_attempt_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const cutoff = new Date();
@@ -129,8 +148,10 @@ export async function handlePushAttemptCleanup(
where: { createdAt: { lt: cutoff } },
});
evt.addMeta("push_attempt_cleanup_deleted", deleted.count);
return jobDone({ deleted: deleted.count });
} catch (err) {
evt.addWarning(`push-attempt-cleanup failed: ${err}`);
return jobFailed("push-attempt cleanup failed", err);
}
});
}
@@ -152,9 +173,9 @@ export interface ArrivalReactionCleanupPayload {
export async function handleArrivalReactionCleanup(
jobs: Job<ArrivalReactionCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.arrival_reaction_cleanup", async (evt) => {
return withBackgroundEvent("job.arrival_reaction_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const cutoff = new Date();
@@ -163,8 +184,10 @@ export async function handleArrivalReactionCleanup(
where: { createdAt: { lt: cutoff } },
});
evt.addMeta("arrival_reaction_cleanup_deleted", deleted.count);
return jobDone({ deleted: deleted.count });
} catch (err) {
evt.addWarning(`arrival-reaction-cleanup failed: ${err}`);
return jobFailed("arrival-reaction cleanup failed", err);
}
});
}
@@ -175,9 +198,9 @@ export interface MeasurementTombstoneCleanupPayload {
export async function handleMeasurementTombstoneCleanup(
jobs: Job<MeasurementTombstoneCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent(
return withBackgroundEvent(
"job.measurement_tombstone_cleanup",
async (evt) => {
const p = getWorkerPrisma();
@@ -208,6 +231,12 @@ export async function handleMeasurementTombstoneCleanup(
"tombstone-cleanup stopped at the batch cap; a backlog remains for the next run",
);
}
return jobDone({
measurements_pruned: measurements.deleted,
mood_pruned: mood.deleted,
intakes_pruned: intakes.deleted,
drained,
});
} catch (err) {
// Rethrow. A retention purge that fails silently every night looks
// exactly like one that has nothing to do; pg-boss recording a failed
@@ -221,32 +250,36 @@ export async function handleMeasurementTombstoneCleanup(
export async function handleRateLimitCleanup(
jobs: Job<RateLimitCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.rate_limit_cleanup", async (evt) => {
return withBackgroundEvent("job.rate_limit_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const result = await p.$executeRaw`
DELETE FROM rate_limits WHERE reset_at < NOW()
`;
evt.addMeta("rate_limit_cleanup_deleted", result);
return jobDone({ deleted: result });
} catch (err) {
evt.addWarning(`rate-limit-cleanup failed: ${err}`);
return jobFailed("rate-limit cleanup failed", err);
}
});
}
export async function handleIdempotencyCleanup(
jobs: Job<IdempotencyCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.idempotency_cleanup", async (evt) => {
return withBackgroundEvent("job.idempotency_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const deleted = await cleanupExpiredIdempotencyKeys(p);
evt.addMeta("idempotency_cleanup_deleted", deleted);
return jobDone({ deleted });
} catch (err) {
evt.addWarning(`idempotency-cleanup failed: ${err}`);
return jobFailed("idempotency-key cleanup failed", err);
}
});
}
@@ -266,26 +299,28 @@ export async function handleIdempotencyCleanup(
*/
export async function handleStepUpElevationCleanup(
jobs: Job<StepUpElevationCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.step_up_elevation_cleanup", async (evt) => {
return withBackgroundEvent("job.step_up_elevation_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const { count } = await p.stepUpElevation.deleteMany({
where: { expiresAt: { lt: new Date() } },
});
evt.addMeta("step_up_elevation_cleanup_deleted", count);
return jobDone({ deleted: count });
} catch (err) {
evt.addWarning(`step-up-elevation-cleanup failed: ${err}`);
return jobFailed("step-up elevation cleanup failed", err);
}
});
}
export async function handleAuditLogCleanup(
jobs: Job<AuditLogCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.audit_log_cleanup", async (evt) => {
return withBackgroundEvent("job.audit_log_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const outcome = await cleanupOldAuditLogs(p);
@@ -296,6 +331,7 @@ export async function handleAuditLogCleanup(
"audit-log-cleanup stopped at the batch cap; a backlog remains for the next run",
);
}
return jobDone({ deleted: outcome.deleted, drained: outcome.drained });
} catch (err) {
// Rethrow — see the tombstone handler. Storage-limitation retention that
// fails quietly is retention that is not happening.
@@ -307,15 +343,17 @@ export async function handleAuditLogCleanup(
export async function handleCoachMessageCleanup(
jobs: Job<CoachMessageCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.coach_message_cleanup", async (evt) => {
return withBackgroundEvent("job.coach_message_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const deleted = await cleanupOldCoachMessages(p);
evt.addMeta("coach_message_cleanup_deleted", deleted);
return jobDone({ deleted });
} catch (err) {
evt.addWarning(`coach-message-cleanup failed: ${err}`);
return jobFailed("coach-message cleanup failed", err);
}
});
}
@@ -326,9 +364,9 @@ export interface McpTokenCleanupPayload {
export async function handleMcpTokenCleanup(
jobs: Job<McpTokenCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.mcp_token_cleanup", async (evt) => {
return withBackgroundEvent("job.mcp_token_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const result = await cleanupExpiredMcpTokens(p);
@@ -340,27 +378,40 @@ export async function handleMcpTokenCleanup(
"mcp_token_cleanup_connections_deleted",
result.connectionsDeleted,
);
return jobDone({
access_tokens_deleted: result.accessTokensDeleted,
connections_deleted: result.connectionsDeleted,
});
} catch (err) {
evt.addWarning(`mcp-token-cleanup failed: ${err}`);
return jobFailed("mcp-token cleanup failed", err);
}
});
}
export async function handleWithingsOAuthStateCleanup(
jobs: Job<WithingsOAuthStateCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.withings_oauth_state_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const deleted = await cleanupExpiredWithingsOAuthStates(p);
evt.addMeta("withings_oauth_state_cleanup_deleted", deleted);
} catch (err) {
// The OAuth flow tolerates a stale row sticking around for an
// extra day — log + carry on so the boss queue doesn't retry-loop.
evt.addWarning(`withings-oauth-state-cleanup failed: ${err}`);
}
});
return withBackgroundEvent(
"job.withings_oauth_state_cleanup",
async (evt) => {
const p = getWorkerPrisma();
try {
const deleted = await cleanupExpiredWithingsOAuthStates(p);
evt.addMeta("withings_oauth_state_cleanup_deleted", deleted);
return jobDone({ deleted });
} catch (err) {
// The OAuth flow tolerates a stale row for an extra day, which used
// to be the argument for swallowing this so the queue would not
// retry-loop. Tolerable is not the same as invisible: the failure is
// now reported, and the retry-loop worry is answered where it belongs,
// by the `retryLimit: 0` this queue is scheduled with.
evt.addWarning(`withings-oauth-state-cleanup failed: ${err}`);
return jobFailed("withings oauth-state cleanup failed", err);
}
},
);
}
export interface OidcNativeHandoffCleanupPayload {
@@ -369,17 +420,21 @@ export interface OidcNativeHandoffCleanupPayload {
export async function handleOidcNativeHandoffCleanup(
jobs: Job<OidcNativeHandoffCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.oidc_native_handoff_cleanup", async (evt) => {
return withBackgroundEvent("job.oidc_native_handoff_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const deleted = await cleanupExpiredOidcNativeHandoffs(p);
evt.addMeta("oidc_native_handoff_cleanup_deleted", deleted);
return jobDone({ deleted });
} catch (err) {
// The handoff flow tolerates a stale row for an extra day (expiry is
// enforced at read) — log + carry on so the boss queue doesn't retry-loop.
// Expiry is enforced at read, so a stale row for an extra day is
// harmless — but see the withings twin above: harmless is not a
// reason to hide the failure. `retryLimit: 0` on the schedule is what
// keeps a doomed statement from running three times a night.
evt.addWarning(`oidc-native-handoff-cleanup failed: ${err}`);
return jobFailed("oidc native-handoff cleanup failed", err);
}
});
}
@@ -390,17 +445,22 @@ export interface WhoopOAuthStateCleanupPayload {
export async function handleWhoopOAuthStateCleanup(
jobs: Job<WhoopOAuthStateCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.whoop_oauth_state_cleanup", async (evt) => {
return withBackgroundEvent("job.whoop_oauth_state_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const deleted = await cleanupExpiredWhoopOAuthStates(p);
evt.addMeta("whoop_oauth_state_cleanup_deleted", deleted);
const ticketsDeleted = await cleanupExpiredWhoopConnectTickets(p);
evt.addMeta("whoop_connect_ticket_cleanup_deleted", ticketsDeleted);
return jobDone({
states_deleted: deleted,
connect_tickets_deleted: ticketsDeleted,
});
} catch (err) {
evt.addWarning(`whoop-oauth-state-cleanup failed: ${err}`);
return jobFailed("whoop oauth-state cleanup failed", err);
}
});
}
+30 -7
View File
@@ -7,6 +7,7 @@
import { type Job } from "pg-boss";
import { fireAndForget } from "@/lib/logging/fire-and-forget";
import { recordError } from "@/lib/jobs/worker-status";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import { runFitbitPollCohort } from "@/lib/fitbit/sync";
import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy";
@@ -23,8 +24,10 @@ export interface FitbitSyncPayload {
userId?: string;
}
export async function handleFitbitSync(jobs: Job<FitbitSyncPayload>[]) {
await withBackgroundEvent("job.fitbit_sync", async (evt) => {
export async function handleFitbitSync(
jobs: Job<FitbitSyncPayload>[],
): Promise<JobOutcome> {
return withBackgroundEvent("job.fitbit_sync", async (evt) => {
const prisma = getWorkerPrisma();
try {
const targets: Array<{ userId: string }> = [];
@@ -37,16 +40,26 @@ export async function handleFitbitSync(jobs: Job<FitbitSyncPayload>[]) {
});
targets.push(...connections);
}
if (targets.length === 0) return;
if (targets.length === 0) {
return jobDone({
total: 0,
users_synced: 0,
users_failed: 0,
measurements_imported: 0,
});
}
// Fan the cohort out with bounded concurrency + per-user error isolation:
// one slow Google response can't stall the whole cohort, and a single
// user's failure is warned without aborting the pass.
let usersFailed = 0;
const { usersSynced, measurementsImported } = await runFitbitPollCohort(
targets.map((t) => t.userId),
{
onUserError: (userId, err) =>
evt.addWarning(`job.fitbit_sync failed for user ${userId}: ${err}`),
onUserError: (userId, err) => {
usersFailed++;
evt.addWarning(`job.fitbit_sync failed for user ${userId}: ${err}`);
},
// v1.18.1 — a fresh reading landed; resolve the user's Vorsorge
// reminders eventfully. Fire-and-forget; the cron is the net.
onUserSynced: (userId, imported) => {
@@ -67,6 +80,14 @@ export async function handleFitbitSync(jobs: Job<FitbitSyncPayload>[]) {
measurements_imported: measurementsImported,
},
});
// The pass is the unit of success: a revoked grant belongs on that user's
// integration ledger, not in a queue failure that retries every account.
return jobDone({
total: targets.length,
users_synced: usersSynced,
users_failed: usersFailed,
measurements_imported: measurementsImported,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -81,15 +102,17 @@ export interface FitbitOAuthStateCleanupPayload {
export async function handleFitbitOAuthStateCleanup(
jobs: Job<FitbitOAuthStateCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.fitbit_oauth_state_cleanup", async (evt) => {
return withBackgroundEvent("job.fitbit_oauth_state_cleanup", async (evt) => {
const p = getWorkerPrisma();
try {
const deleted = await cleanupExpiredFitbitOAuthStates(p);
evt.addMeta("fitbit_oauth_state_cleanup_deleted", deleted);
return jobDone({ deleted });
} catch (err) {
evt.addWarning(`fitbit-oauth-state-cleanup failed: ${err}`);
return jobFailed("fitbit oauth state cleanup failed", err);
}
});
}
+28 -7
View File
@@ -7,6 +7,7 @@
import { type Job } from "pg-boss";
import { fireAndForget } from "@/lib/logging/fire-and-forget";
import { recordError } from "@/lib/jobs/worker-status";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import { runGoogleHealthPollCohort } from "@/lib/google-health/sync";
import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy";
@@ -26,8 +27,8 @@ export interface GoogleHealthSyncPayload {
export async function handleGoogleHealthSync(
jobs: Job<GoogleHealthSyncPayload>[],
) {
await withBackgroundEvent("job.google_health_sync", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.google_health_sync", async (evt) => {
const prisma = getWorkerPrisma();
try {
const targets: Array<{ userId: string }> = [];
@@ -40,19 +41,29 @@ export async function handleGoogleHealthSync(
});
targets.push(...connections);
}
if (targets.length === 0) return;
if (targets.length === 0) {
return jobDone({
total: 0,
users_synced: 0,
users_failed: 0,
measurements_imported: 0,
});
}
// Fan the cohort out with bounded concurrency + per-user error isolation:
// one slow Google response can't stall the whole cohort, and a single
// user's failure is warned without aborting the pass.
let usersFailed = 0;
const { usersSynced, measurementsImported } =
await runGoogleHealthPollCohort(
targets.map((t) => t.userId),
{
onUserError: (userId, err) =>
onUserError: (userId, err) => {
usersFailed++;
evt.addWarning(
`job.google_health_sync failed for user ${userId}: ${err}`,
),
);
},
// A fresh reading landed; resolve the user's Vorsorge reminders
// eventfully. Fire-and-forget; the cron is the net.
onUserSynced: (userId, imported) => {
@@ -73,6 +84,14 @@ export async function handleGoogleHealthSync(
measurements_imported: measurementsImported,
},
});
// The pass is the unit of success: a revoked grant belongs on that user's
// integration ledger, not in a queue failure that retries every account.
return jobDone({
total: targets.length,
users_synced: usersSynced,
users_failed: usersFailed,
measurements_imported: measurementsImported,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -87,17 +106,19 @@ export interface GoogleHealthOAuthStateCleanupPayload {
export async function handleGoogleHealthOAuthStateCleanup(
jobs: Job<GoogleHealthOAuthStateCleanupPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent(
return withBackgroundEvent(
"job.google_health_oauth_state_cleanup",
async (evt) => {
const p = getWorkerPrisma();
try {
const deleted = await cleanupExpiredGoogleHealthOAuthStates(p);
evt.addMeta("google_health_oauth_state_cleanup_deleted", deleted);
return jobDone({ deleted });
} catch (err) {
evt.addWarning(`google-health-oauth-state-cleanup failed: ${err}`);
return jobFailed("google health oauth state cleanup failed", err);
}
},
);
+69 -19
View File
@@ -19,6 +19,7 @@ import {
type InsightStatusGeneratePayload,
runInsightStatusGenerate,
} from "@/lib/jobs/insight-status-generate";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import { generateBloodPressureStatusForUser } from "@/lib/insights/blood-pressure-status";
import { generateWeightStatusForUser } from "@/lib/insights/weight-status";
@@ -69,6 +70,10 @@ export interface MedicationComplianceStatusPayload {
* `status-cron-candidates.ts` for the full division of nightly labour).
* The generators normalise `locale` themselves (de stays de, everything
* else gets English prose).
*
* The pass is what succeeds or fails. One user whose generation threw is that
* user's cold card, reported as the `failed` count; failing the job would
* re-run the whole cohort over it.
*/
export async function runStatusCronGenerate(
taskName: string,
@@ -76,13 +81,15 @@ export async function runStatusCronGenerate(
userId: string,
options: { locale: string | null; force: boolean },
) => Promise<unknown>,
): Promise<void> {
await withBackgroundEvent(taskName, async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent(taskName, async (evt) => {
const prisma = getWorkerPrisma();
try {
const users = await findStatusCronCandidates(prisma);
if (users.length === 0) return;
if (users.length === 0) {
return jobDone({ total: 0, generated: 0, failed: 0 });
}
let generated = 0;
let failed = 0;
@@ -103,6 +110,7 @@ export async function runStatusCronGenerate(
task_name: taskName,
result: { generated, failed, total: users.length },
});
return jobDone({ total: users.length, generated, failed });
} catch (err) {
evt.setError(err);
recordError();
@@ -129,13 +137,15 @@ export async function runStatusCronGenerate(
* inside the batch itself). Net effect: the nightly ladder pays one call per
* user instead of seven, with no queue/registry churn.
*/
async function runStatusBatchCron(taskName: string): Promise<void> {
await withBackgroundEvent(taskName, async (evt) => {
async function runStatusBatchCron(taskName: string): Promise<JobOutcome> {
return withBackgroundEvent(taskName, async (evt) => {
const prisma = getWorkerPrisma();
try {
recordInsightsRun();
const users = await findStatusCronCandidates(prisma);
if (users.length === 0) return;
if (users.length === 0) {
return jobDone({ total: 0, generated: 0, served: 0, failed: 0 });
}
let generated = 0;
let served = 0;
@@ -167,6 +177,10 @@ async function runStatusBatchCron(taskName: string): Promise<void> {
task_name: taskName,
result: { generated, served, failed, total: users.length },
});
// A user whose batch call threw already fell back to the single-card
// path inside the batch, and the six later per-metric crons cover what
// is still cold. The batch pass ran, so it reports the count.
return jobDone({ total: users.length, generated, served, failed });
} catch (err) {
evt.setError(err);
recordError();
@@ -176,14 +190,16 @@ async function runStatusBatchCron(taskName: string): Promise<void> {
});
}
export function handleGeneralStatusGenerate(jobs: Job<GeneralStatusPayload>[]) {
export function handleGeneralStatusGenerate(
jobs: Job<GeneralStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusBatchCron("job.insights.batch");
}
export function handleBloodPressureStatusGenerate(
jobs: Job<BloodPressureStatusPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate(
"job.insights.blood_pressure",
@@ -191,7 +207,9 @@ export function handleBloodPressureStatusGenerate(
);
}
export function handleWeightStatusGenerate(jobs: Job<WeightStatusPayload>[]) {
export function handleWeightStatusGenerate(
jobs: Job<WeightStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate(
"job.insights.weight",
@@ -199,7 +217,9 @@ export function handleWeightStatusGenerate(jobs: Job<WeightStatusPayload>[]) {
);
}
export function handlePulseStatusGenerate(jobs: Job<PulseStatusPayload>[]) {
export function handlePulseStatusGenerate(
jobs: Job<PulseStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate(
"job.insights.pulse",
@@ -207,19 +227,23 @@ export function handlePulseStatusGenerate(jobs: Job<PulseStatusPayload>[]) {
);
}
export function handleBmiStatusGenerate(jobs: Job<BmiStatusPayload>[]) {
export function handleBmiStatusGenerate(
jobs: Job<BmiStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate("job.insights.bmi", generateBmiStatusForUser);
}
export function handleMoodStatusGenerate(jobs: Job<MoodStatusPayload>[]) {
export function handleMoodStatusGenerate(
jobs: Job<MoodStatusPayload>[],
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate("job.insights.mood", generateMoodStatusForUser);
}
export function handleMedicationComplianceStatusGenerate(
jobs: Job<MedicationComplianceStatusPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
return runStatusCronGenerate(
"job.insights.medication_compliance",
@@ -229,8 +253,8 @@ export function handleMedicationComplianceStatusGenerate(
export async function handleInsightPregenerateJob(
jobs: Job<InsightPregeneratePayload>[],
) {
await withBackgroundEvent("job.insight_pregenerate", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.insight_pregenerate", async (evt) => {
// v1.8.7.1 — a forced single-user warm carries `{ userId, force }`;
// the scheduled tick carries neither. Route each job individually so a
// batch that mixes a cron tick with on-demand warms (it never does in
@@ -265,13 +289,30 @@ export async function handleInsightPregenerateJob(
}
}
if (scheduled.length === 0) return;
// A batch of forced warms only is a complete run: every warm either landed
// or threw out of the loop above.
if (scheduled.length === 0) {
return jobDone({ forced: forced.length, scheduled: 0 });
}
try {
const summary = await runInsightPregenerate(getWorkerPrisma());
evt.setBackground({
task_name: "job.insight_pregenerate",
result: { ...summary },
});
return jobDone({
forced: forced.length,
scheduled: scheduled.length,
total: summary.total,
generated: summary.generated,
cached: summary.cached,
unchanged: summary.unchanged,
skipped: summary.skipped,
failed: summary.failed,
budget_blocked: summary.budgetBlocked,
assessments_warmed: summary.assessmentsWarmed,
metric_assessments_warmed: summary.metricAssessmentsWarmed,
});
} catch (err) {
evt.addWarning(
`insight-pregenerate failed: ${
@@ -296,12 +337,18 @@ export async function handleInsightPregenerateJob(
*/
export async function handleInsightStatusGenerate(
jobs: Job<InsightStatusGeneratePayload>[],
) {
await withBackgroundEvent("job.insight_status_generate", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.insight_status_generate", async (evt) => {
let generated = 0;
let malformed = 0;
for (const job of jobs) {
if (!job.data?.userId || !job.data?.metric) continue;
if (!job.data?.userId || !job.data?.metric) {
malformed++;
continue;
}
try {
await runInsightStatusGenerate(job.data);
generated++;
evt.addMeta(
"status_generated",
`${job.data.metric}:${job.data.locale}`,
@@ -321,5 +368,8 @@ export async function handleInsightStatusGenerate(
throw err;
}
}
// Every job in the batch either generated a card or carried no metric to
// generate; a generation that failed left through the rethrow above.
return jobDone({ generated, malformed });
});
}
@@ -10,6 +10,7 @@ import {
type IntakeAutoSkipPayload,
runIntakeAutoSkipPass,
} from "@/lib/jobs/intake-auto-skip";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import { expireStaleInUseItems } from "@/lib/medications/inventory/service";
import { getWorkerPrisma } from "./shared";
@@ -21,18 +22,21 @@ import { getWorkerPrisma } from "./shared";
*/
export async function handleMedicationInventoryExpire(
jobs: Job<MedicationInventoryExpirePayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.medication_inventory_expire", async (evt) => {
return withBackgroundEvent("job.medication_inventory_expire", async (evt) => {
try {
const count = await expireStaleInUseItems({ nowMs: Date.now() });
evt.addMeta("inventory_expired_count", count);
// A pass that found no lapsed pen did its work; zero is a result.
return jobDone({ inventory_expired_count: count });
} catch (err) {
evt.addWarning(
`medication-inventory-expire failed: ${
err instanceof Error ? err.message : String(err)
}`,
);
return jobFailed("inventory expire pass failed", err);
}
});
}
@@ -46,9 +50,11 @@ export async function handleMedicationInventoryExpire(
* unit test can drive it with an in-memory fake Prisma; this wrapper
* threads the worker's pg-boss + background-event plumbing.
*/
export async function handleIntakeAutoSkip(jobs: Job<IntakeAutoSkipPayload>[]) {
export async function handleIntakeAutoSkip(
jobs: Job<IntakeAutoSkipPayload>[],
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.intake_auto_skip", async (evt) => {
return withBackgroundEvent("job.intake_auto_skip", async (evt) => {
const prisma = getWorkerPrisma();
try {
const result = await runIntakeAutoSkipPass(prisma, {
@@ -56,12 +62,17 @@ export async function handleIntakeAutoSkip(jobs: Job<IntakeAutoSkipPayload>[]) {
});
evt.addMeta("intake_auto_skip_count", result.skippedCount);
evt.addMeta("intake_auto_skip_cutoff", result.cutoff.toISOString());
return jobDone({
intake_auto_skip_count: result.skippedCount,
intake_auto_skip_cutoff: result.cutoff.toISOString(),
});
} catch (err) {
evt.addWarning(
`intake-auto-skip failed: ${
err instanceof Error ? err.message : String(err)
}`,
);
return jobFailed("intake auto-skip pass failed", err);
}
});
}
@@ -8,6 +8,7 @@ import { type Job } from "pg-boss";
import { recordError, recordReminderCheck } from "@/lib/jobs/worker-status";
import { recomputeMedicationComplianceForEvent } from "@/lib/rollups/medication-compliance-rollups";
import { decrypt } from "@/lib/crypto";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import { dispatchNotification } from "@/lib/notifications/dispatcher";
import { medicationDoseTag } from "@/lib/notifications/dose-tag";
@@ -137,9 +138,11 @@ export async function cleanupScheduledTelegramDeletions(): Promise<void> {
* Uses phase-based logic (GREEN/YELLOW/ORANGE/RED) to send one notification
* per phase transition rather than every 15 minutes.
*/
export async function handleReminderCheck(jobs: Job<ReminderCheckPayload>[]) {
export async function handleReminderCheck(
jobs: Job<ReminderCheckPayload>[],
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.medication_reminder", async (evt) => {
return withBackgroundEvent("job.medication_reminder", async (evt) => {
const prisma = getWorkerPrisma();
try {
recordReminderCheck();
@@ -176,6 +179,12 @@ export async function handleReminderCheck(jobs: Job<ReminderCheckPayload>[]) {
},
});
// The per-slot facts the tick already emits as wide-event meta, counted
// so the run's outcome carries them as scalars.
let missedDosesMinted = 0;
let notificationsDispatched = 0;
let notificationsFailed = 0;
for (const med of medications) {
const userTz = med.user.timezone || "Europe/Berlin";
const { start: todayStart, end: todayEnd } = getUserTodayBounds(
@@ -440,6 +449,7 @@ export async function handleReminderCheck(jobs: Job<ReminderCheckPayload>[]) {
},
});
missedDosesMinted++;
evt.addMeta("missed_dose", `${med.name}:${slotTime}`);
// v1.4.39 W-MED — refresh the compliance rollup so the
@@ -521,7 +531,12 @@ export async function handleReminderCheck(jobs: Job<ReminderCheckPayload>[]) {
url: `/medications/${med.id}`,
},
});
notificationsDispatched++;
} catch (notifErr) {
// One user's channel failing must not fail the tick — the
// dedup anchor is already stamped, so the count is the only
// trace this slot's notice never left the host.
notificationsFailed++;
evt.addWarning(
`Notification dispatch failed for ${currentPhase} phase ${med.name}: ${notifErr}`,
);
@@ -530,6 +545,13 @@ export async function handleReminderCheck(jobs: Job<ReminderCheckPayload>[]) {
}
}
}
return jobDone({
medications: medications.length,
missed_doses_minted: missedDosesMinted,
notifications_dispatched: notificationsDispatched,
notifications_failed: notificationsFailed,
});
} catch (err) {
evt.setError(err);
recordError();
+49 -8
View File
@@ -6,6 +6,7 @@
*/
import { type Job } from "pg-boss";
import { recordError } from "@/lib/jobs/worker-status";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import { runMoodReminderTick } from "@/lib/jobs/mood-reminder";
import { runCycleReminderTick } from "@/lib/jobs/cycle-reminder";
@@ -30,9 +31,9 @@ export interface MeasurementReminderPayload {
export async function handleMoodReminderCheck(
jobs: Job<MoodReminderPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.mood_reminder", async (evt) => {
return withBackgroundEvent("job.mood_reminder", async (evt) => {
const prisma = getWorkerPrisma();
try {
const summary = await runMoodReminderTick(prisma, new Date());
@@ -47,6 +48,16 @@ export async function handleMoodReminderCheck(
skipped_outside_window: summary.skippedOutsideWindow,
},
});
// A tick that found nobody in their local reminder hour still ran the
// sweep, so it is a success with zero counts.
return jobDone({
candidates_scanned: summary.candidatesScanned,
in_window: summary.inWindow,
dispatched: summary.dispatched,
skipped_already_logged: summary.skippedAlreadyLogged,
skipped_already_dispatched: summary.skippedAlreadyDispatched,
skipped_outside_window: summary.skippedOutsideWindow,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -63,9 +74,9 @@ export async function handleMoodReminderCheck(
*/
export async function handleCycleReminderCheck(
jobs: Job<CycleReminderPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.cycle_reminder", async (evt) => {
return withBackgroundEvent("job.cycle_reminder", async (evt) => {
const prisma = getWorkerPrisma();
try {
const summary = await runCycleReminderTick(prisma, new Date());
@@ -82,6 +93,16 @@ export async function handleCycleReminderCheck(
skipped_outside_window: summary.skippedOutsideWindow,
},
});
return jobDone({
candidates_scanned: summary.candidatesScanned,
in_window: summary.inWindow,
dispatched_period_soon: summary.dispatchedPeriodSoon,
dispatched_period_confirm: summary.dispatchedPeriodConfirm,
suppressed_client_managed: summary.suppressedClientManaged,
suppressed_discreet: summary.suppressedDiscreet,
skipped_already_notified: summary.skippedAlreadyNotified,
skipped_outside_window: summary.skippedOutsideWindow,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -97,9 +118,9 @@ export async function handleCycleReminderCheck(
*/
export async function handleMeasurementReminderCheck(
jobs: Job<MeasurementReminderPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.measurement_reminder", async (evt) => {
return withBackgroundEvent("job.measurement_reminder", async (evt) => {
const prisma = getWorkerPrisma();
try {
const summary = await runMeasurementReminderTick(prisma, new Date());
@@ -116,6 +137,19 @@ export async function handleMeasurementReminderCheck(
failed: summary.failed,
},
});
// `failed` counts per-reminder dispatch failures the tick isolated. The
// sweep itself ran, so those ride out as a count rather than failing the
// job and retrying the whole cohort over one user's channel.
return jobDone({
candidates_scanned: summary.candidatesScanned,
in_window: summary.inWindow,
dispatched: summary.dispatched,
auto_resolved: summary.autoResolved,
skipped_not_due: summary.skippedNotDue,
skipped_outside_window: summary.skippedOutsideWindow,
skipped_no_channel: summary.skippedNoChannel,
failed: summary.failed,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -133,8 +167,8 @@ export async function handleMeasurementReminderCheck(
*/
export async function handleReminderSatisfy(
jobs: Job<ReminderSatisfyPayload>[],
) {
await withBackgroundEvent("job.reminder_satisfy", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.reminder_satisfy", async (evt) => {
const prisma = getWorkerPrisma();
const now = new Date();
// Distinct users in this batch — a multi-source ingest spike can enqueue
@@ -160,6 +194,13 @@ export async function handleReminderSatisfy(
satisfied,
},
});
// A batch whose jobs carried no usable userId resolved nothing and is
// still a success with zero counts.
return jobDone({
users: userIds.length,
candidates_scanned: candidates,
satisfied,
});
} catch (err) {
evt.setError(err);
recordError();
+72 -25
View File
@@ -8,6 +8,7 @@ import { type Job } from "pg-boss";
import { runGeoBackfill } from "@/lib/jobs/geo-backfill";
import { runTlsPinMonitor } from "@/lib/jobs/tls-pin-monitor";
import { type PrDetectionPayload } from "@/lib/jobs/pr-detection";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import { runHostMetricTick } from "@/lib/jobs/host-metric-sampler";
import { aggregateRecommendationFeedback } from "@/lib/jobs/feedback-aggregator";
@@ -34,40 +35,45 @@ export interface TlsPinMonitorPayload {
export async function handleHostMetricSample(
jobs: Job<HostMetricSamplePayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.host_metric_sample", async (evt) => {
return withBackgroundEvent("job.host_metric_sample", async (evt) => {
const p = getWorkerPrisma();
try {
const { pruned } = await runHostMetricTick(p);
evt.addMeta("host_metric_pruned", pruned);
return jobDone({ host_metric_pruned: pruned });
} catch (err) {
// The chart degrades gracefully when samples are missing — log
// and move on rather than poisoning the boss queue with retries.
// The chart degrades gracefully when samples are missing, so the
// warning stays — but a tick that took no sample did not do its work.
evt.addWarning(`host-metric-sample failed: ${err}`);
return jobFailed("host metric sample failed", err);
}
});
}
export async function handleFeedbackAggregator(
jobs: Job<FeedbackAggregatorPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.feedback_aggregator", async (evt) => {
return withBackgroundEvent("job.feedback_aggregator", async (evt) => {
const p = getWorkerPrisma();
try {
const summary = await aggregateRecommendationFeedback(p);
const totalRows = summary.buckets.reduce((acc, b) => acc + b.total, 0);
evt.addMeta("feedback_buckets", summary.buckets.length);
evt.addMeta(
"feedback_total_rows",
summary.buckets.reduce((acc, b) => acc + b.total, 0),
);
evt.addMeta("feedback_total_rows", totalRows);
evt.addMeta("feedback_window_days", summary.windowDays);
return jobDone({
feedback_buckets: summary.buckets.length,
feedback_total_rows: totalRows,
feedback_window_days: summary.windowDays,
});
} catch (err) {
// The admin dashboard tolerates a stale summary — log and move
// on rather than poisoning the boss queue with retries that
// would block the next cleanup window.
// The admin dashboard tolerates a stale summary, so the warning stays —
// but a pass that aggregated nothing did not do its work.
evt.addWarning(`feedback-aggregator failed: ${err}`);
return jobFailed("feedback aggregation failed", err);
}
});
}
@@ -91,18 +97,21 @@ export async function handleFeedbackAggregator(
*/
let geoBackfillRunning = false;
export async function handleGeoBackfill(jobs: Job<GeoBackfillPayload>[]) {
export async function handleGeoBackfill(
jobs: Job<GeoBackfillPayload>[],
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.geo_backfill", async (evt) => {
return withBackgroundEvent("job.geo_backfill", async (evt) => {
if (geoBackfillRunning) {
// Earlier pass still in flight; skip this tick. Idempotent + the
// next tick after the in-flight pass completes will pick up
// anything we miss here.
// anything we miss here. Deliberately ceding to the in-flight pass
// is not a failure, so the tick reports the skip and succeeds.
evt.addWarning(
"geo-backfill skipped — earlier pass still in flight inside the same worker process",
);
evt.addMeta("geo_backfill_skipped", true);
return;
return jobDone({ geo_backfill_skipped: true });
}
geoBackfillRunning = true;
const p = getWorkerPrisma();
@@ -112,38 +121,60 @@ export async function handleGeoBackfill(jobs: Job<GeoBackfillPayload>[]) {
evt.addMeta("geo_backfill_located", summary.located);
evt.addMeta("geo_backfill_carrier_resolved", summary.carrierResolved);
evt.addMeta("geo_backfill_still_unresolved", summary.stillUnresolved);
return jobDone({
geo_backfill_scanned: summary.scanned,
geo_backfill_located: summary.located,
geo_backfill_carrier_resolved: summary.carrierResolved,
geo_backfill_still_unresolved: summary.stillUnresolved,
});
} catch (err) {
// The admin sign-in overview tolerates a stale Standort cell —
// log and move on so a one-off resolver hiccup does not poison
// the queue and block the next pass.
// The admin sign-in overview tolerates a stale Standort cell, so the
// warning stays — but a pass that resolved nothing did not do its work.
evt.addWarning(`geo-backfill failed: ${err}`);
return jobFailed("geo backfill pass failed", err);
} finally {
geoBackfillRunning = false;
}
});
}
export async function handleTlsPinMonitor(jobs: Job<TlsPinMonitorPayload>[]) {
export async function handleTlsPinMonitor(
jobs: Job<TlsPinMonitorPayload>[],
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.tls_pin_monitor", async (evt) => {
return withBackgroundEvent("job.tls_pin_monitor", async (evt) => {
const p = getWorkerPrisma();
try {
const summary = await runTlsPinMonitor(p);
evt.addMeta("tls_pin_monitor_outcome", summary.outcome);
evt.addMeta("tls_pin_monitor_host", summary.host);
evt.addMeta("tls_pin_monitor_known_count", summary.knownPinCount);
// No configured target leaves `host` null; the empty string keeps the
// fact shape scalar, and `outcome: "skipped"` already names that case.
return jobDone({
tls_pin_monitor_outcome: summary.outcome,
tls_pin_monitor_host: summary.host ?? "",
tls_pin_monitor_known_count: summary.knownPinCount,
});
} catch (err) {
// runTlsPinMonitor swallows probe failures internally; anything that
// escapes is unexpected. Log and move on so a one-off failure does not
// poison the queue and block the next tick.
// escapes is unexpected, and a tick that never compared a pin is
// exactly the silence this monitor exists to break.
evt.addWarning(`tls-pin-monitor failed: ${err}`);
return jobFailed("tls pin probe failed", err);
}
});
}
export async function handlePrDetection(
jobs: Job<PrDetectionPayload | { userId?: undefined }>[],
) {
): Promise<JobOutcome> {
// Each job in the batch opens its own wide event; the outcome is reported
// once for the batch, so the per-job facts accumulate out here.
let usersScanned = 0;
let insertedAcrossJobs = 0;
let tiesAcrossJobs = 0;
let usersFailed = 0;
for (const job of jobs) {
await withBackgroundEvent("job.pr_detection", async (evt) => {
const p = getWorkerPrisma();
@@ -170,6 +201,10 @@ export async function handlePrDetection(
insertedTotal += result.inserted;
tiesTotal += result.ties;
} catch (err) {
// One user's detection failing is that user's problem, not the
// pass's: it rides out as a count so the cron is not retried for
// the whole cohort.
usersFailed++;
evt.addWarning(`pr-detection failed for user ${userId}: ${err}`);
}
}
@@ -178,6 +213,18 @@ export async function handlePrDetection(
evt.addMeta("pr_detection_ties", tiesTotal);
evt.addMeta("pr_detection_silent", silent);
evt.addMeta("pr_detection_mode", payloadUserId ? "ingest" : "cron");
usersScanned += userIds.length;
insertedAcrossJobs += insertedTotal;
tiesAcrossJobs += tiesTotal;
});
}
return jobDone({
pr_detection_jobs: jobs.length,
pr_detection_users: usersScanned,
pr_detection_inserted: insertedAcrossJobs,
pr_detection_ties: tiesAcrossJobs,
pr_detection_users_failed: usersFailed,
});
}
+24 -3
View File
@@ -21,6 +21,7 @@ import { type Job } from "pg-boss";
import { fireAndForget } from "@/lib/logging/fire-and-forget";
import { recordError } from "@/lib/jobs/worker-status";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy";
import { isSyncFailureRecorded } from "@/lib/integrations/status";
@@ -58,6 +59,13 @@ export interface PollCohortConfig {
* handler resolves its targets from the batch payloads (single-user enqueue) or
* the cohort query (cron tick), then re-syncs each sequentially with per-user
* error isolation so one revoked grant / outage never starves the cohort.
*
* Outcome semantics for a fan-out pass: the JOB succeeded if the pass ran.
* One user's revoked grant is that user's problem — it belongs on their
* integration ledger, not in a queue-wide failure that retries the whole
* cohort — so per-user failures ride out as the `users_failed` count. Only a
* failure of the pass itself (cohort discovery, the wide-event scope) fails
* the job, and that path already throws.
*/
export function makePollCohortHandler({
taskName,
@@ -65,8 +73,10 @@ export function makePollCohortHandler({
syncUser,
recordFailure,
}: PollCohortConfig) {
return async function handlePollCohort(jobs: Job<PollCohortPayload>[]) {
await withBackgroundEvent(taskName, async (evt) => {
return async function handlePollCohort(
jobs: Job<PollCohortPayload>[],
): Promise<JobOutcome> {
return withBackgroundEvent(taskName, async (evt) => {
const prisma = getWorkerPrisma();
try {
const targets: Array<{ userId: string }> = [];
@@ -77,9 +87,12 @@ export function makePollCohortHandler({
const ids = await findCohort(prisma);
targets.push(...ids.map((userId) => ({ userId })));
}
if (targets.length === 0) return;
if (targets.length === 0) {
return jobDone({ total: 0, users_synced: 0, users_failed: 0 });
}
let usersSynced = 0;
let usersFailed = 0;
let measurementsImported = 0;
for (const { userId } of targets) {
try {
@@ -106,6 +119,7 @@ export function makePollCohortHandler({
);
});
}
usersFailed++;
evt.addWarning(`${taskName} failed for user ${userId}: ${err}`);
}
}
@@ -114,10 +128,17 @@ export function makePollCohortHandler({
task_name: taskName,
result: {
users_synced: usersSynced,
users_failed: usersFailed,
total: targets.length,
measurements_imported: measurementsImported,
},
});
return jobDone({
total: targets.length,
users_synced: usersSynced,
users_failed: usersFailed,
measurements_imported: measurementsImported,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -9,8 +9,8 @@
*
* v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its
* cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`,
* and a `boss.work(QUEUE, …, handler)` binding drains it. The queue-wiring
* guards (`withings-queues`, `whoop-queues`, `fitbit-queues`,
* and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The
* queue-wiring guards (`withings-queues`, `whoop-queues`, `fitbit-queues`,
* `nightscout-queues`, `sleep-timeline-queues`) read THIS module.
*/
import { PgBoss } from "pg-boss";
@@ -78,8 +78,11 @@ import {
type IntegrationBackfillAdmissionPayload,
} from "@/lib/jobs/integration-backfill-admission";
import { workerLog } from "./shared";
import { jobDone } from "@/lib/jobs/job-outcome";
import {
createAndSchedule,
createAndWork,
cronIsTheRetry,
type QueuePolicyTable,
type ScheduleEntry,
} from "./registrar-shared";
@@ -333,7 +336,16 @@ const schedules: ScheduleEntry[] = [
// v1.32.23 — hourly ECG catch-net (:41) so watch-only accounts pull their
// strips without a webhook or a scale.
[WITHINGS_ECG_SYNC_QUEUE, WITHINGS_ECG_SYNC_CRON],
[WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, WITHINGS_OAUTH_STATE_CLEANUP_CRON],
// The five OAuth-state / handoff prunes carry `cronIsTheRetry`: their
// handlers used to warn and carry on, they now fail the job, and a DELETE
// that could not run will not run any better a second later. The daily tick
// is the retry. The sync crons above keep the default retries — a provider
// outage is transient and worth retrying inside the hour.
[
WITHINGS_OAUTH_STATE_CLEANUP_QUEUE,
WITHINGS_OAUTH_STATE_CLEANUP_CRON,
cronIsTheRetry,
],
// v1.11.0 — WHOOP poll-fallback crons. Recovery/sleep/workout catch
// dropped webhooks; cycle is the sole driver (no webhook). Staggered off
// the Withings crons so the hourly ticks don't pile up on one boss poll.
@@ -342,11 +354,19 @@ const schedules: ScheduleEntry[] = [
[WHOOP_WORKOUT_SYNC_QUEUE, WHOOP_WORKOUT_SYNC_CRON],
[WHOOP_CYCLE_SYNC_QUEUE, WHOOP_CYCLE_SYNC_CRON],
// v1.11.0 — daily 03:22 Europe/Berlin prune for expired WHOOP OAuth states.
[WHOOP_OAUTH_STATE_CLEANUP_QUEUE, WHOOP_OAUTH_STATE_CLEANUP_CRON],
[
WHOOP_OAUTH_STATE_CLEANUP_QUEUE,
WHOOP_OAUTH_STATE_CLEANUP_CRON,
cronIsTheRetry,
],
// v1.12.0 — hourly Fitbit poll (:08, staggered off WHOOP/Withings) + the
// daily 03:24 Europe/Berlin prune for expired Fitbit OAuth states.
[FITBIT_SYNC_QUEUE, FITBIT_SYNC_CRON],
[FITBIT_OAUTH_STATE_CLEANUP_QUEUE, FITBIT_OAUTH_STATE_CLEANUP_CRON],
[
FITBIT_OAUTH_STATE_CLEANUP_QUEUE,
FITBIT_OAUTH_STATE_CLEANUP_CRON,
cronIsTheRetry,
],
// v1.26.0 — hourly Google Health poll (:18, staggered off WHOOP/Fitbit/
// Nightscout/Polar/Oura) + the daily 03:26 Europe/Berlin prune for expired
// Google Health OAuth states.
@@ -354,6 +374,7 @@ const schedules: ScheduleEntry[] = [
[
GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE,
GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON,
cronIsTheRetry,
],
// v1.17.0 — hourly Nightscout CGM poll (:11, staggered off the other sync
// ticks).
@@ -364,7 +385,11 @@ const schedules: ScheduleEntry[] = [
// v1.28.x — hourly Strava (:17) OAuth poll.
[STRAVA_SYNC_QUEUE, STRAVA_SYNC_CRON],
// v1.30.x — daily 03:28 Europe/Berlin prune for expired native OIDC handoffs.
[OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, OIDC_NATIVE_HANDOFF_CLEANUP_CRON],
[
OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE,
OIDC_NATIVE_HANDOFF_CLEANUP_CRON,
cronIsTheRetry,
],
];
/**
@@ -450,19 +475,30 @@ export async function registerIntegrationSyncQueues(
// reaches this shared durable queue before doing full-history work. The
// common group is enforced in PostgreSQL across worker processes/nodes.
await boss.work<IntegrationBackfillAdmissionPayload>(
await createAndWork<IntegrationBackfillAdmissionPayload>(
boss,
INTEGRATION_BACKFILL_ADMISSION_QUEUE,
{
localConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY,
groupConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY,
},
async (jobs) => {
// The drain has no per-job error isolation: a runner that throws fails
// the job and the admission retry policy applies. So reaching the end
// means every admitted job did its work, and the counts each runner
// already returns are what the pass reports.
let importedTotal = 0;
let removedTotal = 0;
let deletedTotal = 0;
let markersTotal = 0;
let linkedTotal = 0;
for (const job of jobs) {
const { kind, data } = job.data;
const { userId } = data;
switch (kind) {
case "whoop-backfill": {
const { imported } = await runWhoopBackfillForUser(userId);
importedTotal += imported;
workerLog(
"info",
`[whoop-backfill] user=${userId} imported=${imported}`,
@@ -471,6 +507,7 @@ export async function registerIntegrationSyncQueues(
}
case "fitbit-backfill": {
const { imported } = await runFitbitBackfillForUser(userId);
importedTotal += imported;
workerLog(
"info",
`[fitbit-backfill] user=${userId} imported=${imported}`,
@@ -479,6 +516,7 @@ export async function registerIntegrationSyncQueues(
}
case "google-health-backfill": {
const { imported } = await runGoogleHealthBackfillForUser(userId);
importedTotal += imported;
workerLog(
"info",
`[google-health-backfill] user=${userId} imported=${imported}`,
@@ -488,6 +526,7 @@ export async function registerIntegrationSyncQueues(
case "google-health-sleep-repair": {
const { imported } =
await runGoogleHealthSleepRepairForUser(userId);
importedTotal += imported;
workerLog(
"info",
`[google-health-sleep-repair] user=${userId} imported=${imported}`,
@@ -497,6 +536,8 @@ export async function registerIntegrationSyncQueues(
case "fitbit-sleep-repair": {
const { imported, removed } =
await runFitbitSleepRepairForUser(userId);
importedTotal += imported;
removedTotal += removed;
workerLog(
"info",
`[fitbit-sleep-repair] user=${userId} imported=${imported} removed=${removed}`,
@@ -513,6 +554,8 @@ export async function registerIntegrationSyncQueues(
userId,
data.provider,
);
deletedTotal += deleted;
importedTotal += imported;
workerLog(
"info",
`[sleep-timeline-backfill] user=${userId} provider=${data.provider} deleted=${deleted} imported=${imported}`,
@@ -522,6 +565,8 @@ export async function registerIntegrationSyncQueues(
case "lab-biomarker-backfill": {
const { markers, linked } =
await runLabBiomarkerBackfillForUser(userId);
markersTotal += markers;
linkedTotal += linked;
workerLog(
"info",
`[lab-biomarker-backfill] user=${userId} markers=${markers} linked=${linked}`,
@@ -530,6 +575,7 @@ export async function registerIntegrationSyncQueues(
}
case "strava-backfill": {
const { imported } = await runStravaBackfillForUser(userId);
importedTotal += imported;
workerLog(
"info",
`[strava-backfill] user=${userId} imported=${imported}`,
@@ -538,30 +584,43 @@ export async function registerIntegrationSyncQueues(
}
}
}
return jobDone({
jobs: jobs.length,
imported: importedTotal,
removed: removedTotal,
deleted: deletedTotal,
markers: markersTotal,
linked: linkedTotal,
});
},
);
await boss.work<WithingsSyncPayload>(
await createAndWork<WithingsSyncPayload>(
boss,
WITHINGS_SYNC_QUEUE,
{ localConcurrency: 1 },
handleWithingsFallbackSync,
);
await boss.work<WithingsActivitySyncPayload>(
await createAndWork<WithingsActivitySyncPayload>(
boss,
WITHINGS_ACTIVITY_QUEUE,
{ localConcurrency: 1 },
handleWithingsActivitySync,
);
await boss.work<WithingsSleepSyncPayload>(
await createAndWork<WithingsSleepSyncPayload>(
boss,
WITHINGS_SLEEP_QUEUE,
{ localConcurrency: 1 },
handleWithingsSleepSync,
);
await boss.work<WithingsEcgSyncPayload>(
await createAndWork<WithingsEcgSyncPayload>(
boss,
WITHINGS_ECG_SYNC_QUEUE,
{ localConcurrency: 1 },
handleWithingsEcgSync,
);
await boss.work<WithingsOAuthStateCleanupPayload>(
await createAndWork<WithingsOAuthStateCleanupPayload>(
boss,
WITHINGS_OAUTH_STATE_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleWithingsOAuthStateCleanup,
@@ -569,29 +628,34 @@ export async function registerIntegrationSyncQueues(
// v1.11.0 — WHOOP per-resource sync handlers. Webhook-driven per-user +
// cron full-iteration. Serial concurrency so a backfill-heavy tick never
// crowds the request pool and stays inside WHOOP's 100 req/min app cap.
await boss.work<WhoopSyncPayload>(
await createAndWork<WhoopSyncPayload>(
boss,
WHOOP_RECOVERY_SYNC_QUEUE,
{ localConcurrency: 1 },
handleWhoopRecoverySync,
);
await boss.work<WhoopSyncPayload>(
await createAndWork<WhoopSyncPayload>(
boss,
WHOOP_SLEEP_SYNC_QUEUE,
{ localConcurrency: 1 },
handleWhoopSleepSync,
);
await boss.work<WhoopSyncPayload>(
await createAndWork<WhoopSyncPayload>(
boss,
WHOOP_WORKOUT_SYNC_QUEUE,
{ localConcurrency: 1 },
handleWhoopWorkoutSync,
);
await boss.work<WhoopSyncPayload>(
await createAndWork<WhoopSyncPayload>(
boss,
WHOOP_CYCLE_SYNC_QUEUE,
{ localConcurrency: 1 },
handleWhoopCycleSync,
);
// v1.11.0 — self-converging WHOOP backfill. This source queue retains the
// per-connection singleton and forwards admitted work to the shared drain.
await boss.work<WhoopBackfillPayload>(
await createAndWork<WhoopBackfillPayload>(
boss,
WHOOP_BACKFILL_QUEUE,
{ localConcurrency: WHOOP_BACKFILL_CONCURRENCY },
async (jobs) => {
@@ -601,29 +665,34 @@ export async function registerIntegrationSyncQueues(
data: job.data,
});
}
return jobDone({ jobs: jobs.length });
},
);
await boss.work<WhoopOAuthStateCleanupPayload>(
await createAndWork<WhoopOAuthStateCleanupPayload>(
boss,
WHOOP_OAUTH_STATE_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleWhoopOAuthStateCleanup,
);
// v1.30.x — daily sweep for the native OIDC handoff ledger.
await boss.work<OidcNativeHandoffCleanupPayload>(
await createAndWork<OidcNativeHandoffCleanupPayload>(
boss,
OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleOidcNativeHandoffCleanup,
);
// v1.12.0 — Fitbit poll-sync (cron full-iteration; no webhook). Serial
// concurrency so a backfill-heavy tick never crowds the request pool.
await boss.work<FitbitSyncPayload>(
await createAndWork<FitbitSyncPayload>(
boss,
FITBIT_SYNC_QUEUE,
{ localConcurrency: 1 },
handleFitbitSync,
);
// v1.12.0 — self-converging Fitbit backfill. This source queue retains the
// per-connection singleton and forwards admitted work to the shared drain.
await boss.work<FitbitBackfillPayload>(
await createAndWork<FitbitBackfillPayload>(
boss,
FITBIT_BACKFILL_QUEUE,
{ localConcurrency: FITBIT_BACKFILL_CONCURRENCY },
async (jobs) => {
@@ -633,12 +702,14 @@ export async function registerIntegrationSyncQueues(
data: job.data,
});
}
return jobDone({ jobs: jobs.length });
},
);
// One-shot Fitbit sleep duplicate repair. This source queue retains the
// per-connection singleton and forwards the watermark-safe bounded sleep
// re-read to the shared drain.
await boss.work<FitbitSleepRepairPayload>(
await createAndWork<FitbitSleepRepairPayload>(
boss,
FITBIT_SLEEP_REPAIR_QUEUE,
{ localConcurrency: FITBIT_SLEEP_REPAIR_CONCURRENCY },
async (jobs) => {
@@ -648,16 +719,19 @@ export async function registerIntegrationSyncQueues(
data: job.data,
});
}
return jobDone({ jobs: jobs.length });
},
);
await boss.work<FitbitOAuthStateCleanupPayload>(
await createAndWork<FitbitOAuthStateCleanupPayload>(
boss,
FITBIT_OAUTH_STATE_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleFitbitOAuthStateCleanup,
);
// v1.26.0 — Google Health poll-sync (cron full-iteration; no webhook). Serial
// concurrency so a backfill-heavy tick never crowds the request pool.
await boss.work<GoogleHealthSyncPayload>(
await createAndWork<GoogleHealthSyncPayload>(
boss,
GOOGLE_HEALTH_SYNC_QUEUE,
{ localConcurrency: 1 },
handleGoogleHealthSync,
@@ -665,7 +739,8 @@ export async function registerIntegrationSyncQueues(
// v1.26.0 — self-converging Google Health backfill. This source queue
// retains the per-connection singleton and forwards admitted work to the
// shared drain.
await boss.work<GoogleHealthBackfillPayload>(
await createAndWork<GoogleHealthBackfillPayload>(
boss,
GOOGLE_HEALTH_BACKFILL_QUEUE,
{ localConcurrency: GOOGLE_HEALTH_BACKFILL_CONCURRENCY },
async (jobs) => {
@@ -675,12 +750,14 @@ export async function registerIntegrationSyncQueues(
data: job.data,
});
}
return jobDone({ jobs: jobs.length });
},
);
// v1.28.x — one-shot Google Health sleep duplicate repair. This source
// queue retains the per-connection singleton and forwards the watermark-safe
// full sleep-history re-read to the shared drain.
await boss.work<GoogleHealthSleepRepairPayload>(
await createAndWork<GoogleHealthSleepRepairPayload>(
boss,
GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE,
{ localConcurrency: GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY },
async (jobs) => {
@@ -690,9 +767,11 @@ export async function registerIntegrationSyncQueues(
data: job.data,
});
}
return jobDone({ jobs: jobs.length });
},
);
await boss.work<GoogleHealthOAuthStateCleanupPayload>(
await createAndWork<GoogleHealthOAuthStateCleanupPayload>(
boss,
GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleGoogleHealthOAuthStateCleanup,
@@ -700,7 +779,8 @@ export async function registerIntegrationSyncQueues(
// v1.17.1 — one-shot sleep-timeline backfill. This source queue retains the
// per-(user, provider) singleton and forwards admitted work to the shared
// drain.
await boss.work<SleepTimelineBackfillPayload>(
await createAndWork<SleepTimelineBackfillPayload>(
boss,
SLEEP_TIMELINE_BACKFILL_QUEUE,
{ localConcurrency: SLEEP_TIMELINE_BACKFILL_CONCURRENCY },
async (jobs) => {
@@ -710,11 +790,13 @@ export async function registerIntegrationSyncQueues(
data: job.data,
});
}
return jobDone({ jobs: jobs.length });
},
);
// v1.18.1 — one-shot lab-biomarker backfill. This source queue retains the
// per-user singleton and forwards admitted work to the shared drain.
await boss.work<LabBiomarkerBackfillPayload>(
await createAndWork<LabBiomarkerBackfillPayload>(
boss,
LAB_BIOMARKER_BACKFILL_QUEUE,
{ localConcurrency: LAB_BIOMARKER_BACKFILL_CONCURRENCY },
async (jobs) => {
@@ -724,12 +806,14 @@ export async function registerIntegrationSyncQueues(
data: job.data,
});
}
return jobDone({ jobs: jobs.length });
},
);
// v1.17.0 — Nightscout CGM poll-cohort sync. The hourly cron tick (no
// `userId`) walks every configured instance; one user's unreachable host is
// warned, not fatal.
await boss.work<NightscoutSyncPayload>(
await createAndWork<NightscoutSyncPayload>(
boss,
NIGHTSCOUT_SYNC_QUEUE,
{ localConcurrency: 1 },
handleNightscoutSync,
@@ -737,12 +821,14 @@ export async function registerIntegrationSyncQueues(
// v1.17.0 (F4) — Polar + Oura OAuth poll-cohort sync. The hourly cron tick
// (no `userId`) re-walks every connected user; one user's revoked grant is
// warned, not fatal.
await boss.work<PolarSyncPayload>(
await createAndWork<PolarSyncPayload>(
boss,
POLAR_SYNC_QUEUE,
{ localConcurrency: 1 },
handlePolarSync,
);
await boss.work<OuraSyncPayload>(
await createAndWork<OuraSyncPayload>(
boss,
OURA_SYNC_QUEUE,
{ localConcurrency: 1 },
handleOuraSync,
@@ -750,14 +836,16 @@ export async function registerIntegrationSyncQueues(
// v1.28.x — Strava OAuth poll-cohort sync (poll-only; webhook deferred). The
// hourly cron tick (no `userId`) re-walks every connected user; one user's
// revoked grant is warned, not fatal.
await boss.work<StravaSyncPayload>(
await createAndWork<StravaSyncPayload>(
boss,
STRAVA_SYNC_QUEUE,
{ localConcurrency: 1 },
handleStravaSync,
);
// v1.28.x — self-converging Strava backfill. This source queue retains the
// per-connection singleton and forwards admitted work to the shared drain.
await boss.work<StravaBackfillPayload>(
await createAndWork<StravaBackfillPayload>(
boss,
STRAVA_BACKFILL_QUEUE,
{ localConcurrency: STRAVA_BACKFILL_CONCURRENCY },
async (jobs) => {
@@ -767,6 +855,7 @@ export async function registerIntegrationSyncQueues(
data: job.data,
});
}
return jobDone({ jobs: jobs.length });
},
);
return allQueues;
+283 -63
View File
@@ -10,7 +10,7 @@
*
* v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its
* cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`,
* and a `boss.work(QUEUE, …, handler)` binding drains it. The
* and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The
* `measurement-tombstone-cleanup`, `withings-oauth-state-cleanup`,
* `intake-slot-dedup-queue`, `tls-pin-monitor-queue`, and `geo-backfill`
* guards read THIS module.
@@ -131,9 +131,12 @@ import {
type EnvironmentFetchPayload,
} from "@/lib/jobs/environment-fetch";
import { recordError } from "@/lib/jobs/worker-status";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { workerLog, BOOT_BACKFILL_STAGGER_SECONDS } from "./shared";
import {
createAndSchedule,
createAndWork,
cronIsTheRetry,
type QueuePolicyTable,
type ScheduleEntry,
} from "./registrar-shared";
@@ -411,24 +414,35 @@ const allQueues = [
DOCUMENT_SUMMARY_CATCHUP_QUEUE,
];
// `cronIsTheRetry` (retryLimit 0) marks the passes whose handler now returns
// `jobFailed` where it used to warn and carry on, AND whose failure repeats:
// a bulk DELETE or UPDATE that timed out will time out again on an immediate
// retry, so pg-boss's default two would run the same doomed statement three
// times a night. The queues without it are the ones whose failure is
// transient — the TLS probe, the off-host upload, the restore drill, the
// per-user environment fetch — and those still want the default retries.
const schedules: ScheduleEntry[] = [
// v1.25 (W-ENV) — daily 02:10 Europe/Berlin discovery tick (empty payload)
// that fans out one per-user environment fetch per opted-in account.
[ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON],
[DATA_BACKUP_QUEUE, DATA_BACKUP_CRON],
[RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON],
[IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON],
[RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, cronIsTheRetry],
[IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON, cronIsTheRetry],
[AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON],
[STEP_UP_ELEVATION_CLEANUP_QUEUE, STEP_UP_ELEVATION_CLEANUP_CRON],
[
STEP_UP_ELEVATION_CLEANUP_QUEUE,
STEP_UP_ELEVATION_CLEANUP_CRON,
cronIsTheRetry,
],
[OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON],
[RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON],
[HOST_METRIC_QUEUE, HOST_METRIC_CRON],
[FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON],
[HOST_METRIC_QUEUE, HOST_METRIC_CRON, cronIsTheRetry],
[FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON, cronIsTheRetry],
// v1.4.37 — hourly geo backfill. The helper is idempotent + capped
// at 5 000 rows per pass; running it at :40 every hour catches the
// long tail of audit rows that landed with the offline MMDB
// missing or the online provider unreachable.
[GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON],
[GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON, cronIsTheRetry],
// v1.12.2 ios-coord — every-6-hour TLS leaf SPKI probe (:07 off the
// hourly sync crons). Surfaces a pinned-leaf rotation well inside the
// ≥11-day re-pin window the iOS release owner needs.
@@ -443,14 +457,18 @@ const schedules: ScheduleEntry[] = [
// entities. Flips IN_USE rows whose 30-day clock has blown to
// EXPIRED at 03:30 Europe/Berlin (in the existing 02:xx–03:xx
// maintenance window, right after idempotency-cleanup).
[MEDICATION_INVENTORY_EXPIRE_QUEUE, MEDICATION_INVENTORY_EXPIRE_CRON],
[
MEDICATION_INVENTORY_EXPIRE_QUEUE,
MEDICATION_INVENTORY_EXPIRE_CRON,
cronIsTheRetry,
],
// v1.4.46 — hourly auto-skip for medication intakes the user never
// marked. Cron `5 * * * *` slots off the top-of-the-hour
// reminder-check (:00) so the hourly ticks don't pile up on the same
// boss poll.
[INTAKE_AUTO_SKIP_QUEUE, INTAKE_AUTO_SKIP_CRON],
[INTAKE_AUTO_SKIP_QUEUE, INTAKE_AUTO_SKIP_CRON, cronIsTheRetry],
// v0.5.4 ios-coord — daily mood-reminder dispatch-ledger retention sweep.
[MOOD_REMINDER_CLEANUP_QUEUE, MOOD_REMINDER_CLEANUP_CRON],
[MOOD_REMINDER_CLEANUP_QUEUE, MOOD_REMINDER_CLEANUP_CRON, cronIsTheRetry],
// v1.15.19 — daily 03:28 Europe/Berlin duplicate dose-slot dedup
// discovery. The empty cron payload (no `userId`) is the handler's
// signal to run the discovery fan-out instead of a per-user pass, so
@@ -458,20 +476,24 @@ const schedules: ScheduleEntry[] = [
// a day instead of waiting for the next worker reboot.
[INTAKE_SLOT_DEDUP_QUEUE, INTAKE_SLOT_DEDUP_CRON],
// v1.4.49 — daily 03:35 Europe/Berlin prune for push_attempts.
[PUSH_ATTEMPT_CLEANUP_QUEUE, PUSH_ATTEMPT_CLEANUP_CRON],
[PUSH_ATTEMPT_CLEANUP_QUEUE, PUSH_ATTEMPT_CLEANUP_CRON, cronIsTheRetry],
// v1.31.0 — daily 04:10 Europe/Berlin prune for arrival_reactions.
[ARRIVAL_REACTION_CLEANUP_QUEUE, ARRIVAL_REACTION_CLEANUP_CRON],
[
ARRIVAL_REACTION_CLEANUP_QUEUE,
ARRIVAL_REACTION_CLEANUP_CRON,
cronIsTheRetry,
],
// v1.7.0 — daily 03:40 Europe/Berlin prune for expired measurement
// tombstones.
[MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, MEASUREMENT_TOMBSTONE_CLEANUP_CRON],
// v1.18.7 — daily 03:50 Europe/Berlin prune for stale Coach history.
[COACH_MESSAGE_CLEANUP_QUEUE, COACH_MESSAGE_CLEANUP_CRON],
[COACH_MESSAGE_CLEANUP_QUEUE, COACH_MESSAGE_CLEANUP_CRON, cronIsTheRetry],
// v1.23 — daily 03:55 Europe/Berlin note-encryption backfill discovery.
// The empty cron payload (no `userId`) is the handler's signal to fan out
// one per-user job per account still holding a plaintext note.
[NOTE_ENCRYPTION_BACKFILL_QUEUE, NOTE_ENCRYPTION_BACKFILL_CRON],
// v1.24 — daily 04:00 Europe/Berlin prune for dead MCP connector tokens.
[MCP_TOKEN_CLEANUP_QUEUE, MCP_TOKEN_CLEANUP_CRON],
[MCP_TOKEN_CLEANUP_QUEUE, MCP_TOKEN_CLEANUP_CRON, cronIsTheRetry],
// v1.25 — daily 04:05 Europe/Berlin medication-note encryption backfill
// discovery. The empty cron payload (no `userId`) is the handler's signal to
// fan out one per-user job per account still holding a plaintext medication
@@ -479,14 +501,14 @@ const schedules: ScheduleEntry[] = [
[MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, MED_NOTES_ENCRYPTION_BACKFILL_CRON],
// Document vault — daily 04:10 Europe/Berlin purge for tombstoned
// documents past the 30-day undo grace.
[DOCUMENT_PURGE_QUEUE, DOCUMENT_PURGE_CRON],
[DOCUMENT_PURGE_QUEUE, DOCUMENT_PURGE_CRON, cronIsTheRetry],
// v1.32.1 (issue #588) — every-15-minute orphan-ImportJob sweep. Re-runs
// the same reconcile the boot path uses, so a stuck "unpacking" row
// whose worker crashed/restarted without the boot-time pass catching it
// (heartbeat not yet stale, pg-boss job not yet expired) still converges
// to a visible `failed` state within two ticks instead of staying stuck
// forever on an otherwise-healthy worker.
[IMPORT_JOB_RECONCILE_QUEUE, IMPORT_JOB_RECONCILE_CRON],
[IMPORT_JOB_RECONCILE_QUEUE, IMPORT_JOB_RECONCILE_CRON, cronIsTheRetry],
];
/**
@@ -584,61 +606,76 @@ export async function registerMaintenanceQueues(
): Promise<readonly string[]> {
await createAndSchedule(boss, allQueues, schedules, queuePolicies);
await boss.work<DataBackupPayload>(
await createAndWork<DataBackupPayload>(
boss,
DATA_BACKUP_QUEUE,
{ localConcurrency: 1 },
handleDataBackup,
);
await boss.work<RateLimitCleanupPayload>(
await createAndWork<RateLimitCleanupPayload>(
boss,
RATE_LIMIT_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleRateLimitCleanup,
);
await boss.work<IdempotencyCleanupPayload>(
await createAndWork<IdempotencyCleanupPayload>(
boss,
IDEMPOTENCY_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleIdempotencyCleanup,
);
await boss.work<AuditLogCleanupPayload>(
await createAndWork<AuditLogCleanupPayload>(
boss,
AUDIT_LOG_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleAuditLogCleanup,
);
await boss.work<StepUpElevationCleanupPayload>(
await createAndWork<StepUpElevationCleanupPayload>(
boss,
STEP_UP_ELEVATION_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleStepUpElevationCleanup,
);
await boss.work<OffhostBackupPayload>(
await createAndWork<OffhostBackupPayload>(
boss,
OFFHOST_BACKUP_QUEUE,
{ localConcurrency: 1 },
handleOffhostBackup,
);
// prettier-ignore
await boss.work(RESTORE_DRILL_QUEUE, { localConcurrency: 1 }, handleRestoreDrill);
await boss.work<HostMetricSamplePayload>(
await createAndWork(
boss,
RESTORE_DRILL_QUEUE,
{ localConcurrency: 1 },
handleRestoreDrill,
);
await createAndWork<HostMetricSamplePayload>(
boss,
HOST_METRIC_QUEUE,
{ localConcurrency: 1 },
handleHostMetricSample,
);
await boss.work<FeedbackAggregatorPayload>(
await createAndWork<FeedbackAggregatorPayload>(
boss,
FEEDBACK_AGGREGATOR_QUEUE,
{ localConcurrency: 1 },
handleFeedbackAggregator,
);
await boss.work<GeoBackfillPayload>(
await createAndWork<GeoBackfillPayload>(
boss,
GEO_BACKFILL_QUEUE,
{ localConcurrency: 1 },
handleGeoBackfill,
);
// v1.12.2 ios-coord — TLS leaf SPKI-change monitor. Single-flight: one
// short outbound TLS handshake per tick, no benefit to overlapping ticks.
await boss.work<TlsPinMonitorPayload>(
await createAndWork<TlsPinMonitorPayload>(
boss,
TLS_PIN_MONITOR_QUEUE,
{ localConcurrency: 1 },
handleTlsPinMonitor,
);
await boss.work<MoodReminderCleanupPayload>(
await createAndWork<MoodReminderCleanupPayload>(
boss,
MOOD_REMINDER_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleMoodReminderCleanup,
@@ -647,7 +684,8 @@ export async function registerMaintenanceQueues(
// matches every other cleanup queue; two ticks racing on the same
// DELETE statement is wasted work and the second tick's payload
// would be a no-op anyway.
await boss.work<PushAttemptCleanupPayload>(
await createAndWork<PushAttemptCleanupPayload>(
boss,
PUSH_ATTEMPT_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handlePushAttemptCleanup,
@@ -655,14 +693,16 @@ export async function registerMaintenanceQueues(
// v1.31.0 — daily prune of the data-arrival reaction markers. Single-flight
// like every other cleanup queue: two ticks racing the same DELETE is
// wasted work.
await boss.work<ArrivalReactionCleanupPayload>(
await createAndWork<ArrivalReactionCleanupPayload>(
boss,
ARRIVAL_REACTION_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleArrivalReactionCleanup,
);
// v1.7.0 — daily prune of expired measurement tombstones. Single-flight
// like every other cleanup queue.
await boss.work<MeasurementTombstoneCleanupPayload>(
await createAndWork<MeasurementTombstoneCleanupPayload>(
boss,
MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleMeasurementTombstoneCleanup,
@@ -670,14 +710,16 @@ export async function registerMaintenanceQueues(
// v1.18.7 — daily prune of the encrypted Coach conversation history.
// Single-flight like every other cleanup queue; two ticks racing on the
// same DELETE is wasted work and the second is a no-op.
await boss.work<CoachMessageCleanupPayload>(
await createAndWork<CoachMessageCleanupPayload>(
boss,
COACH_MESSAGE_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleCoachMessageCleanup,
);
// MCP Phase 3 (M2) — daily prune of dead MCP connector access tokens +
// long-revoked connection anchors. Single-flight like every other cleanup.
await boss.work<McpTokenCleanupPayload>(
await createAndWork<McpTokenCleanupPayload>(
boss,
MCP_TOKEN_CLEANUP_QUEUE,
{ localConcurrency: 1 },
handleMcpTokenCleanup,
@@ -685,17 +727,20 @@ export async function registerMaintenanceQueues(
// Document vault — daily purge of tombstoned documents past the 30-day
// undo grace. Single-flight like every other cleanup queue; the underlying
// deleteMany is idempotent so a duplicate tick is a no-op.
await boss.work<DocumentPurgePayload>(
await createAndWork<DocumentPurgePayload>(
boss,
DOCUMENT_PURGE_QUEUE,
{ localConcurrency: 1 },
handleDocumentPurge,
);
await boss.work<PrDetectionPayload>(
await createAndWork<PrDetectionPayload>(
boss,
PR_DETECTION_QUEUE,
{ localConcurrency: PR_DETECTION_CONCURRENCY },
handlePrDetection,
);
await boss.work<MedicationInventoryExpirePayload>(
await createAndWork<MedicationInventoryExpirePayload>(
boss,
MEDICATION_INVENTORY_EXPIRE_QUEUE,
{ localConcurrency: 1 },
handleMedicationInventoryExpire,
@@ -704,7 +749,8 @@ export async function registerMaintenanceQueues(
// Single-flight: two ticks racing against the same row pile is wasted
// work, and the underlying `updateMany` is the canonical idempotent
// shape anyway.
await boss.work<IntakeAutoSkipPayload>(
await createAndWork<IntakeAutoSkipPayload>(
boss,
INTAKE_AUTO_SKIP_QUEUE,
{ localConcurrency: 1 },
handleIntakeAutoSkip,
@@ -712,48 +758,64 @@ export async function registerMaintenanceQueues(
// v1.4.34 — Apple Health export.zip ingest worker. localConcurrency
// caps at 1 because the parse loop is CPU-bound and a concurrent
// second 1 GB import would race the first for RSS.
await boss.work<AppleHealthImportPayload>(
await createAndWork<AppleHealthImportPayload>(
boss,
APPLE_HEALTH_IMPORT_V2_QUEUE,
{ localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY },
async (jobs) => {
// pg-boss v12 work callbacks always receive an array (batched
// worker mode); for our concurrency-1 + batchSize-1 case we just
// process each job sequentially.
//
// Each import is independent work, so a failed one does not abort the
// rest of the batch; the first failure is what the job reports.
let failure: JobOutcome | null = null;
for (const job of jobs) {
await handleAppleHealthImport(job);
const outcome = await handleAppleHealthImport(job);
if (!outcome.ok && failure === null) failure = outcome;
}
return failure ?? jobDone({ jobs: jobs.length });
},
);
// Drain pre-revision-2 backlog by moving each claimed legacy job onto the
// isolated v2 queue. This preserves the ImportJob status id without letting
// the current parser execute under a revision-1 mirror.
await boss.work<AppleHealthImportPayload>(
await createAndWork<AppleHealthImportPayload>(
boss,
APPLE_HEALTH_IMPORT_LEGACY_QUEUE,
{ localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY },
async (jobs) => {
for (const job of jobs) {
await migrateLegacyAppleHealthImport(job);
}
return jobDone({ jobs: jobs.length });
},
);
// v1.32.1 (issue #588) — periodic orphan-ImportJob sweep. Single-flight:
// the underlying `updateMany` is idempotent and two ticks racing the same
// read-then-write pass would just be wasted work.
await boss.work(
await createAndWork(
boss,
IMPORT_JOB_RECONCILE_QUEUE,
{ localConcurrency: 1 },
handleImportJobReconcileTick,
);
await boss.work<MedicationIntakeImportQueuePayload>(
await createAndWork<MedicationIntakeImportQueuePayload>(
boss,
MEDICATION_INTAKE_IMPORT_QUEUE,
{
localConcurrency: MEDICATION_INTAKE_IMPORT_CONCURRENCY,
includeMetadata: true,
},
async (jobs) => {
// Independent per-delivery imports: run them all, report the first
// failure rather than losing it behind a later success.
let failure: JobOutcome | null = null;
for (const job of jobs) {
await handleMedicationIntakeImport(job);
const outcome = await handleMedicationIntakeImport(job);
if (!outcome.ok && failure === null) failure = outcome;
}
return failure ?? jobDone({ jobs: jobs.length });
},
);
@@ -768,10 +830,21 @@ export async function registerMaintenanceQueues(
// above) whose payload omits `userId`. That tick runs the SAME discovery
// fan-out the boot pass uses, so duplicate slots created between deploys
// collapse within a day instead of waiting for the next worker reboot.
await boss.work<Partial<IntakeSlotDedupPayload>>(
await createAndWork<Partial<IntakeSlotDedupPayload>>(
boss,
INTAKE_SLOT_DEDUP_QUEUE,
{ localConcurrency: INTAKE_SLOT_DEDUP_CONCURRENCY },
async (jobs) => {
let users = 0;
let discoveryEnqueued = 0;
let discoverySkipped = 0;
// The discovery helper returns its error instead of throwing, so a
// failed fan-out would otherwise leave no trace on the job at all.
let discoveryFailed = 0;
let slotsCollapsed = 0;
let rowsSoftDeleted = 0;
let rowsNormalised = 0;
let daysRecomputed = 0;
for (const job of jobs) {
const { userId } = job.data;
if (!userId) {
@@ -779,6 +852,9 @@ export async function registerMaintenanceQueues(
// still holding duplicate-slot candidates (singletonKey-coalesced,
// identical to the boot pass).
const result = await enqueueBootTimeIntakeSlotDedup();
discoveryEnqueued += result.enqueued;
discoverySkipped += result.skipped;
if (result.error) discoveryFailed++;
workerLog(
"info",
`[intake-slot-dedup] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`,
@@ -787,6 +863,11 @@ export async function registerMaintenanceQueues(
}
try {
const summary = await dedupeUserIntakeSlots(userId);
users++;
slotsCollapsed += summary.slotsCollapsed;
rowsSoftDeleted += summary.rowsSoftDeleted;
rowsNormalised += summary.rowsNormalised;
daysRecomputed += summary.daysRecomputed;
workerLog(
"info",
`[intake-slot-dedup] user=${userId} slotsCollapsed=${summary.slotsCollapsed} rowsSoftDeleted=${summary.rowsSoftDeleted} rowsNormalised=${summary.rowsNormalised} daysRecomputed=${summary.daysRecomputed}`,
@@ -797,6 +878,17 @@ export async function registerMaintenanceQueues(
throw err;
}
}
return jobDone({
jobs: jobs.length,
users,
discovery_enqueued: discoveryEnqueued,
discovery_skipped: discoverySkipped,
discovery_failed: discoveryFailed,
slots_collapsed: slotsCollapsed,
rows_soft_deleted: rowsSoftDeleted,
rows_normalised: rowsNormalised,
days_recomputed: daysRecomputed,
});
},
);
@@ -806,14 +898,26 @@ export async function registerMaintenanceQueues(
// empty-userId payload is the daily discovery tick and re-fans the per-user
// jobs (singletonKey-coalesced, identical to the boot pass). Serial
// concurrency so the migration never crowds the request pool.
await boss.work<Partial<NoteEncryptionBackfillPayload>>(
await createAndWork<Partial<NoteEncryptionBackfillPayload>>(
boss,
NOTE_ENCRYPTION_BACKFILL_QUEUE,
{ localConcurrency: NOTE_ENCRYPTION_BACKFILL_CONCURRENCY },
async (jobs) => {
let users = 0;
let discoveryEnqueued = 0;
let discoverySkipped = 0;
// The discovery helper returns its error instead of throwing, so a
// failed fan-out would otherwise leave no trace on the job at all.
let discoveryFailed = 0;
let measurements = 0;
let moodEntries = 0;
for (const job of jobs) {
const { userId } = job.data;
if (!userId) {
const result = await enqueueBootTimeNoteEncryptionBackfill();
discoveryEnqueued += result.enqueued;
discoverySkipped += result.skipped;
if (result.error) discoveryFailed++;
workerLog(
"info",
`[note-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`,
@@ -823,6 +927,9 @@ export async function registerMaintenanceQueues(
try {
const { measurementsMigrated, moodEntriesMigrated } =
await runNoteEncryptionBackfillForUser(userId);
users++;
measurements += measurementsMigrated;
moodEntries += moodEntriesMigrated;
workerLog(
"info",
`[note-encryption-backfill] user=${userId} measurements=${measurementsMigrated} moodEntries=${moodEntriesMigrated}`,
@@ -837,11 +944,21 @@ export async function registerMaintenanceQueues(
throw err;
}
}
return jobDone({
jobs: jobs.length,
users,
discovery_enqueued: discoveryEnqueued,
discovery_skipped: discoverySkipped,
discovery_failed: discoveryFailed,
measurements_migrated: measurements,
mood_entries_migrated: moodEntries,
});
},
);
// v1.23 — admin-triggered encryption-key rotation (on-demand, singleton).
await boss.work<EncryptionKeyRotatePayload>(
await createAndWork<EncryptionKeyRotatePayload>(
boss,
ENCRYPTION_KEY_ROTATE_QUEUE,
{ localConcurrency: ENCRYPTION_KEY_ROTATE_CONCURRENCY },
handleEncryptionKeyRotate,
@@ -854,14 +971,27 @@ export async function registerMaintenanceQueues(
// payload is the daily discovery tick and re-fans the per-user jobs
// (singletonKey-coalesced, identical to the boot pass). Serial concurrency
// so the migration never crowds the request pool.
await boss.work<Partial<MedNotesEncryptionBackfillPayload>>(
await createAndWork<Partial<MedNotesEncryptionBackfillPayload>>(
boss,
MED_NOTES_ENCRYPTION_BACKFILL_QUEUE,
{ localConcurrency: MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY },
async (jobs) => {
let users = 0;
let discoveryEnqueued = 0;
let discoverySkipped = 0;
// The discovery helper returns its error instead of throwing, so a
// failed fan-out would otherwise leave no trace on the job at all.
let discoveryFailed = 0;
let sideEffects = 0;
let doseChanges = 0;
let inventoryItems = 0;
for (const job of jobs) {
const { userId } = job.data;
if (!userId) {
const result = await enqueueBootTimeMedNotesEncryptionBackfill();
discoveryEnqueued += result.enqueued;
discoverySkipped += result.skipped;
if (result.error) discoveryFailed++;
workerLog(
"info",
`[med-notes-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`,
@@ -874,6 +1004,10 @@ export async function registerMaintenanceQueues(
doseChangesMigrated,
inventoryItemsMigrated,
} = await runMedNotesEncryptionBackfillForUser(userId);
users++;
sideEffects += sideEffectsMigrated;
doseChanges += doseChangesMigrated;
inventoryItems += inventoryItemsMigrated;
workerLog(
"info",
`[med-notes-encryption-backfill] user=${userId} sideEffects=${sideEffectsMigrated} doseChanges=${doseChangesMigrated} inventoryItems=${inventoryItemsMigrated}`,
@@ -888,6 +1022,16 @@ export async function registerMaintenanceQueues(
throw err;
}
}
return jobDone({
jobs: jobs.length,
users,
discovery_enqueued: discoveryEnqueued,
discovery_skipped: discoverySkipped,
discovery_failed: discoveryFailed,
side_effects_migrated: sideEffects,
dose_changes_migrated: doseChanges,
inventory_items_migrated: inventoryItems,
});
},
);
@@ -895,16 +1039,25 @@ export async function registerMaintenanceQueues(
// trigger endpoint sends one per-user job; this handler indexes that user's
// not-yet-indexed documents (consent + budget gated, bounded + resumable).
// Serial concurrency so the provider calls never crowd the request pool.
await boss.work<ContentIndexBackfillPayload>(
await createAndWork<ContentIndexBackfillPayload>(
boss,
CONTENT_INDEX_BACKFILL_QUEUE,
{ localConcurrency: CONTENT_INDEX_BACKFILL_CONCURRENCY },
async (jobs) => {
let users = 0;
let skipped = 0;
let documentsIndexed = 0;
for (const job of jobs) {
const { userId } = job.data;
if (!userId) continue;
if (!userId) {
skipped++;
continue;
}
try {
const { indexed, reason } =
await runContentIndexBackfillForUser(userId);
users++;
documentsIndexed += indexed;
workerLog(
"info",
`[document-content-index-backfill] user=${userId} indexed=${indexed} reason=${reason}`,
@@ -919,6 +1072,12 @@ export async function registerMaintenanceQueues(
throw err;
}
}
return jobDone({
jobs: jobs.length,
users,
skipped,
documents_indexed: documentsIndexed,
});
},
);
@@ -926,15 +1085,22 @@ export async function registerMaintenanceQueues(
// (one job per stored document); provider-first (vision) when configured +
// consented, else local text-layer extraction. Serial concurrency so the
// provider calls + PDF parse never crowd the request pool.
await boss.work<DocumentIndexPayload>(
await createAndWork<DocumentIndexPayload>(
boss,
DOCUMENT_INDEX_QUEUE,
{ localConcurrency: DOCUMENT_INDEX_CONCURRENCY },
async (jobs) => {
let documents = 0;
let skipped = 0;
for (const job of jobs) {
const { userId, documentId } = job.data;
if (!userId || !documentId) continue;
if (!userId || !documentId) {
skipped++;
continue;
}
try {
await runDocumentIndex(job.data);
documents++;
} catch (err) {
recordError();
workerLog(
@@ -945,6 +1111,7 @@ export async function registerMaintenanceQueues(
throw err;
}
}
return jobDone({ jobs: jobs.length, documents, skipped });
},
);
@@ -952,15 +1119,22 @@ export async function registerMaintenanceQueues(
// upload (one job per stored document); decodes + downscales the original to
// a small encrypted JPEG. Pure local compute; serial concurrency so the
// canvas/pdfjs decode never crowds the request pool.
await boss.work<DocumentThumbnailPayload>(
await createAndWork<DocumentThumbnailPayload>(
boss,
DOCUMENT_THUMBNAIL_QUEUE,
{ localConcurrency: DOCUMENT_THUMBNAIL_CONCURRENCY },
async (jobs) => {
let documents = 0;
let skipped = 0;
for (const job of jobs) {
const { userId, documentId } = job.data;
if (!userId || !documentId) continue;
if (!userId || !documentId) {
skipped++;
continue;
}
try {
await runDocumentThumbnail(job.data);
documents++;
} catch (err) {
recordError();
workerLog(
@@ -971,6 +1145,7 @@ export async function registerMaintenanceQueues(
throw err;
}
}
return jobDone({ jobs: jobs.length, documents, skipped });
},
);
@@ -979,15 +1154,22 @@ export async function registerMaintenanceQueues(
// the `documentsAutoAiRead` opt-in is ON (egress consent + budget gated) and
// persists the summary encrypted. Serial concurrency so the provider call
// never crowds the request pool.
await boss.work<DocumentSummaryPayload>(
await createAndWork<DocumentSummaryPayload>(
boss,
DOCUMENT_SUMMARY_QUEUE,
{ localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY },
async (jobs) => {
let documents = 0;
let skipped = 0;
for (const job of jobs) {
const { userId, documentId } = job.data;
if (!userId || !documentId) continue;
if (!userId || !documentId) {
skipped++;
continue;
}
try {
await runDocumentSummaryJob(job.data);
documents++;
} catch (err) {
recordError();
workerLog(
@@ -998,6 +1180,7 @@ export async function registerMaintenanceQueues(
throw err;
}
}
return jobDone({ jobs: jobs.length, documents, skipped });
},
);
@@ -1007,15 +1190,24 @@ export async function registerMaintenanceQueues(
// documents (the upload-time enqueue no-opped while the flag was OFF). It
// only enqueues — the opt-in, egress consent and budget gates all still run
// per document inside the summary job. Serial concurrency.
await boss.work<SummaryCatchUpPayload>(
await createAndWork<SummaryCatchUpPayload>(
boss,
DOCUMENT_SUMMARY_CATCHUP_QUEUE,
{ localConcurrency: DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY },
async (jobs) => {
let users = 0;
let skipped = 0;
let summariesEnqueued = 0;
for (const job of jobs) {
const { userId } = job.data;
if (!userId) continue;
if (!userId) {
skipped++;
continue;
}
try {
await runSummaryCatchUpForUser(userId);
const { enqueued } = await runSummaryCatchUpForUser(userId);
users++;
summariesEnqueued += enqueued;
} catch (err) {
recordError();
workerLog(
@@ -1026,6 +1218,12 @@ export async function registerMaintenanceQueues(
throw err;
}
}
return jobDone({
jobs: jobs.length,
users,
skipped,
summaries_enqueued: summariesEnqueued,
});
},
);
@@ -1033,15 +1231,24 @@ export async function registerMaintenanceQueues(
// discovery sends one per-user job; this handler fans out per-document
// thumbnail jobs for that user's not-yet-thumbnailed documents. Serial
// concurrency; cheap discovery + enqueue only.
await boss.work<ThumbnailBackfillPayload>(
await createAndWork<ThumbnailBackfillPayload>(
boss,
DOCUMENT_THUMBNAIL_BACKFILL_QUEUE,
{ localConcurrency: DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY },
async (jobs) => {
let users = 0;
let skipped = 0;
let thumbnailsEnqueued = 0;
for (const job of jobs) {
const { userId } = job.data;
if (!userId) continue;
if (!userId) {
skipped++;
continue;
}
try {
const { enqueued } = await runThumbnailBackfillForUser(userId);
users++;
thumbnailsEnqueued += enqueued;
workerLog(
"info",
`[document-thumbnail-backfill] user=${userId} enqueued=${enqueued}`,
@@ -1056,6 +1263,12 @@ export async function registerMaintenanceQueues(
throw err;
}
}
return jobDone({
jobs: jobs.length,
users,
skipped,
thumbnails_enqueued: thumbnailsEnqueued,
});
},
);
@@ -1063,13 +1276,20 @@ export async function registerMaintenanceQueues(
// payload) fans out one per-user job per opted-in account; the queue also
// serves the on-demand backfill payloads from the settings surface. Serial
// concurrency so the staggered outbound fetches never crowd the request pool.
await boss.work<EnvironmentFetchPayload>(
await createAndWork<EnvironmentFetchPayload>(
boss,
ENVIRONMENT_FETCH_QUEUE,
{ localConcurrency: ENVIRONMENT_FETCH_CONCURRENCY },
async (jobs) => {
// A discovery tick and a per-user backfill share this queue and are
// independent of each other, so the pass runs to the end and reports
// the first failure it saw.
let failure: JobOutcome | null = null;
for (const job of jobs) {
await handleEnvironmentFetch(boss, job.data);
const outcome = await handleEnvironmentFetch(boss, job.data);
if (!outcome.ok && failure === null) failure = outcome;
}
return failure ?? jobDone({ jobs: jobs.length });
},
);
+22 -7
View File
@@ -7,9 +7,15 @@
*
* v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its
* cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`,
* and a `boss.work(QUEUE, …, handler)` binding drains it. The
* and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The
* `cycle-reminder-queue` and `measurement-reminder-queue` guards read THIS
* module.
*
* Retry policy: every queue here keeps the pg-boss default. These ticks are
* fan-out passes whose per-user failures never fail the job, so a job-level
* failure means the pass itself could not run — a transient the default two
* retries are the right answer to. The satisfy queue carries its own
* `retryLimit` at send time (see `reminder-satisfy.ts`).
*/
import { PgBoss } from "pg-boss";
import {
@@ -17,7 +23,11 @@ import {
REMINDER_SATISFY_CONCURRENCY,
type ReminderSatisfyPayload,
} from "@/lib/jobs/reminder-satisfy";
import { createAndSchedule, type ScheduleEntry } from "./registrar-shared";
import {
createAndSchedule,
createAndWork,
type ScheduleEntry,
} from "./registrar-shared";
import {
ReminderCheckPayload,
handleReminderCheck,
@@ -121,7 +131,8 @@ export async function registerReminderQueues(
): Promise<readonly string[]> {
await createAndSchedule(boss, allQueues, schedules);
await boss.work<ReminderCheckPayload>(
await createAndWork<ReminderCheckPayload>(
boss,
QUEUE_NAME,
{ localConcurrency: 1 },
handleReminderCheck,
@@ -130,7 +141,8 @@ export async function registerReminderQueues(
// two reminder ticks from interleaving against the same user row;
// the dedup ledger would still save us, but skipping the race here
// avoids spurious P2002 errors in the wide-event log.
await boss.work<MoodReminderPayload>(
await createAndWork<MoodReminderPayload>(
boss,
MOOD_REMINDER_QUEUE,
{ localConcurrency: 1 },
handleMoodReminderCheck,
@@ -138,7 +150,8 @@ export async function registerReminderQueues(
// v1.15 — single-flight cycle-reminder worker. localConcurrency=1 keeps
// two ticks from racing the fire-and-forget `push_attempts` ledger that
// anchors the per-day idempotency, exactly like the mood-reminder worker.
await boss.work<CycleReminderPayload>(
await createAndWork<CycleReminderPayload>(
boss,
CYCLE_REMINDER_QUEUE,
{ localConcurrency: 1 },
handleCycleReminderCheck,
@@ -147,7 +160,8 @@ export async function registerReminderQueues(
// localConcurrency=1 keeps two ticks from racing the `nextDueAt`
// advance that anchors the per-cycle idempotency, exactly like the
// mood / cycle reminder workers.
await boss.work<MeasurementReminderPayload>(
await createAndWork<MeasurementReminderPayload>(
boss,
MEASUREMENT_REMINDER_QUEUE,
{ localConcurrency: 1 },
handleMeasurementReminderCheck,
@@ -155,7 +169,8 @@ export async function registerReminderQueues(
// v1.18.1 — eventful Vorsorge satisfaction. Resolves a user's reminders
// against their just-landed measurement / lab. Read-heavy on the user's
// own data; the same small concurrency budget as PR detection.
await boss.work<ReminderSatisfyPayload>(
await createAndWork<ReminderSatisfyPayload>(
boss,
REMINDER_SATISFY_QUEUE,
{ localConcurrency: REMINDER_SATISFY_CONCURRENCY },
handleReminderSatisfy,
+171 -14
View File
@@ -10,7 +10,7 @@
*
* v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its
* cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`,
* and a `boss.work(QUEUE, …, handler)` binding drains it. The
* and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The
* `drain-cumulative-queue`, `mean-consolidation-queue`, and
* `stress-strain-retention-queue` (dense-retention facet) guards read THIS
* module.
@@ -104,9 +104,12 @@ import {
} from "./shared";
import {
createAndSchedule,
createAndWork,
cronIsTheRetry,
type QueuePolicyTable,
type ScheduleEntry,
} from "./registrar-shared";
import { jobDone, jobFailed } from "@/lib/jobs/job-outcome";
// v1.4.37 W7c — nightly drain of per-sample APPLE_HEALTH cumulative rows.
// Collapses each user × cumulative-type × calendar-day bucket into one
// `stats:…` row so the list view stops painting hundreds of step chunks per
@@ -170,7 +173,7 @@ const schedules: ScheduleEntry[] = [
// v1.4.37 W7c — nightly fold of per-sample APPLE_HEALTH cumulative
// rows into one row per day per type. Slots between the
// audit-log cleanup (03:15) and the feedback aggregator (04:00).
[DRAIN_CUMULATIVE_QUEUE, DRAIN_CUMULATIVE_CRON],
[DRAIN_CUMULATIVE_QUEUE, DRAIN_CUMULATIVE_CRON, cronIsTheRetry],
];
/**
@@ -274,7 +277,8 @@ export async function registerRollupQueues(
// the DAY bucket is already recomputed synchronously by the hook
// itself. Concurrency-2 keeps two recomputes in flight without
// crowding the dashboard request pool.
await boss.work<RollupRecomputePayload>(
await createAndWork<RollupRecomputePayload>(
boss,
ROLLUP_RECOMPUTE_QUEUE,
{ localConcurrency: ROLLUP_RECOMPUTE_CONCURRENCY },
async (jobs) => {
@@ -287,6 +291,10 @@ export async function registerRollupQueues(
to: new Date(payload.to),
});
}
// No per-bucket isolation here on purpose: a recompute that throws
// fails the job, so reaching this line means every bucket in the batch
// was folded.
return jobDone({ buckets_recomputed: jobs.length });
},
);
@@ -295,18 +303,25 @@ export async function registerRollupQueues(
// full `recomputeUserRollups` against the default 5-year window
// across every granularity. Serial concurrency so the populator
// never crowds the dashboard request pool.
await boss.work<RollupFullBackfillPayload>(
await createAndWork<RollupFullBackfillPayload>(
boss,
ROLLUP_FULL_BACKFILL_QUEUE,
{ localConcurrency: ROLLUP_FULL_BACKFILL_CONCURRENCY },
async (jobs) => {
let rowsUpsertedTotal = 0;
for (const job of jobs) {
const { userId } = job.data;
const { rowsUpserted, durationMs } = await recomputeUserRollups(userId);
rowsUpsertedTotal += rowsUpserted;
workerLog(
"info",
`[rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`,
);
}
return jobDone({
users: jobs.length,
rows_upserted: rowsUpsertedTotal,
});
},
);
@@ -315,19 +330,29 @@ export async function registerRollupQueues(
// step rows; this handler collapses them into one daily-total row per
// calendar day and soft-deletes the originals. Serial concurrency so
// the populator never crowds the dashboard request pool.
await boss.work<StepConsolidationPayload>(
await createAndWork<StepConsolidationPayload>(
boss,
STEP_CONSOLIDATION_QUEUE,
{ localConcurrency: STEP_CONSOLIDATION_CONCURRENCY },
async (jobs) => {
let daysTotal = 0;
let legacyRowsTotal = 0;
for (const job of jobs) {
const { userId } = job.data;
const { daysConsolidated, legacyRowsSoftDeleted } =
await runStepConsolidationForUser(userId);
daysTotal += daysConsolidated;
legacyRowsTotal += legacyRowsSoftDeleted;
workerLog(
"info",
`[step-consolidation] user=${userId} days=${daysConsolidated} legacyRowsSoftDeleted=${legacyRowsSoftDeleted}`,
);
}
return jobDone({
users: jobs.length,
days_consolidated: daysTotal,
legacy_rows_soft_deleted: legacyRowsTotal,
});
},
);
@@ -335,18 +360,39 @@ export async function registerRollupQueues(
// GOOGLE_HEALTH / FITBIT daily-total step rows the pre-fix consolidation
// swept and removes the shadow MANUAL totals it minted. Serial
// concurrency so the repair never crowds the dashboard request pool.
await boss.work<StepConsolidationRepairPayload>(
await createAndWork<StepConsolidationRepairPayload>(
boss,
STEP_CONSOLIDATION_REPAIR_QUEUE,
{ localConcurrency: STEP_CONSOLIDATION_REPAIR_CONCURRENCY },
async (jobs) => {
let rowsResurrected = 0;
let wedgeSkipped = 0;
let manualMintsRemoved = 0;
let daysRecomputed = 0;
let failures = 0;
for (const job of jobs) {
const { userId } = job.data;
const summary = await runStepConsolidationRepairForUser(userId);
rowsResurrected += summary.rowsResurrected;
wedgeSkipped += summary.wedgeSkipped;
manualMintsRemoved += summary.manualMintsRemoved;
daysRecomputed += summary.daysRecomputed;
// The repair isolates its own per-day failures and reports them as a
// count, so they ride out as a fact rather than failing the job.
failures += summary.failures;
workerLog(
"info",
`[step-consolidation-repair] user=${userId} resurrected=${summary.rowsResurrected} wedgeSkipped=${summary.wedgeSkipped} manualMintsRemoved=${summary.manualMintsRemoved} daysRecomputed=${summary.daysRecomputed} failures=${summary.failures}`,
);
}
return jobDone({
users: jobs.length,
rows_resurrected: rowsResurrected,
wedge_skipped: wedgeSkipped,
manual_mints_removed: manualMintsRemoved,
days_recomputed: daysRecomputed,
failures,
});
},
);
@@ -356,18 +402,28 @@ export async function registerRollupQueues(
// it and re-runs detection silently so the honest re-derived best takes
// its place. Serial concurrency so the repair never crowds the
// dashboard request pool.
await boss.work<CumulativePrRederivePayload>(
await createAndWork<CumulativePrRederivePayload>(
boss,
CUMULATIVE_PR_REDERIVE_QUEUE,
{ localConcurrency: CUMULATIVE_PR_REDERIVE_CONCURRENCY },
async (jobs) => {
let rowsDeleted = 0;
let rowsReinserted = 0;
for (const job of jobs) {
const { userId } = job.data;
const summary = await runCumulativePrRederivationForUser(userId);
rowsDeleted += summary.rowsDeleted;
rowsReinserted += summary.rowsReinserted;
workerLog(
"info",
`[cumulative-pr-rederive] user=${userId} rowsDeleted=${summary.rowsDeleted} rowsReinserted=${summary.rowsReinserted}`,
);
}
return jobDone({
users: jobs.length,
rows_deleted: rowsDeleted,
rows_reinserted: rowsReinserted,
});
},
);
@@ -376,19 +432,29 @@ export async function registerRollupQueues(
// mean-type rows; this handler collapses each completed day to its
// mean and soft-deletes the originals. Serial concurrency so the
// populator never crowds the dashboard request pool.
await boss.work<MeanConsolidationPayload>(
await createAndWork<MeanConsolidationPayload>(
boss,
MEAN_CONSOLIDATION_QUEUE,
{ localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY },
async (jobs) => {
let daysTotal = 0;
let perSampleRowsTotal = 0;
for (const job of jobs) {
const { userId } = job.data;
const { daysConsolidated, perSampleRowsSoftDeleted } =
await runMeanConsolidationForUser(userId);
daysTotal += daysConsolidated;
perSampleRowsTotal += perSampleRowsSoftDeleted;
workerLog(
"info",
`[mean-consolidation] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted}`,
);
}
return jobDone({
users: jobs.length,
days_consolidated: daysTotal,
per_sample_rows_soft_deleted: perSampleRowsTotal,
});
},
);
@@ -399,10 +465,14 @@ export async function registerRollupQueues(
// soft-deletes the originals, keeping the in-window intra-day shape intact
// for the Stress engine. Serial concurrency so the backfill never crowds
// the request pool.
await boss.work<DenseIntradayRetentionPayload>(
await createAndWork<DenseIntradayRetentionPayload>(
boss,
DENSE_INTRADAY_RETENTION_QUEUE,
{ localConcurrency: DENSE_INTRADAY_RETENTION_CONCURRENCY },
async (jobs) => {
let daysTotal = 0;
let perSampleRowsTotal = 0;
let derivedRestingRowsTotal = 0;
for (const job of jobs) {
const { userId } = job.data;
try {
@@ -411,6 +481,9 @@ export async function registerRollupQueues(
perSampleRowsSoftDeleted,
derivedRestingRowsUpserted,
} = await runDenseIntradayRetentionForUser(userId);
daysTotal += daysConsolidated;
perSampleRowsTotal += perSampleRowsSoftDeleted;
derivedRestingRowsTotal += derivedRestingRowsUpserted;
workerLog(
"info",
`[dense-intraday-retention] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted} derivedRestingRowsUpserted=${derivedRestingRowsUpserted}`,
@@ -425,6 +498,12 @@ export async function registerRollupQueues(
throw err;
}
}
return jobDone({
users: jobs.length,
days_consolidated: daysTotal,
per_sample_rows_soft_deleted: perSampleRowsTotal,
derived_resting_rows_upserted: derivedRestingRowsTotal,
});
},
);
@@ -435,10 +514,15 @@ export async function registerRollupQueues(
// Self-converging: a rebuilt day drops out of the discovery pairing, so
// the pass runs once per install. Serial concurrency so the rebuild never
// crowds the request pool.
await boss.work<DenseIntradayHourlyRebuildPayload>(
await createAndWork<DenseIntradayHourlyRebuildPayload>(
boss,
DENSE_INTRADAY_HOURLY_REBUILD_QUEUE,
{ localConcurrency: DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY },
async (jobs) => {
let daysRebuiltTotal = 0;
let hourlyRowsTotal = 0;
let dailyRowsRetiredTotal = 0;
let daysSkippedTotal = 0;
for (const job of jobs) {
const { userId } = job.data;
try {
@@ -448,6 +532,10 @@ export async function registerRollupQueues(
dailyRowsRetired,
daysSkippedNoTombstones,
} = await runDenseIntradayHourlyRebuildForUser(userId);
daysRebuiltTotal += daysRebuilt;
hourlyRowsTotal += hourlyRowsUpserted;
dailyRowsRetiredTotal += dailyRowsRetired;
daysSkippedTotal += daysSkippedNoTombstones;
workerLog(
"info",
`[dense-intraday-hourly-rebuild] user=${userId} daysRebuilt=${daysRebuilt} hourlyRowsUpserted=${hourlyRowsUpserted} dailyRowsRetired=${dailyRowsRetired} daysSkippedNoTombstones=${daysSkippedNoTombstones}`,
@@ -462,6 +550,13 @@ export async function registerRollupQueues(
throw err;
}
}
return jobDone({
users: jobs.length,
days_rebuilt: daysRebuiltTotal,
hourly_rows_upserted: hourlyRowsTotal,
daily_rows_retired: dailyRowsRetiredTotal,
days_skipped_no_tombstones: daysSkippedTotal,
});
},
);
@@ -471,7 +566,8 @@ export async function registerRollupQueues(
// No current read path consumes these buckets — they exist so a
// future cross-granularity reader can ship without a backfill
// step. Concurrency-2 mirrors the measurement-rollup worker.
await boss.work<MoodRollupRecomputePayload>(
await createAndWork<MoodRollupRecomputePayload>(
boss,
MOOD_ROLLUP_RECOMPUTE_QUEUE,
{ localConcurrency: MOOD_ROLLUP_RECOMPUTE_CONCURRENCY },
async (jobs) => {
@@ -483,6 +579,7 @@ export async function registerRollupQueues(
to: new Date(payload.to),
});
}
return jobDone({ buckets_recomputed: jobs.length });
},
);
@@ -491,10 +588,12 @@ export async function registerRollupQueues(
// but zero rollup rows; this handler folds the full 5-year window
// across every granularity. Concurrency-1 so the populator never
// crowds the request pool.
await boss.work<MoodRollupFullBackfillPayload>(
await createAndWork<MoodRollupFullBackfillPayload>(
boss,
MOOD_ROLLUP_FULL_BACKFILL_QUEUE,
{ localConcurrency: MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY },
async (jobs) => {
let rowsUpsertedTotal = 0;
for (const job of jobs) {
const { userId } = job.data;
try {
@@ -508,6 +607,7 @@ export async function registerRollupQueues(
userId,
{ replace: true },
);
rowsUpsertedTotal += rowsUpserted;
workerLog(
"info",
`[mood-rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`,
@@ -522,6 +622,10 @@ export async function registerRollupQueues(
throw err;
}
}
return jobDone({
users: jobs.length,
rows_upserted: rowsUpsertedTotal,
});
},
);
@@ -530,15 +634,18 @@ export async function registerRollupQueues(
// but zero rollup coverage; this handler folds the trailing 90-day
// window per account. Concurrency-1 so the populator never crowds
// the request pool.
await boss.work<MedicationComplianceBackfillPayload>(
await createAndWork<MedicationComplianceBackfillPayload>(
boss,
MEDICATION_COMPLIANCE_BACKFILL_QUEUE,
{ localConcurrency: MEDICATION_COMPLIANCE_BACKFILL_CONCURRENCY },
async (jobs) => {
let rowsUpsertedTotal = 0;
for (const job of jobs) {
const { userId } = job.data;
try {
const { rowsUpserted, durationMs } =
await recomputeUserMedicationCompliance(userId);
rowsUpsertedTotal += rowsUpserted;
workerLog(
"info",
`[medication-compliance-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`,
@@ -553,6 +660,10 @@ export async function registerRollupQueues(
throw err;
}
}
return jobDone({
users: jobs.length,
rows_upserted: rowsUpsertedTotal,
});
},
);
@@ -563,10 +674,26 @@ export async function registerRollupQueues(
// Concurrency-1 so the drain never crowds the dashboard request pool
// and a long backfill on the maintainer's account (300 k+ measurement rows)
// stays a single sequential walk.
await boss.work<DrainCumulativePayload>(
await createAndWork<DrainCumulativePayload>(
boss,
DRAIN_CUMULATIVE_QUEUE,
{ localConcurrency: 1 },
async (jobs) => {
// The three drains below are independent passes sharing one nightly
// tick, so one failing pass must not stop the other two from running.
// The first failure is held back and reported once the tick is over —
// that keeps the existing order and still fails the job, which is what
// the swallowed errors never did.
let firstFailure: { reason: string; cause: unknown } | undefined;
let passesFailed = 0;
let bucketsCollapsed = 0;
let perSampleRowsDeleted = 0;
let meanDaysConsolidated = 0;
let meanRowsSoftDeleted = 0;
let denseDaysConsolidated = 0;
let denseRowsSoftDeleted = 0;
let denseSkipped = false;
for (const job of jobs) {
try {
const summary = await drainPerSampleCumulative(getWorkerPrisma(), {
@@ -574,6 +701,8 @@ export async function registerRollupQueues(
cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS,
log: (line) => workerLog("info", line),
});
bucketsCollapsed += summary.totals.bucketsCollapsed;
perSampleRowsDeleted += summary.totals.perSampleRowsDeleted;
workerLog(
"info",
`[drain-cumulative] triggeredAt=${job.data.triggeredAt} usersScanned=${summary.totals.usersScanned} bucketsCollapsed=${summary.totals.bucketsCollapsed} perSampleRowsDeleted=${summary.totals.perSampleRowsDeleted} dailyRowsUpserted=${summary.totals.dailyRowsUpserted}`,
@@ -581,6 +710,8 @@ export async function registerRollupQueues(
} catch (err) {
recordError();
await reportWorkerError("drain-cumulative", err);
passesFailed++;
firstFailure ??= { reason: "cumulative drain failed", cause: err };
}
// v1.8.5 — fold the daily-MEAN drain onto the same nightly tick as
@@ -600,6 +731,8 @@ export async function registerRollupQueues(
cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS,
log: (line) => workerLog("info", line),
});
meanDaysConsolidated += meanSummary.totals.daysConsolidated;
meanRowsSoftDeleted += meanSummary.totals.perSampleRowsSoftDeleted;
workerLog(
"info",
`[mean-consolidation] triggeredAt=${job.data.triggeredAt} usersScanned=${meanSummary.totals.usersScanned} daysConsolidated=${meanSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${meanSummary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${meanSummary.totals.dailyRowsUpserted} daysFailed=${meanSummary.totals.daysFailed}`,
@@ -609,6 +742,8 @@ export async function registerRollupQueues(
await reportWorkerError("mean-consolidation", err, {
tick: "nightly",
});
passesFailed++;
firstFailure ??= { reason: "mean consolidation failed", cause: err };
}
// v1.10.0 WX-E — fold the dense intra-day retention drain onto the
@@ -624,6 +759,7 @@ export async function registerRollupQueues(
// shipped.
try {
if (!DENSE_INTRADAY_RETENTION_ENABLED) {
denseSkipped = true;
workerLog(
"info",
"[dense-intraday-retention] disabled via DENSE_INTRADAY_RETENTION_ENABLED — skipping the nightly walk (operator kill-switch)",
@@ -636,6 +772,9 @@ export async function registerRollupQueues(
log: (line) => workerLog("info", line),
},
);
denseDaysConsolidated += denseSummary.totals.daysConsolidated;
denseRowsSoftDeleted +=
denseSummary.totals.perSampleRowsSoftDeleted;
workerLog(
"info",
`[dense-intraday-retention] triggeredAt=${job.data.triggeredAt} usersScanned=${denseSummary.totals.usersScanned} daysConsolidated=${denseSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${denseSummary.totals.perSampleRowsSoftDeleted} hourlyRowsUpserted=${denseSummary.totals.hourlyRowsUpserted} dailyRowsRetired=${denseSummary.totals.dailyRowsRetired}`,
@@ -646,8 +785,26 @@ export async function registerRollupQueues(
await reportWorkerError("dense-intraday-retention", err, {
tick: "nightly",
});
passesFailed++;
firstFailure ??= { reason: "dense retention failed", cause: err };
}
}
const did = {
passes_failed: passesFailed,
buckets_collapsed: bucketsCollapsed,
per_sample_rows_deleted: perSampleRowsDeleted,
mean_days_consolidated: meanDaysConsolidated,
mean_rows_soft_deleted: meanRowsSoftDeleted,
dense_days_consolidated: denseDaysConsolidated,
dense_rows_soft_deleted: denseRowsSoftDeleted,
dense_skipped: denseSkipped,
};
if (firstFailure) {
return jobFailed(firstFailure.reason, firstFailure.cause, did);
}
return jobDone(did);
},
);
+254 -117
View File
@@ -10,8 +10,8 @@
*
* v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its
* cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`,
* and a `boss.work(QUEUE, …, handler)` binding drains it. The status-wiring
* guards (`status-cron-candidates`, `insight-pregenerate`,
* and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The
* status-wiring guards (`status-cron-candidates`, `insight-pregenerate`,
* `insight-status-generate`, `recovery-score-queue`,
* `stress-strain-retention-queue`, `period-narrative-warm`,
* `coach-memory-refresh-queue`) read THIS module.
@@ -101,9 +101,11 @@ import {
type InsightStatusGeneratePayload,
} from "@/lib/jobs/insight-status-generate";
import { withBackgroundEvent } from "@/lib/logging/background";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { getWorkerPrisma, workerLog } from "./shared";
import {
createAndSchedule,
createAndWork,
insightRetryOptions,
type QueuePolicyTable,
type ScheduleEntry,
@@ -363,37 +365,44 @@ export async function registerStatusQueues(
): Promise<readonly string[]> {
await createAndSchedule(boss, allQueues, schedules, queuePolicies);
await boss.work<GeneralStatusPayload>(
await createAndWork<GeneralStatusPayload>(
boss,
GENERAL_STATUS_QUEUE,
{ localConcurrency: 1 },
handleGeneralStatusGenerate,
);
await boss.work<BloodPressureStatusPayload>(
await createAndWork<BloodPressureStatusPayload>(
boss,
BLOOD_PRESSURE_STATUS_QUEUE,
{ localConcurrency: 1 },
handleBloodPressureStatusGenerate,
);
await boss.work<WeightStatusPayload>(
await createAndWork<WeightStatusPayload>(
boss,
WEIGHT_STATUS_QUEUE,
{ localConcurrency: 1 },
handleWeightStatusGenerate,
);
await boss.work<PulseStatusPayload>(
await createAndWork<PulseStatusPayload>(
boss,
PULSE_STATUS_QUEUE,
{ localConcurrency: 1 },
handlePulseStatusGenerate,
);
await boss.work<BmiStatusPayload>(
await createAndWork<BmiStatusPayload>(
boss,
BMI_STATUS_QUEUE,
{ localConcurrency: 1 },
handleBmiStatusGenerate,
);
await boss.work<MoodStatusPayload>(
await createAndWork<MoodStatusPayload>(
boss,
MOOD_STATUS_QUEUE,
{ localConcurrency: 1 },
handleMoodStatusGenerate,
);
await boss.work<MedicationComplianceStatusPayload>(
await createAndWork<MedicationComplianceStatusPayload>(
boss,
MEDICATION_COMPLIANCE_STATUS_QUEUE,
{ localConcurrency: 1 },
handleMedicationComplianceStatusGenerate,
@@ -409,7 +418,8 @@ export async function registerStatusQueues(
// (the cohort walk is itself sequential per user, and the content-hash
// gate + per-user budget gate inside the handler cover the rare
// double-tick overlap the old single slot serialised away).
await boss.work<InsightPregeneratePayload>(
await createAndWork<InsightPregeneratePayload>(
boss,
INSIGHT_PREGENERATE_QUEUE,
{ localConcurrency: 2 },
handleInsightPregenerateJob,
@@ -420,17 +430,23 @@ export async function registerStatusQueues(
// night's samples into one job, and the handler's marker re-check makes a
// rare double-fire a no-op, so one slot is enough and bounds the provider
// concurrency the same way the nightly path does.
await boss.work<MorningDigestRefreshPayload>(
await createAndWork<MorningDigestRefreshPayload>(
boss,
MORNING_DIGEST_REFRESH_QUEUE,
{ localConcurrency: 1 },
async (jobs) => {
await withBackgroundEvent("job.morning_digest_refresh", async (evt) => {
async (jobs): Promise<JobOutcome> => {
return withBackgroundEvent("job.morning_digest_refresh", async (evt) => {
// The counts the batch already decides on: one refresh per job, and
// the finalised subset that reaches the morning push seam.
let refreshed = 0;
let finalised = 0;
for (const job of jobs) {
try {
const result = await runMorningDigestRefresh(
getWorkerPrisma(),
job.data,
);
refreshed++;
evt.addMeta(
"morning_refresh",
`${result.status}:${result.comprehensive ?? "none"}`,
@@ -442,6 +458,7 @@ export async function registerStatusQueues(
// there, so a user who never opted in or is outside the window is a
// clean no-op. The fallback cron covers the no-sleep-arrived case.
if (result.status === "finalised") {
finalised++;
const pushResult = await maybeDispatchDailyBriefing(
getWorkerPrisma(),
job.data.userId,
@@ -457,6 +474,7 @@ export async function registerStatusQueues(
throw err;
}
}
return jobDone({ refreshed, finalised });
});
},
);
@@ -464,7 +482,8 @@ export async function registerStatusQueues(
// read-only status route on a cold card. Low concurrency so a first
// visit that cold-misses several cards can't saturate the Prisma pool
// or fan out an unbounded number of concurrent provider calls.
await boss.work<InsightStatusGeneratePayload>(
await createAndWork<InsightStatusGeneratePayload>(
boss,
INSIGHT_STATUS_GENERATE_QUEUE,
{ localConcurrency: INSIGHT_STATUS_GENERATE_CONCURRENCY },
handleInsightStatusGenerate,
@@ -474,56 +493,84 @@ export async function registerStatusQueues(
// eligible user and upserts one `COMPUTED RECOVERY_SCORE` row per scored
// day (idempotent — a re-fire overwrites in place). Single-flight so two
// ticks never double-walk the cohort.
await boss.work(RECOVERY_SCORE_QUEUE, { localConcurrency: 1 }, async () => {
try {
const summary = await runRecoveryScore(getWorkerPrisma());
workerLog(
"info",
`[recovery-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`,
);
} catch (err) {
recordError();
workerLog("error", "[recovery-score] pass failed", err);
throw err;
}
});
await createAndWork(
boss,
RECOVERY_SCORE_QUEUE,
{ localConcurrency: 1 },
async (): Promise<JobOutcome> => {
try {
const summary = await runRecoveryScore(getWorkerPrisma());
workerLog(
"info",
`[recovery-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`,
);
return jobDone({
considered: summary.considered,
stored: summary.stored,
insufficient: summary.insufficient,
errored: summary.errored,
});
} catch (err) {
recordError();
workerLog("error", "[recovery-score] pass failed", err);
throw err;
}
},
);
// v1.15.20 — proactive Coach nudge. Single-flight; the push-attempts
// ledger caps a user at one nudge per rolling week, so an overlapping
// tick would only waste reads. Deterministic triggers, no AI call.
await boss.work(COACH_NUDGE_QUEUE, { localConcurrency: 1 }, async () => {
await withBackgroundEvent("job.coach_nudge", async (evt) => {
try {
const summary = await runCoachNudgeTick(getWorkerPrisma(), new Date());
evt.setBackground({
task_name: "job.coach_nudge",
result: {
await createAndWork(
boss,
COACH_NUDGE_QUEUE,
{ localConcurrency: 1 },
async (): Promise<JobOutcome> => {
return withBackgroundEvent("job.coach_nudge", async (evt) => {
try {
const summary = await runCoachNudgeTick(
getWorkerPrisma(),
new Date(),
);
evt.setBackground({
task_name: "job.coach_nudge",
result: {
candidates_scanned: summary.candidatesScanned,
dispatched: summary.dispatched,
persisted: summary.persisted,
skipped_opted_out: summary.skippedOptedOut,
skipped_no_provider: summary.skippedNoProvider,
skipped_recent_nudge: summary.skippedRecentNudge,
skipped_recent_engagement: summary.skippedRecentEngagement,
skipped_no_trigger: summary.skippedNoTrigger,
skipped_no_channel: summary.skippedNoChannel,
failed: summary.failed,
},
});
// A per-user failure never fails this pass — it rides out as the
// `failed` count, exactly as it does in the wide event.
return jobDone({
candidates_scanned: summary.candidatesScanned,
dispatched: summary.dispatched,
persisted: summary.persisted,
skipped_opted_out: summary.skippedOptedOut,
skipped_no_provider: summary.skippedNoProvider,
skipped_recent_nudge: summary.skippedRecentNudge,
skipped_recent_engagement: summary.skippedRecentEngagement,
skipped_no_trigger: summary.skippedNoTrigger,
skipped_no_channel: summary.skippedNoChannel,
failed: summary.failed,
},
});
} catch (err) {
evt.setError(err);
recordError();
throw err;
}
});
});
});
} catch (err) {
evt.setError(err);
recordError();
throw err;
}
});
},
);
// v1.16.11 — medication low-stock pass. Single-flight; the
// per-medication stamp makes a re-fire idempotent (already-notified
// crossings skip), so an overlapping tick would only waste reads.
await boss.work(
await createAndWork(
boss,
MEDICATION_LOW_STOCK_QUEUE,
{ localConcurrency: 1 },
async () => {
await withBackgroundEvent("job.medication_low_stock", async (evt) => {
async (): Promise<JobOutcome> => {
return withBackgroundEvent("job.medication_low_stock", async (evt) => {
try {
const summary = await runMedicationLowStockTick(
getWorkerPrisma(),
@@ -544,6 +591,15 @@ export async function registerStatusQueues(
failed: summary.failed,
},
});
// A per-medication failure never fails this pass — it rides out as
// the `failed` count.
return jobDone({
users_scanned: summary.usersScanned,
medications_evaluated: summary.medicationsEvaluated,
notified: summary.notified,
rearmed: summary.rearmed,
failed: summary.failed,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -556,7 +612,8 @@ export async function registerStatusQueues(
// to claim a durable unique row, so two slots would only race each other for
// a claim exactly one of them can win. It makes no provider call — a
// module-graph test pins that — so one slot costs nothing in throughput.
await boss.work<DataArrival>(
await createAndWork<DataArrival>(
boss,
DATA_ARRIVAL_QUEUE,
{ localConcurrency: 1 },
handleDataArrival,
@@ -564,7 +621,8 @@ export async function registerStatusQueues(
// v1.31.0 — the arrival reaction line. Single-flight: this is the spine's
// ONE provider call, and a second concurrent slot would only race the same
// durable `generatedAt` claim exactly one of them can win.
await boss.work<ReactionLineJob>(
await createAndWork<ReactionLineJob>(
boss,
REACTION_LINE_QUEUE,
{ localConcurrency: 1 },
handleReactionLine,
@@ -573,7 +631,8 @@ export async function registerStatusQueues(
// path the spine deliberately does not walk, so it stays off the request pool
// and cannot fan out concurrent completions when several workouts land in one
// sync.
await boss.work<WorkoutInsightGeneratePayload>(
await createAndWork<WorkoutInsightGeneratePayload>(
boss,
WORKOUT_INSIGHT_GENERATE_QUEUE,
{ localConcurrency: WORKOUT_INSIGHT_GENERATE_CONCURRENCY },
handleWorkoutInsightGenerate,
@@ -583,84 +642,131 @@ export async function registerStatusQueues(
// is suppressed), and the tick only loads a digest for opted-in users at
// their exact local fallback hour. No provider call — it reads the cached
// digest and dispatches the deterministic line + cached lead.
await boss.work(DAILY_BRIEFING_QUEUE, { localConcurrency: 1 }, async () => {
await withBackgroundEvent("job.daily_briefing", async (evt) => {
try {
const summary = await runDailyBriefingTick(
getWorkerPrisma(),
new Date(),
);
evt.setBackground({
task_name: "job.daily_briefing",
result: {
await createAndWork(
boss,
DAILY_BRIEFING_QUEUE,
{ localConcurrency: 1 },
async (): Promise<JobOutcome> => {
return withBackgroundEvent("job.daily_briefing", async (evt) => {
try {
const summary = await runDailyBriefingTick(
getWorkerPrisma(),
new Date(),
);
evt.setBackground({
task_name: "job.daily_briefing",
result: {
candidates_scanned: summary.candidatesScanned,
in_slot: summary.inSlot,
sent: summary.sent,
suppressed_frequency: summary.suppressedFrequency,
no_digest: summary.noDigest,
opted_out: summary.optedOut,
module_off: summary.moduleOff,
no_channel: summary.noChannel,
outside_window: summary.outsideWindow,
failed: summary.failed,
},
});
// Most ticks land outside every user's slot; that is the tick doing
// its job, so a zero `sent` is a success with zero counts.
return jobDone({
candidates_scanned: summary.candidatesScanned,
in_slot: summary.inSlot,
sent: summary.sent,
suppressed_frequency: summary.suppressedFrequency,
no_digest: summary.noDigest,
opted_out: summary.optedOut,
module_off: summary.moduleOff,
no_channel: summary.noChannel,
outside_window: summary.outsideWindow,
failed: summary.failed,
},
});
} catch (err) {
evt.setError(err);
recordError();
throw err;
}
});
});
});
} catch (err) {
evt.setError(err);
recordError();
throw err;
}
});
},
);
// v1.10.0 — computed scores (WX-E). Nightly Stress-score (HRV-derived
// proxy) compute + store. Single-flight so two ticks never double-walk
// the cohort. The runner iterates every eligible user and upserts one
// `COMPUTED STRESS_SCORE` row per scored day (idempotent — a re-fire
// overwrites in place).
await boss.work(STRESS_SCORE_QUEUE, { localConcurrency: 1 }, async () => {
try {
const summary = await runStressScore(getWorkerPrisma());
workerLog(
"info",
`[stress-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`,
);
} catch (err) {
recordError();
workerLog("error", "[stress-score] pass failed", err);
throw err;
}
});
await createAndWork(
boss,
STRESS_SCORE_QUEUE,
{ localConcurrency: 1 },
async (): Promise<JobOutcome> => {
try {
const summary = await runStressScore(getWorkerPrisma());
workerLog(
"info",
`[stress-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`,
);
return jobDone({
considered: summary.considered,
stored: summary.stored,
insufficient: summary.insufficient,
errored: summary.errored,
});
} catch (err) {
recordError();
workerLog("error", "[stress-score] pass failed", err);
throw err;
}
},
);
// v1.10.0 — computed scores (WX-E). Nightly Strain-score (Banister TRIMP
// cardio-load) compute + store. Single-flight; upserts one `COMPUTED
// STRAIN_SCORE` row per scored day (idempotent).
await boss.work(STRAIN_SCORE_QUEUE, { localConcurrency: 1 }, async () => {
try {
const summary = await runStrainScore(getWorkerPrisma());
workerLog(
"info",
`[strain-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`,
);
} catch (err) {
recordError();
workerLog("error", "[strain-score] pass failed", err);
throw err;
}
});
await createAndWork(
boss,
STRAIN_SCORE_QUEUE,
{ localConcurrency: 1 },
async (): Promise<JobOutcome> => {
try {
const summary = await runStrainScore(getWorkerPrisma());
workerLog(
"info",
`[strain-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`,
);
return jobDone({
considered: summary.considered,
stored: summary.stored,
insufficient: summary.insufficient,
errored: summary.errored,
});
} catch (err) {
recordError();
workerLog("error", "[strain-score] pass failed", err);
throw err;
}
},
);
// v1.11.0 — period-narrative warm. A scheduled tick (no `userId`) runs the
// boundary-gated nightly fan-out; a `userId` payload runs a single-user warm
// enqueued by the read-only GET on a cold/stale read. Single-flight so two
// ticks never double-walk the cohort; the per-user budget gate covers the
// fan-out and the enqueue `singletonKey` covers the single-user path.
await boss.work<PeriodNarrativePayload>(
await createAndWork<PeriodNarrativePayload>(
boss,
PERIOD_NARRATIVE_QUEUE,
{ localConcurrency: 1 },
async (jobs) => {
async (jobs): Promise<JobOutcome> => {
// Two modes on one queue, so the outcome counts them separately: a
// non-boundary night is a scheduled pass that generated nothing, which
// is a success with zero counts, not an idle queue.
let singleUser = 0;
let scheduled = 0;
let generated = 0;
let failed = 0;
for (const job of jobs) {
try {
if (job.data?.userId) {
await warmOneNarrative(job.data);
singleUser++;
} else {
const summary = await runPeriodNarrativeWarm(getWorkerPrisma());
scheduled++;
generated += summary.generated;
failed += summary.failed;
workerLog(
"info",
`[period-narrative] periods=${summary.periods.join(",") || "none"} total=${summary.total} generated=${summary.generated} cached=${summary.cached} skipped=${summary.skipped} insufficient=${summary.insufficient} failed=${summary.failed} budget=${summary.budgetBlocked}`,
@@ -674,26 +780,45 @@ export async function registerStatusQueues(
throw err;
}
}
return jobDone({
single_user: singleUser,
scheduled,
generated,
failed,
});
},
);
// v1.11.1 — combined Coach memory refresh: rolling conversation summary +
// durable fact extraction for one long conversation. localConcurrency 1 so a
// burst of long-conversation turns can't fan out concurrent provider calls;
// each step is budget-gated inside runStatusCompletion and fault-isolated.
await boss.work<CoachMemoryRefreshPayload>(
await createAndWork<CoachMemoryRefreshPayload>(
boss,
COACH_MEMORY_REFRESH_QUEUE,
{ localConcurrency: 1 },
async (jobs) => {
async (jobs): Promise<JobOutcome> => {
// An incomplete payload is skipped rather than failed, so it has to be
// countable — otherwise a queue full of them reads as a clean run.
let refreshed = 0;
let skippedIncomplete = 0;
for (const job of jobs) {
if (!job.data?.conversationId || !job.data?.userId) continue;
if (!job.data?.conversationId || !job.data?.userId) {
skippedIncomplete++;
continue;
}
try {
await runCoachMemoryRefresh(job.data);
refreshed++;
} catch (err) {
recordError();
workerLog("error", "[coach-memory-refresh] failed", err);
throw err;
}
}
return jobDone({
refreshed,
skipped_incomplete: skippedIncomplete,
});
},
);
@@ -701,11 +826,12 @@ export async function registerStatusQueues(
// reminder to `due` and minting plan-review reminders are both idempotent
// across re-fires (an already-due reminder is not re-flipped; a plan whose
// reviewDate was cleared is not re-minted). No provider call, no push.
await boss.work(
await createAndWork(
boss,
COACH_REMINDER_SWEEP_QUEUE,
{ localConcurrency: 1 },
async () => {
await withBackgroundEvent("job.coach_reminder_sweep", async (evt) => {
async (): Promise<JobOutcome> => {
return withBackgroundEvent("job.coach_reminder_sweep", async (evt) => {
try {
const summary = await runCoachReminderSweep(
getWorkerPrisma(),
@@ -719,6 +845,11 @@ export async function registerStatusQueues(
errored: summary.errored,
},
});
return jobDone({
reminders_due: summary.remindersDue,
plan_reviews_minted: summary.planReviewsMinted,
errored: summary.errored,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -732,11 +863,12 @@ export async function registerStatusQueues(
// reminders the sweep minted and writes the grounded before/after read-back
// into the plan's encrypted outcome column, flipping it to `reviewed`.
// Idempotent (a reviewed plan is not re-processed). No provider call, no push.
await boss.work(
await createAndWork(
boss,
COACH_PLAN_REVIEW_QUEUE,
{ localConcurrency: 1 },
async () => {
await withBackgroundEvent("job.coach_plan_review", async (evt) => {
async (): Promise<JobOutcome> => {
return withBackgroundEvent("job.coach_plan_review", async (evt) => {
try {
const summary = await runCoachPlanReviewTick(
getWorkerPrisma(),
@@ -750,6 +882,11 @@ export async function registerStatusQueues(
errored: summary.errored,
},
});
return jobDone({
reviewed: summary.reviewed,
insufficient: summary.insufficient,
errored: summary.errored,
});
} catch (err) {
evt.setError(err);
recordError();
+70 -1
View File
@@ -16,13 +16,18 @@
* assert a single aggregate `allQueues` (defence in depth: the per-registrar
* arrays AND the boot-level union both have to agree).
*/
import { PgBoss } from "pg-boss";
import { PgBoss, type WorkOptions } from "pg-boss";
import { prisma } from "@/lib/db";
import { workerLog } from "./shared";
import { withBackgroundEvent } from "@/lib/logging/background";
import { annotate } from "@/lib/logging/context";
import {
runJob,
type JobHandler,
type JobWithMetadataHandler,
} from "@/lib/jobs/run-job";
/**
* Cron schedule tuple: `[queueName, cronExpression, sendOptions?]`. The
@@ -162,6 +167,32 @@ export const insightRetryOptions = {
retryBackoff: true,
} as const;
/**
* Retry policy for a cron-driven pass whose failure mode is deterministic.
*
* Handlers now return a `JobOutcome`, and the retention cleanups that used to
* warn-and-return fail their job instead. That is the point — but it walks
* into pg-boss's queue default of `retryLimit: 2` with no delay and no
* backoff, so a bulk DELETE that timed out against a large trailing edge
* would run twice more, immediately, against the same rows. Three doomed
* sequential scans a night is a worse outcome than the silence it replaced.
*
* For this class the next cron tick IS the retry, and it arrives with the
* whole gap as backoff. So the schedule declares zero retries: the failure is
* recorded once, reaches the operator once, and the pass tries again on its
* own cadence.
*
* It rides on the SCHEDULE rather than on `createQueue`, deliberately.
* pg-boss's `create_queue()` ends in ON CONFLICT DO NOTHING, so a queue-level
* default would be inert on every already-running instance — the same trap
* `reconcileQueuePolicies` above exists to work around. Send options apply to
* every job the cron mints, new database or old.
*
* Use it only where the failure genuinely repeats. A transient failure (a TLS
* probe, an S3 fetch, a provider call) wants the default retries, not this.
*/
export const cronIsTheRetry = { retryLimit: 0 } as const;
/**
* Create every queue in `queues`, then schedule every cron in `schedules`.
* Centralised so each registrar provisions before it schedules in the exact
@@ -198,3 +229,41 @@ export async function createAndSchedule(
);
}
}
/**
* Bind a handler to a queue. The one legal way to reach `boss.work` in this
* tree — the `healthlog/job-handler-outcome` lint rule refuses a bare
* `boss.work(` anywhere else.
*
* The point of routing every binding through here is the handler type:
* `JobHandler<T>` resolves to `Promise<JobOutcome>`, so a handler cannot be
* bound unless it returns a value declaring what it did. `runJob` then fails
* the pg-boss job on `ok: false` instead of completing it. Falling off the
* end of a handler, which used to read as success, is now a type error.
*
* Argument order matches `boss.work(queue, options, handler)` so a binding
* reads the same as it did before, with `boss` moved to the front.
*/
export async function createAndWork<T>(
boss: PgBoss,
queue: string,
options: WorkOptions & { includeMetadata: true },
handler: JobWithMetadataHandler<T>,
): Promise<void>;
export async function createAndWork<T>(
boss: PgBoss,
queue: string,
options: WorkOptions,
handler: JobHandler<T>,
): Promise<void>;
export async function createAndWork<T>(
boss: PgBoss,
queue: string,
options: WorkOptions,
handler: JobHandler<T> | JobWithMetadataHandler<T>,
): Promise<void> {
// `JobWithMetadata<T>` extends `Job<T>`, so the metadata handler accepts
// everything the plain one does; the overloads above are what keeps the
// call sites honest about which shape they asked pg-boss for.
await boss.work<T>(queue, options, runJob(queue, handler as JobHandler<T>));
}
+35 -7
View File
@@ -7,6 +7,7 @@
import { type Job } from "pg-boss";
import { fireAndForget } from "@/lib/logging/fire-and-forget";
import { recordError } from "@/lib/jobs/worker-status";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import {
syncUserRecovery,
@@ -66,6 +67,10 @@ type ByIdSync = (userId: string, resourceId: string) => Promise<number>;
* - A cron tick (no `userId` on any job) walks every connection's collection.
*
* One user's parked-at-reauth state never starves the rest of the cohort.
*
* The pass is what the outcome judges: a per-user failure is that user's
* problem, recorded on their integration ledger and counted as `users_failed`,
* not a queue-wide failure that would retry the whole cohort.
*/
export async function runWhoopResourceSync(
taskName: string,
@@ -73,8 +78,8 @@ export async function runWhoopResourceSync(
jobs: Job<WhoopSyncPayload>[],
syncFn: CollectionSync,
byIdFn?: ByIdSync,
): Promise<void> {
await withBackgroundEvent(taskName, async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent(taskName, async (evt) => {
const prisma = getWorkerPrisma();
try {
// (userId, optional resourceId) work items from the webhook payloads.
@@ -94,9 +99,17 @@ export async function runWhoopResourceSync(
});
targets.push(...connections);
}
if (targets.length === 0) return;
if (targets.length === 0) {
return jobDone({
total: 0,
users_synced: 0,
users_failed: 0,
measurements_imported: 0,
});
}
let usersSynced = 0;
let usersFailed = 0;
let measurementsImported = 0;
for (const { userId, resourceId } of targets) {
try {
@@ -123,6 +136,7 @@ export async function runWhoopResourceSync(
if (!isSyncFailureRecorded(err)) {
await recordWhoopSyncFailure(userId, err, leg);
}
usersFailed++;
evt.addWarning(`${taskName} failed for user ${userId}: ${err}`);
}
}
@@ -135,6 +149,12 @@ export async function runWhoopResourceSync(
measurements_imported: measurementsImported,
},
});
return jobDone({
total: targets.length,
users_synced: usersSynced,
users_failed: usersFailed,
measurements_imported: measurementsImported,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -143,7 +163,9 @@ export async function runWhoopResourceSync(
});
}
export function handleWhoopRecoverySync(jobs: Job<WhoopSyncPayload>[]) {
export function handleWhoopRecoverySync(
jobs: Job<WhoopSyncPayload>[],
): Promise<JobOutcome> {
return runWhoopResourceSync(
"job.whoop_recovery_sync",
"recovery",
@@ -153,7 +175,9 @@ export function handleWhoopRecoverySync(jobs: Job<WhoopSyncPayload>[]) {
);
}
export function handleWhoopSleepSync(jobs: Job<WhoopSyncPayload>[]) {
export function handleWhoopSleepSync(
jobs: Job<WhoopSyncPayload>[],
): Promise<JobOutcome> {
return runWhoopResourceSync(
"job.whoop_sleep_sync",
"sleep",
@@ -163,7 +187,9 @@ export function handleWhoopSleepSync(jobs: Job<WhoopSyncPayload>[]) {
);
}
export function handleWhoopWorkoutSync(jobs: Job<WhoopSyncPayload>[]) {
export function handleWhoopWorkoutSync(
jobs: Job<WhoopSyncPayload>[],
): Promise<JobOutcome> {
return runWhoopResourceSync(
"job.whoop_workout_sync",
"workout",
@@ -173,7 +199,9 @@ export function handleWhoopWorkoutSync(jobs: Job<WhoopSyncPayload>[]) {
);
}
export function handleWhoopCycleSync(jobs: Job<WhoopSyncPayload>[]) {
export function handleWhoopCycleSync(
jobs: Job<WhoopSyncPayload>[],
): Promise<JobOutcome> {
// Cycle has no webhook (poll-only), so no fetch-by-id path.
return runWhoopResourceSync(
"job.whoop_cycle_sync",
+78 -12
View File
@@ -7,6 +7,7 @@
import { type Job } from "pg-boss";
import { fireAndForget } from "@/lib/logging/fire-and-forget";
import { recordError, recordWithingsSync } from "@/lib/jobs/worker-status";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
import {
retryDueWithingsWebhookSubscriptions,
@@ -50,15 +51,20 @@ export interface WithingsSleepSyncPayload {
*/
export async function handleWithingsFallbackSync(
jobs: Job<WithingsSyncPayload>[],
) {
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.withings_sync", async (evt) => {
return withBackgroundEvent("job.withings_sync", async (evt) => {
const prisma = getWorkerPrisma();
try {
recordWithingsSync();
// The subscription repair is an auxiliary leg, not the pass this queue is
// named for. Failing the job on it would skip the fallback sync that every
// user relies on when a webhook goes missing, so it rides out as a fact.
let subscriptionRepairFailed = false;
try {
await retryDueWithingsWebhookSubscriptions();
} catch {
subscriptionRepairFailed = true;
evt.addWarning(
"Withings subscription repair failed; continuing fallback sync",
);
@@ -68,10 +74,17 @@ export async function handleWithingsFallbackSync(
});
if (connections.length === 0) {
return;
return jobDone({
total: 0,
users_synced: 0,
users_failed: 0,
measurements_imported: 0,
subscription_repair_failed: subscriptionRepairFailed,
});
}
let usersSynced = 0;
let usersFailed = 0;
let measurementsImported = 0;
for (const connection of connections) {
@@ -87,6 +100,7 @@ export async function handleWithingsFallbackSync(
});
}
} catch {
usersFailed++;
evt.addWarning(`Fallback sync failed for user ${connection.userId}`);
}
}
@@ -99,6 +113,13 @@ export async function handleWithingsFallbackSync(
measurements_imported: measurementsImported,
},
});
return jobDone({
total: connections.length,
users_synced: usersSynced,
users_failed: usersFailed,
measurements_imported: measurementsImported,
subscription_repair_failed: subscriptionRepairFailed,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -127,8 +148,8 @@ export async function handleWithingsFallbackSync(
*/
export async function handleWithingsEcgSync(
jobs: Job<WithingsEcgSyncPayload>[],
) {
await withBackgroundEvent("job.withings_ecg_sync", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.withings_ecg_sync", async (evt) => {
const prisma = getWorkerPrisma();
try {
const targets: Array<{
@@ -152,9 +173,17 @@ export async function handleWithingsEcgSync(
});
targets.push(...connections);
}
if (targets.length === 0) return;
if (targets.length === 0) {
return jobDone({
total: 0,
users_synced: 0,
users_failed: 0,
recordings_imported: 0,
});
}
let usersSynced = 0;
let usersFailed = 0;
let recordingsImported = 0;
for (const { userId, startdate, enddate } of targets) {
try {
@@ -165,6 +194,7 @@ export async function handleWithingsEcgSync(
recordingsImported += await syncUserEcg(userId, options);
usersSynced++;
} catch (err) {
usersFailed++;
evt.addWarning(`Withings ECG sync failed for user ${userId}: ${err}`);
}
}
@@ -177,6 +207,12 @@ export async function handleWithingsEcgSync(
recordings_imported: recordingsImported,
},
});
return jobDone({
total: targets.length,
users_synced: usersSynced,
users_failed: usersFailed,
recordings_imported: recordingsImported,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -203,8 +239,8 @@ export async function handleWithingsEcgSync(
*/
export async function handleWithingsActivitySync(
jobs: Job<WithingsActivitySyncPayload>[],
) {
await withBackgroundEvent("job.withings_activity_sync", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.withings_activity_sync", async (evt) => {
const prisma = getWorkerPrisma();
try {
const targets: Array<{ userId: string }> = [];
@@ -220,9 +256,17 @@ export async function handleWithingsActivitySync(
});
targets.push(...connections);
}
if (targets.length === 0) return;
if (targets.length === 0) {
return jobDone({
total: 0,
users_synced: 0,
users_failed: 0,
measurements_imported: 0,
});
}
let usersSynced = 0;
let usersFailed = 0;
let measurementsImported = 0;
for (const { userId } of targets) {
try {
@@ -235,6 +279,7 @@ export async function handleWithingsActivitySync(
});
}
} catch {
usersFailed++;
evt.addWarning(`Withings activity sync failed for user ${userId}`);
}
}
@@ -247,6 +292,12 @@ export async function handleWithingsActivitySync(
measurements_imported: measurementsImported,
},
});
return jobDone({
total: targets.length,
users_synced: usersSynced,
users_failed: usersFailed,
measurements_imported: measurementsImported,
});
} catch (err) {
evt.setError(err);
recordError();
@@ -262,8 +313,8 @@ export async function handleWithingsActivitySync(
*/
export async function handleWithingsSleepSync(
jobs: Job<WithingsSleepSyncPayload>[],
) {
await withBackgroundEvent("job.withings_sleep_sync", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.withings_sleep_sync", async (evt) => {
const prisma = getWorkerPrisma();
try {
const targets: Array<{ userId: string }> = [];
@@ -278,9 +329,17 @@ export async function handleWithingsSleepSync(
});
targets.push(...connections);
}
if (targets.length === 0) return;
if (targets.length === 0) {
return jobDone({
total: 0,
users_synced: 0,
users_failed: 0,
measurements_imported: 0,
});
}
let usersSynced = 0;
let usersFailed = 0;
let measurementsImported = 0;
for (const { userId } of targets) {
try {
@@ -293,6 +352,7 @@ export async function handleWithingsSleepSync(
});
}
} catch {
usersFailed++;
evt.addWarning(`Withings sleep sync failed for user ${userId}`);
}
}
@@ -305,6 +365,12 @@ export async function handleWithingsSleepSync(
measurements_imported: measurementsImported,
},
});
return jobDone({
total: targets.length,
users_synced: usersSynced,
users_failed: usersFailed,
measurements_imported: measurementsImported,
});
} catch (err) {
evt.setError(err);
recordError();
+24 -3
View File
@@ -30,6 +30,7 @@ import {
type S3Like,
} from "@/lib/jobs/offhost-backup";
import { reportWorkerError } from "@/lib/jobs/report-worker-error";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
export const RESTORE_DRILL_QUEUE = "data-restore-drill";
@@ -149,9 +150,11 @@ export async function runRestoreDrill(
* page); every other failure goes through `reportWorkerError` so the
* operator hears about a rotting backup chain.
*/
export async function handleRestoreDrill(jobs: Job<object>[]): Promise<void> {
export async function handleRestoreDrill(
jobs: Job<object>[],
): Promise<JobOutcome> {
void jobs;
await withBackgroundEvent("job.restore_drill", async (evt) => {
return withBackgroundEvent("job.restore_drill", async (evt) => {
try {
const report = await runRestoreDrill();
evt.addMeta("restore_drill_object_key", report.objectKey);
@@ -181,13 +184,31 @@ export async function handleRestoreDrill(jobs: Job<object>[]): Promise<void> {
{ objectKey: report.objectKey, ageDays: report.ageDays },
);
}
// A stale chain is a verdict about the uploader, not a failure of the
// drill: the artefact was fetched, decrypted and parsed, and the page
// above already reached the operator. Failing the job here would retry a
// reading that cannot change until the next upload lands.
return jobDone({
restore_drill_object_key: report.objectKey,
restore_drill_age_days: report.ageDays,
restore_drill_ciphertext_bytes: report.ciphertextBytes,
restore_drill_plaintext_bytes: report.plaintextBytes,
restore_drill_measurements: report.recordCounts.measurements,
restore_drill_medications: report.recordCounts.medications,
restore_drill_intake_events: report.recordCounts.intakeEvents,
restore_drill_mood_entries: report.recordCounts.moodEntries,
restore_drill_stale: report.stale,
});
} catch (err) {
if (err instanceof OffhostBackupNotConfiguredError) {
evt.addWarning(`restore-drill skipped: ${err.message}`);
return;
// Most self-hosters never set the S3 vars. Nothing to verify is not a
// failed verification, and a monthly failed job would be noise.
return jobDone({ skipped: "offhost backup not configured" });
}
evt.addWarning(`restore-drill failed: ${err}`);
await reportWorkerError(RESTORE_DRILL_QUEUE, err);
return jobFailed("restore drill failed", err);
}
});
}
+94
View File
@@ -0,0 +1,94 @@
/**
* The wrapper that turns a returned `JobOutcome` into a pg-boss result.
*
* `boss.work(queue, opts, handler)` completes a job when the handler's
* promise resolves, whatever it resolves to. Wrapping every handler in
* `runJob` moves that decision from "the function returned" to "the function
* said it succeeded": an `ok: false` outcome is reported and rethrown, so
* pg-boss records a failed job, the queue's retry policy applies, and the
* failed-job readers in `job-failures.ts` surface it to the operator.
*
* This changes runtime behaviour, not only visibility. Handlers that used to
* catch, warn and return now fail their job. For a queue whose failure mode
* is deterministic — a DELETE that times out will time out again — retrying
* is wasted load, so the retry policy of every converted queue has to be a
* decision rather than a default.
*
* The wrapper deliberately does not open a wide event of its own. Handlers
* own their `withBackgroundEvent` scope and annotate inside it; a second
* event here would double every worker line for no added fact.
*/
import type { Job, JobWithMetadata } from "pg-boss";
import type { JobFact, JobFacts, JobOutcome } from "./job-outcome";
import { reportWorkerError } from "./report-worker-error";
/** A converted handler: it says what it did instead of just finishing. */
export type JobHandler<T> = (jobs: Job<T>[]) => Promise<JobOutcome>;
/** The same, for a queue bound with `includeMetadata: true`. */
export type JobWithMetadataHandler<T> = (
jobs: JobWithMetadata<T>[],
) => Promise<JobOutcome>;
/**
* The error a failed outcome becomes. Carries the queue and the handler's
* own reason so the pg-boss `output` column, the GlitchTip event and the
* container log all name the same thing.
*/
export class JobFailure extends Error {
readonly queue: string;
readonly reason: string;
constructor(queue: string, reason: string, cause?: unknown) {
super(`${queue}: ${reason}`, cause === undefined ? undefined : { cause });
this.name = "JobFailure";
this.queue = queue;
this.reason = reason;
// Carry the thrown error's stack. Without this the only stack the
// operator ever sees points at `runJob`, which is the same for all 101
// queues and says nothing about where the work actually broke.
if (cause instanceof Error && cause.stack) {
this.stack = `${this.name}: ${this.message}\ncaused by ${cause.stack}`;
}
}
}
function factMeta(did: JobFacts | undefined): Record<string, JobFact> {
const meta: Record<string, JobFact> = {};
for (const [key, value] of Object.entries(did ?? {})) meta[key] = value;
return meta;
}
/**
* Wrap a handler for `boss.work`. Resolves on `ok: true`; reports and
* rethrows on `ok: false`.
*
* A handler that throws is left alone — the throw already fails the job, and
* re-wrapping it would only bury the original stack. `runJob` exists for the
* case the throw never happens.
*/
export function runJob<J>(
queue: string,
handler: (jobs: J[]) => Promise<JobOutcome>,
): (jobs: J[]) => Promise<void> {
return async (jobs: J[]): Promise<void> => {
const outcome = await handler(jobs);
if (outcome.ok) return;
const failure = new JobFailure(queue, outcome.reason, outcome.cause);
const causeMessage =
outcome.cause instanceof Error
? outcome.cause.message
: typeof outcome.cause === "string"
? outcome.cause
: undefined;
await reportWorkerError(queue, failure, {
...factMeta(outcome.did),
...(causeMessage === undefined ? {} : { cause: causeMessage }),
});
throw failure;
};
}
+11 -2
View File
@@ -48,6 +48,7 @@ import { pickCanonicalWorkoutRows } from "@/lib/measurements/pick-canonical-work
import { defaultLocale, locales, type Locale } from "@/lib/i18n/config";
import { finalizeStatusSummary } from "@/lib/insights/status-shared";
import { runStatusCompletion } from "@/lib/insights/status-provider";
import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome";
import { annotate } from "@/lib/logging/context";
import { withBackgroundEvent } from "@/lib/logging/background";
import { resolveModuleMap } from "@/lib/modules/gate";
@@ -524,10 +525,14 @@ export async function runWorkoutInsightGenerate(
export async function handleWorkoutInsightGenerate(
jobs: Job<WorkoutInsightGeneratePayload>[],
): Promise<void> {
await withBackgroundEvent("job.workout_insight_generate", async (evt) => {
): Promise<JobOutcome> {
return withBackgroundEvent("job.workout_insight_generate", async (evt) => {
let generated = 0;
let skipped = 0;
for (const job of jobs) {
const outcome = await runWorkoutInsightGenerate(job.data);
if (outcome.status === "skipped") skipped++;
else generated++;
evt.addMeta(
"workout_insight",
outcome.status === "skipped"
@@ -535,5 +540,9 @@ export async function handleWorkoutInsightGenerate(
: `generated:${outcome.providerType}`,
);
}
// Every gate in front of the provider returns `skipped`, so a pass that
// wrote no paragraph at all is still a pass that ran. Only a transient
// fault escapes `runWorkoutInsightGenerate`, and that one throws.
return jobDone({ total: jobs.length, generated, skipped });
});
}