diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index ae77487b9..0dba141af 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 121, - "total": 143770, + "count": 116, + "total": 137648, "entries": { "app/api/dashboard/summary/route.ts": 1032, "app/api/documents/inbound/route.ts": 811, @@ -89,18 +89,13 @@ "lib/insights/mood-status.ts": 814, "lib/insights/status-cache.ts": 812, "lib/integrations/status.ts": 1445, - "lib/jobs/apple-health-import-worker.ts": 801, - "lib/jobs/backup-restore.ts": 888, - "lib/jobs/coach-nudge.ts": 1089, - "lib/jobs/insight-pregenerate.ts": 1266, - "lib/jobs/medication-intake-import.ts": 822, - "lib/jobs/offhost-backup.ts": 1168, + "lib/jobs/backup-restore.ts": 887, + "lib/jobs/coach-nudge.ts": 1084, + "lib/jobs/insight-pregenerate.ts": 1259, + "lib/jobs/medication-intake-import.ts": 819, + "lib/jobs/offhost-backup.ts": 1147, "lib/jobs/reaction-line.ts": 824, "lib/jobs/reminder/medication-reminder-check.ts": 818, - "lib/jobs/reminder/register-integration-sync.ts": 1051, - "lib/jobs/reminder/register-maintenance.ts": 2197, - "lib/jobs/reminder/register-rollup.ts": 1156, - "lib/jobs/reminder/register-status.ts": 878, "lib/managed-profiles/handover.ts": 986, "lib/mcp/record-search.ts": 1467, "lib/mcp/rich-reads.ts": 1467, @@ -117,7 +112,7 @@ "lib/rollups/measurement-rollups.ts": 1496, "lib/search/search-records.ts": 996, "lib/sharing/grants.ts": 1046, - "lib/support/report.ts": 905, + "lib/support/report.ts": 903, "lib/telegram-webhook-handlers.ts": 1341, "lib/validations/inbound-documents.ts": 1130, "lib/whoop/client.ts": 917, diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index dd571a095..7cc32eb6d 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 410, - "total": 147795, + "count": 403, + "total": 144626, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314, @@ -368,13 +368,6 @@ "lib/jobs/reaction-line.ts :: runReactionLine": 337, "lib/jobs/reminder/medication-reminder-check.ts :: handleReminderCheck": 619, "lib/jobs/reminder/medication-reminder-check.ts :: handleReminderCheck > withBackgroundEvent(arg)": 614, - "lib/jobs/reminder/register-integration-sync.ts :: enqueueIntegrationSyncBootDiscovery": 224, - "lib/jobs/reminder/register-integration-sync.ts :: registerIntegrationSyncQueues": 320, - "lib/jobs/reminder/register-maintenance.ts :: enqueueMaintenanceBootDiscovery": 232, - "lib/jobs/reminder/register-maintenance.ts :: registerMaintenanceQueues": 997, - "lib/jobs/reminder/register-rollup.ts :: enqueueRollupBootDiscovery": 238, - "lib/jobs/reminder/register-rollup.ts :: registerRollupQueues": 616, - "lib/jobs/reminder/register-status.ts :: registerStatusQueues": 542, "lib/jobs/workout-insight-generate.ts :: runWorkoutInsightGenerate": 324, "lib/labs/biomarker-merge.ts :: runMerge": 254, "lib/labs/ocr-extract.ts :: runOcrExtraction": 207, diff --git a/src/__tests__/baselines/test-source-reads.json b/src/__tests__/baselines/test-source-reads.json index 2aff0e4d8..309eef2f2 100644 --- a/src/__tests__/baselines/test-source-reads.json +++ b/src/__tests__/baselines/test-source-reads.json @@ -1,7 +1,7 @@ { "$comment": "tests that read a source file by path, per test file under src/. Maintained by src/__tests__/test-realism-ratchet.test.ts.", - "count": 367, - "total": 367, + "count": 349, + "total": 349, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 1, "__tests__/admin-data-wipe-completeness.test.ts": 1, @@ -314,44 +314,26 @@ "lib/jobs/__tests__/ai-job-capability-inventory.test.ts": 1, "lib/jobs/__tests__/apple-health-import-ecg.test.ts": 1, "lib/jobs/__tests__/apple-health-import-export-date.test.ts": 1, - "lib/jobs/__tests__/apple-health-import-reconcile.test.ts": 1, "lib/jobs/__tests__/apple-health-import-send-policy.test.ts": 1, - "lib/jobs/__tests__/coach-memory-refresh-queue.test.ts": 1, "lib/jobs/__tests__/coach-message-cleanup-queue.test.ts": 1, - "lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts": 1, "lib/jobs/__tests__/cycle-reminder-queue.test.ts": 1, - "lib/jobs/__tests__/daily-briefing.test.ts": 1, "lib/jobs/__tests__/data-arrival-provider-isolation.test.ts": 1, "lib/jobs/__tests__/data-arrival-queue.test.ts": 1, "lib/jobs/__tests__/data-backup-send-policy.test.ts": 1, - "lib/jobs/__tests__/drain-cumulative-queue.test.ts": 1, "lib/jobs/__tests__/fitbit-queues.test.ts": 1, "lib/jobs/__tests__/geo-backfill.test.ts": 1, "lib/jobs/__tests__/geolite2-fetch-queue.test.ts": 1, "lib/jobs/__tests__/google-health-queues.test.ts": 1, - "lib/jobs/__tests__/insight-pregenerate.test.ts": 1, - "lib/jobs/__tests__/insight-status-generate.test.ts": 1, - "lib/jobs/__tests__/intake-slot-dedup-queue.test.ts": 1, "lib/jobs/__tests__/integration-backfill-admission.test.ts": 1, "lib/jobs/__tests__/job-failures-retention.test.ts": 1, "lib/jobs/__tests__/job-outcome-source-inventory.test.ts": 1, "lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts": 1, - "lib/jobs/__tests__/mean-consolidation-queue.test.ts": 1, "lib/jobs/__tests__/measurement-reminder-queue.test.ts": 1, - "lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts": 1, - "lib/jobs/__tests__/nightscout-queues.test.ts": 1, - "lib/jobs/__tests__/period-narrative-warm.test.ts": 1, - "lib/jobs/__tests__/queue-policy-tables.test.ts": 1, - "lib/jobs/__tests__/read-side-effect-queues.test.ts": 1, "lib/jobs/__tests__/recovery-score-queue.test.ts": 1, - "lib/jobs/__tests__/sleep-timeline-queues.test.ts": 1, "lib/jobs/__tests__/status-cron-candidates.test.ts": 1, - "lib/jobs/__tests__/step-consolidation-repair-queue.test.ts": 1, "lib/jobs/__tests__/stress-strain-retention-queue.test.ts": 1, - "lib/jobs/__tests__/subsystem-surface.test.ts": 1, "lib/jobs/__tests__/tls-pin-monitor-queue.test.ts": 1, "lib/jobs/__tests__/whoop-queues.test.ts": 1, - "lib/jobs/__tests__/withings-ecg-worker.test.ts": 1, "lib/jobs/__tests__/withings-queues.test.ts": 1, "lib/jobs/__tests__/workout-insight-queue.test.ts": 1, "lib/jobs/reminder/__tests__/integration-cohort-outcomes.test.ts": 1, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 65ad892a5..d98d3866f 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", - "count": 2954, - "total": 10499, + "count": 2924, + "total": 10176, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -2288,59 +2288,34 @@ "lib/integrations/metric-freshness.ts": 1, "lib/integrations/status.ts": 19, "lib/intl/__tests__/formatter-cache.test.ts": 1, - "lib/jobs/__tests__/apple-health-import-reconcile.test.ts": 2, - "lib/jobs/__tests__/apple-health-import-send-policy.test.ts": 1, - "lib/jobs/__tests__/coach-memory-refresh-queue.test.ts": 2, - "lib/jobs/__tests__/coach-message-cleanup-queue.test.ts": 1, + "lib/jobs/__tests__/apple-health-import-reconcile.test.ts": 1, "lib/jobs/__tests__/coach-nudge.test.ts": 3, "lib/jobs/__tests__/coach-reminder-sweep.test.ts": 1, - "lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts": 1, - "lib/jobs/__tests__/cycle-reminder-queue.test.ts": 2, "lib/jobs/__tests__/cycle-reminder.test.ts": 2, - "lib/jobs/__tests__/data-arrival-queue.test.ts": 1, - "lib/jobs/__tests__/drain-cumulative-queue.test.ts": 4, "lib/jobs/__tests__/feedback-aggregator.test.ts": 2, "lib/jobs/__tests__/fitbit-backfill.test.ts": 1, - "lib/jobs/__tests__/fitbit-queues.test.ts": 2, - "lib/jobs/__tests__/geo-backfill.test.ts": 3, - "lib/jobs/__tests__/google-health-queues.test.ts": 1, + "lib/jobs/__tests__/geo-backfill.test.ts": 1, "lib/jobs/__tests__/google-health-sleep-repair.test.ts": 1, "lib/jobs/__tests__/hero-line-json-envelope.test.ts": 1, "lib/jobs/__tests__/insight-pregenerate-retry-enqueue.test.ts": 1, - "lib/jobs/__tests__/insight-pregenerate.test.ts": 16, - "lib/jobs/__tests__/insight-status-generate.test.ts": 2, + "lib/jobs/__tests__/insight-pregenerate.test.ts": 14, "lib/jobs/__tests__/intake-auto-skip.test.ts": 3, "lib/jobs/__tests__/intake-auto-take.test.ts": 1, - "lib/jobs/__tests__/intake-slot-dedup-queue.test.ts": 6, "lib/jobs/__tests__/job-run-report.test.ts": 1, - "lib/jobs/__tests__/mean-consolidation-queue.test.ts": 2, - "lib/jobs/__tests__/measurement-reminder-queue.test.ts": 2, "lib/jobs/__tests__/measurement-reminder.test.ts": 6, - "lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts": 2, "lib/jobs/__tests__/medication-inventory-expire.test.ts": 1, "lib/jobs/__tests__/medication-low-stock.test.ts": 2, "lib/jobs/__tests__/medication-reminder-check.test.ts": 2, "lib/jobs/__tests__/mood-reminder.test.ts": 2, - "lib/jobs/__tests__/nightscout-queues.test.ts": 3, - "lib/jobs/__tests__/period-narrative-warm.test.ts": 1, "lib/jobs/__tests__/reaction-line-degradation.test.ts": 2, - "lib/jobs/__tests__/recovery-score-queue.test.ts": 2, "lib/jobs/__tests__/recovery-score.test.ts": 1, "lib/jobs/__tests__/reminder-phases.test.ts": 1, "lib/jobs/__tests__/reminder-satisfy.test.ts": 3, "lib/jobs/__tests__/report-worker-error.test.ts": 1, "lib/jobs/__tests__/run-job.test.ts": 1, "lib/jobs/__tests__/sleep-timeline-backfill.test.ts": 1, - "lib/jobs/__tests__/sleep-timeline-queues.test.ts": 2, - "lib/jobs/__tests__/status-cron-candidates.test.ts": 2, - "lib/jobs/__tests__/step-consolidation-repair-queue.test.ts": 1, - "lib/jobs/__tests__/stress-strain-retention-queue.test.ts": 2, - "lib/jobs/__tests__/tls-pin-monitor-queue.test.ts": 2, "lib/jobs/__tests__/whoop-backfill.test.ts": 1, - "lib/jobs/__tests__/whoop-queues.test.ts": 2, "lib/jobs/__tests__/whoop-resource-sync.test.ts": 1, - "lib/jobs/__tests__/withings-queues.test.ts": 2, - "lib/jobs/__tests__/workout-insight-queue.test.ts": 1, "lib/jobs/apple-health-import-worker.ts": 4, "lib/jobs/coach-message-cleanup.ts": 1, "lib/jobs/coach-nudge-ai.ts": 3, @@ -2396,7 +2371,7 @@ "lib/jobs/recovery-score.ts": 1, "lib/jobs/reminder-phases.ts": 1, "lib/jobs/reminder-satisfy.ts": 1, - "lib/jobs/reminder-worker.ts": 4, + "lib/jobs/reminder-worker.ts": 3, "lib/jobs/reminder/__tests__/cleanup-handlers-visibility.test.ts": 1, "lib/jobs/reminder/__tests__/queue-retention.test.ts": 1, "lib/jobs/reminder/cleanup-handlers.ts": 7, @@ -2410,11 +2385,6 @@ "lib/jobs/reminder/oura-sync.ts": 1, "lib/jobs/reminder/polar-sync.ts": 2, "lib/jobs/reminder/poll-cohort.ts": 2, - "lib/jobs/reminder/register-integration-sync.ts": 63, - "lib/jobs/reminder/register-maintenance.ts": 114, - "lib/jobs/reminder/register-reminders.ts": 15, - "lib/jobs/reminder/register-rollup.ts": 37, - "lib/jobs/reminder/register-status.ts": 38, "lib/jobs/reminder/registrar-shared.ts": 2, "lib/jobs/reminder/strava-sync.ts": 1, "lib/jobs/reminder/whoop-sync.ts": 3, diff --git a/src/__tests__/guard-floor-guard.test.ts b/src/__tests__/guard-floor-guard.test.ts index 7578cf6ce..687ec7d31 100644 --- a/src/__tests__/guard-floor-guard.test.ts +++ b/src/__tests__/guard-floor-guard.test.ts @@ -63,8 +63,6 @@ const NO_FLOOR_NEEDED: Readonly> = { "named walks, each must positively match the keyset cursor", "components/charts/__tests__/touch-action-guard.test.ts": "named chart wrappers, each must positively declare touch-pan-y", - "__tests__/moodlog-removal-guard.test.ts": - "absence checks over named paths, columns and keys; no matcher to drift", // Unit tests of runtime code whose name ends in "guard". They read no // source and sweep nothing. diff --git a/src/__tests__/helpers/source-metrics.ts b/src/__tests__/helpers/source-metrics.ts index f03d8b365..5fa9d3a2d 100644 --- a/src/__tests__/helpers/source-metrics.ts +++ b/src/__tests__/helpers/source-metrics.ts @@ -197,6 +197,8 @@ export interface ModuleImports { value: string[]; /** True when every statement is an `export … from` (or an import). */ reexportOnly: boolean; + /** The value imports evaluated when the module loads: `value` without `import("…")`. */ + staticValue: string[]; } /** @@ -240,6 +242,7 @@ export function moduleImports(sf: ts.SourceFile): ModuleImports { } other += 1; } + const staticValue = [...value]; const visit = (node: ts.Node): void => { if ( ts.isCallExpression(node) && @@ -252,7 +255,7 @@ export function moduleImports(sf: ts.SourceFile): ModuleImports { ts.forEachChild(node, visit); }; visit(sf); - return { value, reexportOnly: reexports > 0 && other === 0 }; + return { value, staticValue, reexportOnly: reexports > 0 && other === 0 }; } export interface ImportGraph { @@ -374,3 +377,40 @@ export function backEdgeCycles(edges: Map): string[][] { for (const v of [...edges.keys()].sort()) walk(v); return cycles; } + +/** + * Every module under `src/` that loading `entries` evaluates: static value + * imports and re-exports, transitively. A dynamic `import("…")` is not + * followed, since it loads only when it runs. Paths are relative to `src/`. + */ +export function staticImportClosure(entries: readonly string[]): Set { + const cache = ts.createModuleResolutionCache( + process.cwd(), + (x) => x, + COMPILER_OPTIONS, + ); + const seen = new Set(); + const stack = [...entries]; + while (stack.length > 0) { + const rel = stack.pop()!; + if (seen.has(rel)) continue; + seen.add(rel); + const abs = join(SRC, rel); + for (const spec of moduleImports(parse(rel, readFileSync(abs, "utf8"))) + .staticValue) { + const hit = ts.resolveModuleName( + spec, + abs, + COMPILER_OPTIONS, + ts.sys, + cache, + ).resolvedModule; + if (!hit || hit.isExternalLibraryImport) continue; + if (!hit.resolvedFileName.startsWith(`${SRC}/`)) continue; + const target = hit.resolvedFileName.slice(SRC.length + 1); + if (target.startsWith("generated/") || target.endsWith(".d.ts")) continue; + stack.push(target); + } + } + return seen; +} diff --git a/src/__tests__/job-queue-runtime-guard.test.ts b/src/__tests__/job-queue-runtime-guard.test.ts index 48da4c47d..90b14d8ec 100644 --- a/src/__tests__/job-queue-runtime-guard.test.ts +++ b/src/__tests__/job-queue-runtime-guard.test.ts @@ -11,23 +11,23 @@ * `src/lib/jobs/queue-runtime.ts` declares every queue `short`, `budgeted` or * `long`. This guard holds the tree to it: * - * - every queue bound with `createAndWork` is in the table, and every table - * entry is bound; - * - every `boss.send` and every schedule tuple names a queue in the table; + * - every queue the worker registry declares (`jobs/registry/`) is in the + * table, and every table entry is declared; + * - every `boss.send` and every scheduled queue names a queue in the table; * - a `long` queue has an expiry between the default and pg-boss's 24-hour - * ceiling, every send and schedule for it carries `expireInSeconds` (or an - * identifier the table names as its carrier), it stops on the job's budget - * where the table says (`jobBudget` / `jobDeadline`, or a declared - * one-shot), and it is exclusive per identity where the table says - * (`lockedPass` at the binding, or the named marker); + * ceiling, every send for it carries `expireInSeconds` (or an identifier + * the table names as its carrier) and its schedule's send options do, it + * stops on the job's budget where the table says (`jobBudget` / + * `jobDeadline`, or a declared one-shot), and it is exclusive per identity + * where the table says (the definition's `lock`, or the named marker); * - a `budgeted` queue stops on the budget and relies on the default expiry; * - a `short` queue's sends set no expiry of their own, which would mean the * job is longer than the table claims. * * Floors fail the guard if a matcher stops matching, and the "plants" cases * run the checks over a planted long queue that lacks each protection in turn - * (the mutation proof kept in the suite). Removing `lockedPass` from the - * pr-detection binding, or the expiry from its cron tuple, fails the real + * (the mutation proof kept in the suite). Removing the `lock` from the + * pr-detection definition, or the expiry from its schedule, fails the real * checks below. */ import { describe, expect, it } from "vitest"; @@ -39,19 +39,14 @@ import { DEFAULT_JOB_EXPIRE_SECONDS, QUEUE_RUNTIME, type QueueRuntime, - type SourceSite, } from "@/lib/jobs/queue-runtime"; import { SCHEDULED_AND_SENT_QUEUES } from "@/lib/jobs/reminder/registrar-shared"; import { HEAVY_QUEUES } from "@/lib/jobs/heavy-lane"; +import type { JobDefinition } from "@/lib/jobs/registry/define-job"; +import { allJobDefinitions } from "@/lib/jobs/registry/index"; const SRC = join(process.cwd(), "src"); -const REGISTRARS = [ - "lib/jobs/reminder/register-integration-sync.ts", - "lib/jobs/reminder/register-maintenance.ts", - "lib/jobs/reminder/register-reminders.ts", - "lib/jobs/reminder/register-rollup.ts", - "lib/jobs/reminder/register-status.ts", -]; +const DEFINITIONS = allJobDefinitions(); /** pg-boss refuses an expiry of 24 hours or more. */ const MAX_EXPIRE_SECONDS = 24 * 60 * 60; @@ -120,27 +115,6 @@ interface Site { text: string; } -/** Every `createAndWork(boss, QUEUE, …)` in the registrars. */ -function bindings( - texts: ReadonlyMap, - definitions: Definitions, -): Site[] { - const out: Site[] = []; - for (const file of REGISTRARS) { - const text = code(texts.get(file) ?? ""); - for (const m of text.matchAll( - /createAndWork\s*(?:<[^(]*?>)?\(\s*boss\s*,\s*([A-Z][A-Z0-9_]*)/g, - )) { - const open = text.indexOf("(", m.index); - const queue = resolve(definitions, file, m[1]); - if (queue) { - out.push({ file, queue, text: text.slice(open, closeOf(text, open)) }); - } - } - } - return out; -} - /** Every `boss.send(QUEUE, …)` / `getGlobalBoss()?.send(QUEUE, …)`. */ function sends( texts: ReadonlyMap, @@ -162,31 +136,6 @@ function sends( return out; } -/** Every `[QUEUE, CRON, options?]` tuple in a registrar's `schedules`. */ -function schedules( - texts: ReadonlyMap, - definitions: Definitions, -): Site[] { - const out: Site[] = []; - for (const file of REGISTRARS) { - const text = code(texts.get(file) ?? ""); - const m = /const\s+schedules\s*:\s*ScheduleEntry\[\]\s*=\s*\[/.exec(text); - if (!m) continue; - const open = m.index + m[0].length - 1; - const body = text.slice(open + 1, closeOf(text, open) - 1); - for (let i = 0; i < body.length; i += 1) { - if (body[i] !== "[") continue; - const end = closeOf(body, i); - const tuple = body.slice(i, end); - const ident = /^\[\s*([A-Z][A-Z0-9_]*)/.exec(tuple)?.[1]; - const queue = ident ? resolve(definitions, file, ident) : null; - if (queue) out.push({ file, queue, text: tuple }); - i = end - 1; - } - } - return out; -} - /** The text of `fn` (a function or a const) in `file`. */ function bodyOf( texts: ReadonlyMap, @@ -201,14 +150,6 @@ function bodyOf( return text.slice(m.index, end < 0 ? undefined : end + 2); } -function siteText( - texts: ReadonlyMap, - site: SourceSite, - binding: Site | undefined, -): string { - return site === "binding" ? (binding?.text ?? "") : bodyOf(texts, site); -} - const STOPS = /\bjob(?:Budget|Deadline)\s*\(/; /** Every finding for one queue's declaration against the tree. */ @@ -216,13 +157,12 @@ function check( queue: string, entry: QueueRuntime, texts: ReadonlyMap, - all: { bindings: Site[]; sends: Site[]; schedules: Site[] }, + all: { definitions: readonly JobDefinition[]; sends: Site[] }, ): string[] { const problems: string[] = []; - const binding = all.bindings.find((b) => b.queue === queue); - const carriers = [...all.sends, ...all.schedules].filter( - (s) => s.queue === queue, - ); + const definition = all.definitions.find((d) => d.queue === queue); + const sends = all.sends.filter((s) => s.queue === queue); + const scheduledExpiry = definition?.scheduleOptions?.expireInSeconds; if (entry.runtime === "long") { if ( !(entry.expireInSeconds > DEFAULT_JOB_EXPIRE_SECONDS) || @@ -230,85 +170,86 @@ function check( ) { problems.push(`${queue}: expiry ${entry.expireInSeconds}s out of range`); } - for (const site of carriers) { + for (const site of sends) { const carried = /\bexpireInSeconds\b/.test(site.text) || entry.expiryVia.some((v) => new RegExp(`\\b${v}\\b`).test(site.text)); if (!carried) { - problems.push( - `${queue}: ${site.file} sends or schedules it without an expiry`, - ); + problems.push(`${queue}: ${site.file} sends it without an expiry`); } } + if (definition?.cron && typeof scheduledExpiry !== "number") { + problems.push(`${queue}: its schedule carries no expiry`); + } if (typeof entry.stop === "object" && "oneShot" in entry.stop) { if (entry.stop.oneShot.length < 20) { problems.push(`${queue}: a one-shot needs its reason`); } - } else if (!STOPS.test(siteText(texts, entry.stop, binding))) { + } else if (!STOPS.test(bodyOf(texts, entry.stop))) { problems.push(`${queue}: does not stop on the job's budget`); } if (entry.exclusive === "lockedPass") { - if (!/\blockedPass\s*\(/.test(binding?.text ?? "")) { - problems.push(`${queue}: binding holds no lockedPass`); + if (!definition?.lock) { + problems.push(`${queue}: definition holds no lock`); } } else if ( - !siteText(texts, entry.exclusive.at, binding).includes( - entry.exclusive.marker, - ) + !bodyOf(texts, entry.exclusive.at).includes(entry.exclusive.marker) ) { problems.push(`${queue}: no "${entry.exclusive.marker}" where declared`); } } else { if (entry.runtime === "budgeted") { - if (!STOPS.test(siteText(texts, entry.budget, binding))) { + if (!STOPS.test(bodyOf(texts, entry.budget))) { problems.push(`${queue}: does not stop on the job's budget`); } } - for (const site of carriers) { + for (const site of sends) { if (/\bexpireInSeconds\b/.test(site.text)) { problems.push( `${queue}: ${site.file} sets an expiry on a queue declared ${entry.runtime}`, ); } } + if (scheduledExpiry !== undefined) { + problems.push( + `${queue}: its schedule sets an expiry on a queue declared ${entry.runtime}`, + ); + } } return problems; } function scan(texts: ReadonlyMap) { - const definitions = definitionsOf(texts); - return { - bindings: bindings(texts, definitions), - sends: sends(texts, definitions), - schedules: schedules(texts, definitions), - }; + return { sends: sends(texts, definitionsOf(texts)) }; } describe("every queue declares its runtime", () => { - const all = scan(source); - const bound = new Set(all.bindings.map((b) => b.queue)); + const all = { definitions: DEFINITIONS, ...scan(source) }; + const declared = new Set(DEFINITIONS.map((d) => d.queue)); - it("finds the bindings, sends and schedules it is meant to police", () => { - // Pinned below the counts on 2026-10-01 (112 / 64 / 74). - expect(all.bindings.length).toBeGreaterThanOrEqual(105); + it("finds the declarations, sends and schedules it is meant to police", () => { + // Pinned below the counts on 2026-10-23 (122 / 64 / 78). + expect(DEFINITIONS.length).toBeGreaterThanOrEqual(115); expect(all.sends.length).toBeGreaterThanOrEqual(50); - expect(all.schedules.length).toBeGreaterThanOrEqual(70); + expect(DEFINITIONS.filter((d) => d.cron).length).toBeGreaterThanOrEqual(70); expect( Object.values(QUEUE_RUNTIME).filter((e) => e.runtime === "long").length, ).toBeGreaterThanOrEqual(15); }); - it("every bound queue is in the table, and every entry is bound", () => { - expect([...bound].filter((q) => !(q in QUEUE_RUNTIME)).sort()).toEqual([]); + it("every declared queue is in the table, and every entry is declared", () => { + expect([...declared].filter((q) => !(q in QUEUE_RUNTIME)).sort()).toEqual( + [], + ); expect( Object.keys(QUEUE_RUNTIME) - .filter((q) => !bound.has(q)) + .filter((q) => !declared.has(q)) .sort(), ).toEqual([]); }); - it("every send and schedule names a queue in the table", () => { - const unknown = [...all.sends, ...all.schedules] + it("every send names a queue in the table", () => { + const unknown = all.sends .filter((s) => !(s.queue in QUEUE_RUNTIME)) .map((s) => `${s.file} → ${s.queue}`); expect(unknown).toEqual([]); @@ -322,51 +263,56 @@ describe("every queue declares its runtime", () => { }); it("plants: a long queue missing any protection is caught", () => { - const registrar = REGISTRARS[1]; - const planted = (binding: string, schedule: string) => + const file = "lib/jobs/planted.ts"; + const texts = (handler: string) => new Map([ ...source, - [ - registrar, - `const PLANT_QUEUE = "planted-long";\n` + - `const schedules: ScheduleEntry[] = [\n ${schedule},\n];\n` + - `async function register(boss) {\n await createAndWork(boss, PLANT_QUEUE, { localConcurrency: 1 }, ${binding});\n}\n`, - ], + [file, `export async function plant(jobs) {\n ${handler}\n}\n`], ]); const entry: QueueRuntime = { runtime: "long", expireInSeconds: 3600, expiryVia: [], - stop: "binding", + stop: { file, fn: "plant" }, exclusive: "lockedPass", why: "planted", }; - const findings = (binding: string, schedule: string) => { - const texts = planted(binding, schedule); - return check("planted-long", entry, texts, scan(texts)); + const planted = (overrides: Partial): JobDefinition => ({ + queue: "planted-long", + cron: "0 3 * * *", + scheduleOptions: { expireInSeconds: 3600 }, + work: { localConcurrency: 1 }, + lock: () => "pass", + load: async () => async () => ({ ok: true, did: {} }), + ...overrides, + }); + const findings = ( + handler: string, + overrides: Partial = {}, + ) => { + const t = texts(handler); + return check("planted-long", entry, t, { + definitions: [planted(overrides)], + ...scan(t), + }); }; - const safe = [ - "lockedPass(PLANT_QUEUE, () => WHOLE_PASS, async (jobs) => run(jobBudget(jobs)))", - "[PLANT_QUEUE, CRON, { expireInSeconds: 3600 }]", - ] as const; - expect(findings(...safe)).toEqual([]); - expect(findings(safe[0], "[PLANT_QUEUE, CRON]")).toEqual([ - `planted-long: ${registrar} sends or schedules it without an expiry`, + const budgeted = "return run(jobBudget(jobs));"; + expect(findings(budgeted)).toEqual([]); + expect(findings(budgeted, { scheduleOptions: { retryLimit: 0 } })).toEqual([ + "planted-long: its schedule carries no expiry", ]); - expect( - findings("lockedPass(PLANT_QUEUE, () => WHOLE_PASS, run)", safe[1]), - ).toEqual(["planted-long: does not stop on the job's budget"]); - expect(findings("async (jobs) => run(jobBudget(jobs))", safe[1])).toEqual([ - "planted-long: binding holds no lockedPass", + expect(findings("return run(jobs);")).toEqual([ + "planted-long: does not stop on the job's budget", + ]); + expect(findings(budgeted, { lock: undefined })).toEqual([ + "planted-long: definition holds no lock", ]); expect( check( "planted-long", { ...entry, expireInSeconds: 900 }, - ...((): [ReadonlyMap, ReturnType] => { - const texts = planted(...safe); - return [texts, scan(texts)]; - })(), + texts(budgeted), + { definitions: [planted({})], ...scan(texts(budgeted)) }, ), ).toEqual(["planted-long: expiry 900s out of range"]); }); @@ -374,12 +320,13 @@ describe("every queue declares its runtime", () => { /** * Queues a file sends to through a variable (`boss.send(queueName, …)`): every - * known queue name the file spells out, as a constant or a map value, counts - * as sent, because the matcher above cannot tell which one the variable holds. + * known queue the file names, as a literal, a map value or an imported + * `*_QUEUE` constant, counts as sent, because the matcher above cannot tell which one the variable holds. */ function dynamicallySent(texts: ReadonlyMap): Set { + const definitions = definitionsOf(texts); const out = new Set(); - for (const [, raw] of texts) { + for (const [file, raw] of texts) { const text = code(raw); if ( !/\b(?:boss|getGlobalBoss\(\)\??)\s*\.\s*send\s*(?:<[^(]*?>)?\(\s*[a-z]/.test( @@ -391,13 +338,20 @@ function dynamicallySent(texts: ReadonlyMap): Set { for (const m of text.matchAll(/"([a-z0-9-]+)"/g)) { if (m[1] in QUEUE_RUNTIME) out.add(m[1]); } + // A queue named through an imported constant counts the same. + for (const m of text.matchAll(/\b([A-Z][A-Z0-9_]*_QUEUE)\b/g)) { + const queue = resolve(definitions, file, m[1]); + if (queue && queue in QUEUE_RUNTIME) out.add(queue); + } } return out; } describe("a cron-fed queue polls slowly, a sent one does not", () => { const all = scan(source); - const scheduled = new Set(all.schedules.map((s) => s.queue)); + const scheduled = new Set( + DEFINITIONS.filter((d) => d.cron).map((d) => d.queue), + ); const sent = new Set([ ...all.sends.map((s) => s.queue), ...dynamicallySent(source), @@ -418,26 +372,14 @@ describe("a cron-fed queue polls slowly, a sent one does not", () => { }); describe("heavy jobs share the lane", () => { - const all = scan(source); - const laned = all.bindings.filter((b) => /\bheavyLane\s*\(/.test(b.text)); + const laned = DEFINITIONS.filter((d) => d.lane); - it("binds every heavy queue in the lane, and nothing else", () => { - expect(laned.map((b) => b.queue).sort()).toEqual( + it("puts every heavy queue in the lane, and nothing else", () => { + expect(laned.map((d) => d.queue).sort()).toEqual( Object.keys(HEAVY_QUEUES).sort(), ); }); - it("holds a lockedPass outside the lane, so an expiry retry finds the run going", () => { - // Outside, a retry pg-boss starts while the expired run still holds the - // lane finds the lock and settles; inside, the lane would defer it and - // run the job a second time once the first is done. - const inverted = laned - .filter((b) => /\blockedPass\s*\(/.test(b.text)) - .filter((b) => b.text.indexOf("lockedPass") > b.text.indexOf("heavyLane")) - .map((b) => b.queue); - expect(inverted).toEqual([]); - }); - it("declares every heavy queue long, so its expiry, stop and exclusivity are proven", () => { // A heavy job is the one most likely to run past pg-boss's default // expiry, and the retry that follows an expiry runs it a second time. @@ -464,41 +406,28 @@ describe("heavy jobs share the lane", () => { // A deferred job returns from active to created. Under `short` (one // queued job per key) and `stately` (one per state) a twin already // queued would refuse it, and the job would fail instead of waiting. - const policies = policiesOf(source); - expect(policies.size).toBeGreaterThanOrEqual(40); - const refused = Object.keys(HEAVY_QUEUES) - .filter((queue) => /^(?:short|stately)$/.test(policies.get(queue) ?? "")) - .sort(); - expect(refused).toEqual([]); - // The planted case: the same check over a table that declares one. - const planted = new Map(source); - const file = REGISTRARS[2]; - planted.set( - file, - `const DATA_BACKUP_QUEUE = "data-backup";\nconst queuePolicies: QueuePolicyTable = {\n [DATA_BACKUP_QUEUE]: { policy: "short", reason: "x" },\n};\n`, + const refusing = (definitions: readonly JobDefinition[]) => + definitions + .filter( + (d) => d.lane && /^(?:short|stately)$/.test(d.policy?.policy ?? ""), + ) + .map((d) => d.queue) + .sort(); + expect(DEFINITIONS.filter((d) => d.policy).length).toBeGreaterThanOrEqual( + 40, ); - expect(policiesOf(planted).get("data-backup")).toBe("short"); + expect(refusing(DEFINITIONS)).toEqual([]); + // The planted case: the same check over a heavy queue declared `short`. + expect( + refusing([ + { + queue: "data-backup", + policy: { policy: "short", reason: "x" }, + work: { localConcurrency: 1 }, + lane: true, + load: async () => async () => ({ ok: true, did: {} }), + }, + ]), + ).toEqual(["data-backup"]); }); }); - -/** Every `[QUEUE]: { policy: "…" }` in a registrar's `queuePolicies`. */ -function policiesOf(texts: ReadonlyMap): Map { - const definitions = definitionsOf(texts); - const out = new Map(); - for (const file of REGISTRARS) { - const text = code(texts.get(file) ?? ""); - const m = /const\s+queuePolicies\s*:\s*QueuePolicyTable\s*=\s*\{/.exec( - text, - ); - if (!m) continue; - const open = m.index + m[0].length - 1; - const body = text.slice(open, closeOf(text, open)); - for (const entry of body.matchAll( - /\[\s*([A-Z][A-Z0-9_]*)\s*\]\s*:\s*\{\s*policy\s*:\s*"([a-z]+)"/g, - )) { - const queue = resolve(definitions, file, entry[1]); - if (queue) out.set(queue, entry[2]); - } - } - return out; -} diff --git a/src/__tests__/moodlog-removal-guard.test.ts b/src/__tests__/moodlog-removal-guard.test.ts index b2b218b14..c012aed0f 100644 --- a/src/__tests__/moodlog-removal-guard.test.ts +++ b/src/__tests__/moodlog-removal-guard.test.ts @@ -18,11 +18,12 @@ */ import { readFileSync, existsSync } from "node:fs"; import path from "node:path"; + +import { allJobDefinitions } from "@/lib/jobs/registry/index"; import { describe, expect, it } from "vitest"; import { INTEGRATION_CADENCE } from "@/lib/integrations/sync-verdict"; import { INTEGRATION_DISPLAY_NAMES } from "@/components/settings/integrations/integration-fallback-row"; -import { readSource } from "@/__tests__/helpers/source-files"; const ROOT = path.resolve(__dirname, "../.."); @@ -76,12 +77,9 @@ describe("moodLog integration stays removed", () => { }); it("registers no moodlog queue, cron or handler binding", () => { - const registrar = readSource( - "src/lib/jobs/reminder/register-integration-sync.ts", - "export async function registerIntegrationSyncQueues", - ); - expect(registrar).not.toContain("moodlog"); - expect(registrar).not.toContain("MoodLog"); + const queues = allJobDefinitions().map((job) => job.queue); + expect(queues.length).toBeGreaterThan(100); + expect(queues.filter((queue) => /mood-?log/i.test(queue))).toEqual([]); }); it("carries no moodlog integration key on the status envelope", () => { diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index 0aac898e9..495777c14 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -32,10 +32,10 @@ import { import { readBackupPassActivity } from "@/lib/jobs/backup-pass-attempts"; import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status"; import { - OFFHOST_BACKUP_QUEUE, offhostBackupConfigured, probeOffhostLifecycle, } from "@/lib/jobs/offhost-backup"; +import { OFFHOST_BACKUP_QUEUE } from "@/lib/jobs/offhost-backup-queue"; import { classifyOffhostBackup, OFFHOST_BACKUP_PERIOD_HOURS, diff --git a/src/app/api/admin/encryption/rotate/route.ts b/src/app/api/admin/encryption/rotate/route.ts index e3f8dc063..98ded9eef 100644 --- a/src/app/api/admin/encryption/rotate/route.ts +++ b/src/app/api/admin/encryption/rotate/route.ts @@ -31,10 +31,10 @@ import { annotate } from "@/lib/logging/context"; import { auditLog } from "@/lib/auth/audit"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { - ENCRYPTION_KEY_ROTATE_QUEUE, ENCRYPTION_KEY_ROTATE_SINGLETON, ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS, } from "@/lib/jobs/encryption-key-rotate"; +import { ENCRYPTION_KEY_ROTATE_QUEUE } from "@/lib/jobs/encryption-key-rotate-queue"; import { getActiveKeyId } from "@/lib/crypto"; import { NextRequest } from "next/server"; diff --git a/src/app/api/admin/import-apple-health-export/route.ts b/src/app/api/admin/import-apple-health-export/route.ts index 798061698..ddddadf5d 100644 --- a/src/app/api/admin/import-apple-health-export/route.ts +++ b/src/app/api/admin/import-apple-health-export/route.ts @@ -21,12 +21,12 @@ import { auditLog } from "@/lib/auth/audit"; import { checkRateLimit } from "@/lib/rate-limit"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { - APPLE_HEALTH_IMPORT_V2_QUEUE, APPLE_HEALTH_IMPORT_KIND, APPLE_HEALTH_IMPORT_PARSER_REVISION, APPLE_HEALTH_IMPORT_SEND_OPTIONS, type AppleHealthImportPayload, } from "@/lib/jobs/apple-health-import-worker"; +import { APPLE_HEALTH_IMPORT_V2_QUEUE } from "@/lib/jobs/apple-health-import-queue"; import { streamMultipartToDisk } from "@/lib/multipart/stream-to-disk"; import { createImportJobUnlessBusy, diff --git a/src/app/api/fitbit/callback/route.ts b/src/app/api/fitbit/callback/route.ts index a8284ae92..0e089baff 100644 --- a/src/app/api/fitbit/callback/route.ts +++ b/src/app/api/fitbit/callback/route.ts @@ -15,7 +15,7 @@ import { } from "@/lib/fitbit/client"; import { getUserFitbitCredentials } from "@/lib/fitbit/credentials"; import { FITBIT_OAUTH_STATE_COOKIE } from "@/lib/fitbit/oauth-state"; -import { FITBIT_BACKFILL_QUEUE } from "@/lib/jobs/fitbit-backfill"; +import { FITBIT_BACKFILL_QUEUE } from "@/lib/fitbit/jobs/queues"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { markReconnected } from "@/lib/integrations/status"; import { Prisma } from "@/generated/prisma/client"; diff --git a/src/app/api/google-health/callback/route.ts b/src/app/api/google-health/callback/route.ts index 4c0ac3d98..21a3638e4 100644 --- a/src/app/api/google-health/callback/route.ts +++ b/src/app/api/google-health/callback/route.ts @@ -17,7 +17,7 @@ import { } from "@/lib/google-health/client"; import { getUserGoogleHealthCredentials } from "@/lib/google-health/credentials"; import { GOOGLE_HEALTH_OAUTH_STATE_COOKIE } from "@/lib/google-health/oauth-state"; -import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "@/lib/jobs/google-health-backfill"; +import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "@/lib/jobs/google-health-backfill-queue"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { markReconnected } from "@/lib/integrations/status"; import { Prisma } from "@/generated/prisma/client"; diff --git a/src/app/api/import/apple-health-export/route.ts b/src/app/api/import/apple-health-export/route.ts index 20eb69061..d5e35b179 100644 --- a/src/app/api/import/apple-health-export/route.ts +++ b/src/app/api/import/apple-health-export/route.ts @@ -30,12 +30,12 @@ import { emitSignal } from "@/lib/logging/signal"; import { checkRateLimit } from "@/lib/rate-limit"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { - APPLE_HEALTH_IMPORT_V2_QUEUE, APPLE_HEALTH_IMPORT_KIND, APPLE_HEALTH_IMPORT_PARSER_REVISION, APPLE_HEALTH_IMPORT_SEND_OPTIONS, type AppleHealthImportPayload, } from "@/lib/jobs/apple-health-import-worker"; +import { APPLE_HEALTH_IMPORT_V2_QUEUE } from "@/lib/jobs/apple-health-import-queue"; import { streamMultipartToDisk } from "@/lib/multipart/stream-to-disk"; import { createImportJobUnlessBusy, diff --git a/src/app/api/import/health-connect-export/route.ts b/src/app/api/import/health-connect-export/route.ts index d8eb2b6b8..594c7fe1c 100644 --- a/src/app/api/import/health-connect-export/route.ts +++ b/src/app/api/import/health-connect-export/route.ts @@ -28,10 +28,10 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { HEALTH_CONNECT_IMPORT_KIND, HEALTH_CONNECT_IMPORT_PARSER_REVISION, - HEALTH_CONNECT_IMPORT_QUEUE, HEALTH_CONNECT_IMPORT_SEND_OPTIONS, type HealthConnectImportPayload, } from "@/lib/jobs/health-connect-import-worker"; +import { HEALTH_CONNECT_IMPORT_QUEUE } from "@/lib/jobs/health-connect-import-queue"; import { streamMultipartToDisk } from "@/lib/multipart/stream-to-disk"; import { createImportJobUnlessBusy, diff --git a/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts b/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts index 9966968a8..e04be16be 100644 --- a/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts +++ b/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts @@ -63,10 +63,8 @@ import { auditLog } from "@/lib/auth/audit"; import { getSession } from "@/lib/auth/session"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { checkRateLimit } from "@/lib/rate-limit"; -import { - MEDICATION_INTAKE_IMPORT_QUEUE, - MEDICATION_INTAKE_IMPORT_STALE_AFTER_MS, -} from "@/lib/jobs/medication-intake-import"; +import { MEDICATION_INTAKE_IMPORT_STALE_AFTER_MS } from "@/lib/jobs/medication-intake-import"; +import { MEDICATION_INTAKE_IMPORT_QUEUE } from "@/lib/jobs/medication-intake-import-queue"; const SESSION_OK = { session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, diff --git a/src/app/api/oura/callback/route.ts b/src/app/api/oura/callback/route.ts index 26b2e677e..350cdaf5c 100644 --- a/src/app/api/oura/callback/route.ts +++ b/src/app/api/oura/callback/route.ts @@ -27,10 +27,8 @@ import { markReconnected } from "@/lib/integrations/status"; import { OURA_SYNC_QUEUE } from "@/lib/jobs/integration-poll-queues"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { integrationBackfillSourceOptions } from "@/lib/jobs/integration-backfill-admission"; -import { - OURA_BACKFILL_QUEUE, - type OuraBackfillPayload, -} from "@/lib/jobs/oura-backfill"; +import { type OuraBackfillPayload } from "@/lib/jobs/oura-backfill"; +import { OURA_BACKFILL_QUEUE } from "@/lib/jobs/oura-backfill-queue"; import { NextRequest, NextResponse } from "next/server"; /** diff --git a/src/app/api/strava/callback/route.ts b/src/app/api/strava/callback/route.ts index e94544c3c..0bf259bf9 100644 --- a/src/app/api/strava/callback/route.ts +++ b/src/app/api/strava/callback/route.ts @@ -18,7 +18,7 @@ import { } from "@/lib/oauth/signed-state"; import { runOAuthCallbackOnce } from "@/lib/oauth/callback-claim"; import { markReconnected } from "@/lib/integrations/status"; -import { STRAVA_BACKFILL_QUEUE } from "@/lib/jobs/strava-backfill"; +import { STRAVA_BACKFILL_QUEUE } from "@/lib/jobs/strava-backfill-queue"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { NextRequest, NextResponse } from "next/server"; diff --git a/src/app/api/whoop/callback/route.ts b/src/app/api/whoop/callback/route.ts index 9bf1cef17..036c54565 100644 --- a/src/app/api/whoop/callback/route.ts +++ b/src/app/api/whoop/callback/route.ts @@ -14,7 +14,7 @@ import { import { getUserWhoopCredentials } from "@/lib/whoop/credentials"; import { WHOOP_OAUTH_STATE_COOKIE } from "@/lib/whoop/oauth-state"; import { buildReturnSchemeRedirect } from "@/lib/whoop/return-scheme"; -import { WHOOP_BACKFILL_QUEUE } from "@/lib/jobs/whoop-backfill"; +import { WHOOP_BACKFILL_QUEUE } from "@/lib/jobs/whoop-backfill-queue"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { markReconnected } from "@/lib/integrations/status"; import { Prisma } from "@/generated/prisma/client"; diff --git a/src/lib/fitbit/jobs/definitions.ts b/src/lib/fitbit/jobs/definitions.ts new file mode 100644 index 000000000..a4308c82d --- /dev/null +++ b/src/lib/fitbit/jobs/definitions.ts @@ -0,0 +1,98 @@ +/** + * Every queue the Fitbit integration runs, as one block of the worker + * registry. + * + * Google turns the classic Fitbit Web API off on 2026-10-30 + * (`lib/fitbit/retirement.ts`). Until the integration is removed it keeps + * syncing; removing it then is this folder plus the one line in + * `jobs/registry/integration-sync.ts` that spreads these definitions in. + */ +import { bootStaggerSecondsFor } from "@/lib/jobs/integration-backfill-admission"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + defineJob, + type BootDiscovery, + type JobDefinition, +} from "@/lib/jobs/registry/define-job"; + +import { + FITBIT_BACKFILL_CONCURRENCY, + FITBIT_BACKFILL_QUEUE, + FITBIT_OAUTH_STATE_CLEANUP_CRON, + FITBIT_OAUTH_STATE_CLEANUP_QUEUE, + FITBIT_SLEEP_REPAIR_CONCURRENCY, + FITBIT_SLEEP_REPAIR_QUEUE, + FITBIT_SYNC_CRON, + FITBIT_SYNC_QUEUE, +} from "./queues"; + +export const FITBIT_JOBS: readonly JobDefinition[] = [ + // The cron walks every connection; serial so a backfill-heavy tick never + // crowds the request pool. + defineJob({ + queue: FITBIT_SYNC_QUEUE, + cron: FITBIT_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/fitbit-sync")).handleFitbitSync, + }), + // The source queue keeps the per-connection singleton and forwards + // admitted work to the shared drain. + defineJob({ + queue: FITBIT_BACKFILL_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-connection full-history backfill; boot-discovery driven and self-converging.", + }, + work: { localConcurrency: FITBIT_BACKFILL_CONCURRENCY }, + load: async ({ boss }) => + ( + await import("@/lib/jobs/integration-backfill-forward") + ).forwardToBackfillAdmission(boss, "fitbit-backfill"), + }), + // One-shot sleep duplicate repair: a bounded sleep re-read per connection, + // forwarded to the shared drain. + defineJob({ + queue: FITBIT_SLEEP_REPAIR_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-connection one-shot sleep re-read; discovery drops the connection once it is marked repaired.", + }, + work: { localConcurrency: FITBIT_SLEEP_REPAIR_CONCURRENCY }, + load: async ({ boss }) => + ( + await import("@/lib/jobs/integration-backfill-forward") + ).forwardToBackfillAdmission(boss, "fitbit-sleep-repair"), + }), + defineJob({ + queue: FITBIT_OAUTH_STATE_CLEANUP_QUEUE, + cron: FITBIT_OAUTH_STATE_CLEANUP_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/fitbit-sync")) + .handleFitbitOAuthStateCleanup, + }), +]; + +/** The Fitbit backfill and sleep repair, offered again at every boot. */ +export const FITBIT_BOOT_DISCOVERY: readonly BootDiscovery[] = [ + { + name: "fitbit-backfill", + run: async () => + ( + await import("@/lib/jobs/fitbit-backfill") + ).enqueueBootTimeFitbitBackfill(bootStaggerSecondsFor("fitbit-backfill")), + }, + { + name: "fitbit-sleep-repair", + run: async () => + ( + await import("@/lib/jobs/fitbit-sleep-repair") + ).enqueueBootTimeFitbitSleepRepair( + bootStaggerSecondsFor("fitbit-sleep-repair"), + ), + }, +]; diff --git a/src/lib/fitbit/jobs/queues.ts b/src/lib/fitbit/jobs/queues.ts new file mode 100644 index 000000000..1bd11c674 --- /dev/null +++ b/src/lib/fitbit/jobs/queues.ts @@ -0,0 +1,30 @@ +/** + * The queue facts of the Fitbit jobs: names and concurrency. They live apart + * from the handlers so the worker registry and every sender can read them + * without loading the Fitbit sync. + */ +export const FITBIT_SLEEP_REPAIR_QUEUE = "fitbit-sleep-repair"; + +/** + * Serial concurrency — a repair walks a year of sleep history for one account + * in 30-day requests against the tight per-user budget; concurrency-1 keeps it + * from crowding the request pool, matching `FITBIT_BACKFILL_CONCURRENCY`. + */ +export const FITBIT_SLEEP_REPAIR_CONCURRENCY = 1; + +export const FITBIT_BACKFILL_QUEUE = "fitbit-backfill"; + +/** + * Serial concurrency — a backfill walks years of history for one account and is + * rate-bounded by Google's per-app quota; concurrency-1 keeps it from crowding + * the request pool, matching `WHOOP_BACKFILL_CONCURRENCY`. + */ +export const FITBIT_BACKFILL_CONCURRENCY = 1; + +/** Hourly Fitbit poll (no webhook). */ +export const FITBIT_SYNC_QUEUE = "fitbit-sync"; +export const FITBIT_SYNC_CRON = "8 * * * *"; + +/** Daily sweep of the Fitbit OAuth state ledger. */ +export const FITBIT_OAUTH_STATE_CLEANUP_QUEUE = "fitbit-oauth-state-cleanup"; +export const FITBIT_OAUTH_STATE_CLEANUP_CRON = "24 3 * * *"; diff --git a/src/lib/jobs/__tests__/apple-health-import-legacy-bridge.test.ts b/src/lib/jobs/__tests__/apple-health-import-legacy-bridge.test.ts index 1b50253ee..30191031c 100644 --- a/src/lib/jobs/__tests__/apple-health-import-legacy-bridge.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-legacy-bridge.test.ts @@ -25,10 +25,10 @@ vi.mock("@/lib/jobs/boss-instance", () => ({ import { APPLE_HEALTH_IMPORT_PARSER_REVISION, APPLE_HEALTH_IMPORT_SEND_OPTIONS, - APPLE_HEALTH_IMPORT_V2_QUEUE, migrateLegacyAppleHealthImport, type AppleHealthImportPayload, } from "../apple-health-import-worker"; +import { APPLE_HEALTH_IMPORT_V2_QUEUE } from "@/lib/jobs/apple-health-import-queue"; const payload: AppleHealthImportPayload = { userId: "user-1", diff --git a/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts b/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts index 6cfaf5e4c..270fe9565 100644 --- a/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts @@ -1,5 +1,5 @@ /** - * v1.32.1 (issue #588) — periodic orphan-`ImportJob` reconcile. + * Periodic orphan-`ImportJob` reconcile (issue #588). * * Before this change `reconcileOrphanImportJobs()` only ran once, at * worker boot. A worker that crashed mid-extraction (OOM, restart) and @@ -11,10 +11,9 @@ * that one boot-time pass. These tests cover both the periodic-tick * wiring (queue provisioned, cron scheduled, handler bound) and the * reconcile logic itself (a genuinely-abandoned row flips to `failed`; - * a row another worker is still actively running does not). + * a row another worker is still actively running does not). The wiring is + * read off the worker registry. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ @@ -45,19 +44,22 @@ vi.mock("@/lib/jobs/boss-instance", () => ({ import { APPLE_HEALTH_IMPORT_PARSER_REVISION, - APPLE_HEALTH_IMPORT_V2_QUEUE, - IMPORT_JOB_RECONCILE_CRON, - IMPORT_JOB_RECONCILE_QUEUE, handleImportJobReconcileTick, reconcileOrphanImportJobs, stagedImportFilesInUse, } from "../apple-health-import-worker"; +import { + APPLE_HEALTH_IMPORT_V2_QUEUE, + IMPORT_JOB_RECONCILE_CRON, + IMPORT_JOB_RECONCILE_QUEUE, +} from "@/lib/jobs/apple-health-import-queue"; import { sweepStaleImportStaging } from "@/lib/import/apple-health-staging"; - -const maintenanceSource = readFileSync( - join(process.cwd(), "src/lib/jobs/reminder/register-maintenance.ts"), - "utf8", -); +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; beforeEach(() => { vi.resetAllMocks(); @@ -66,30 +68,21 @@ beforeEach(() => { }); describe("periodic reconcile — wiring", () => { - it("provisions the queue, schedules a 15-minute cron, and binds the handler", () => { + it("provisions the queue, schedules a 15-minute cron, and binds the handler", async () => { expect(IMPORT_JOB_RECONCILE_QUEUE).toBe("apple-health-import-reconcile"); expect(IMPORT_JOB_RECONCILE_CRON).toBe("*/15 * * * *"); - const allQueues = maintenanceSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bIMPORT_JOB_RECONCILE_QUEUE\b/); - - const schedules = maintenanceSource.match( - /const schedules[\s\S]*?=\s*\[([\s\S]*?)\];/, - ); - expect(schedules).not.toBeNull(); - // The third element pins the retry policy. It is load-bearing now that + expect(groupOf(IMPORT_JOB_RECONCILE_QUEUE).name).toBe("maintenance"); + const job = jobOf(IMPORT_JOB_RECONCILE_QUEUE); + expect(job.cron).toBe(IMPORT_JOB_RECONCILE_CRON); + // The schedule options pin the retry policy. It is load-bearing now that // the handler reports a failed outcome instead of swallowing: without // `cronIsTheRetry` a database outage would run the same doomed sweep // three times per tick, and the next tick is already the retry. - expect(schedules![1]).toMatch( - /\[IMPORT_JOB_RECONCILE_QUEUE,\s*IMPORT_JOB_RECONCILE_CRON,\s*cronIsTheRetry\]/, - ); + expect(job.scheduleOptions).toBe(cronIsTheRetry); - expect(maintenanceSource).toMatch( - /createAndWork\(\s*boss,\s*IMPORT_JOB_RECONCILE_QUEUE[\s\S]{0,120}handleImportJobReconcileTick/, + expect(await loadedHandler(IMPORT_JOB_RECONCILE_QUEUE)).toBe( + handleImportJobReconcileTick, ); }); }); diff --git a/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts b/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts index 8a3ac767b..1d645df5c 100644 --- a/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts @@ -1,6 +1,5 @@ /** - * v1.28.33 (issue #486) — send-policy pins for the Apple Health import - * queue. + * Send-policy pins for the Apple Health import queue (issue #486). * * The queue defaults (retryLimit 2, expireInSeconds 900) redelivered a * long-running import after its staged `/tmp` upload was already @@ -21,20 +20,27 @@ vi.mock("@/lib/db", () => ({ import { APPLE_HEALTH_IMPORT_PARSER_REVISION, - APPLE_HEALTH_IMPORT_LEGACY_QUEUE, APPLE_HEALTH_IMPORT_SEND_OPTIONS, - APPLE_HEALTH_IMPORT_V2_QUEUE, } from "../apple-health-import-worker"; +import { + APPLE_HEALTH_IMPORT_LEGACY_QUEUE, + APPLE_HEALTH_IMPORT_V2_QUEUE, +} from "@/lib/jobs/apple-health-import-queue"; +import { + handleAppleHealthImportJobs, + handleLegacyAppleHealthImportJobs, +} from "@/lib/jobs/reminder/maintenance-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; const ROUTE_FILES = [ "src/app/api/import/apple-health-export/route.ts", "src/app/api/admin/import-apple-health-export/route.ts", ]; -const maintenanceSource = readFileSync( - join(process.cwd(), "src/lib/jobs/reminder/register-maintenance.ts"), - "utf8", -); const workerSource = readFileSync( join(process.cwd(), "src/lib/jobs/apple-health-import-worker.ts"), "utf8", @@ -80,21 +86,18 @@ describe("apple health import — parser revision boundary", () => { expect(APPLE_HEALTH_IMPORT_LEGACY_QUEUE).toBe("apple-health-import"); }); - it("drains current-parser jobs separately while bridging legacy backlog", () => { - const allQueues = maintenanceSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, + it("drains current-parser jobs separately while bridging legacy backlog", async () => { + expect(groupOf(APPLE_HEALTH_IMPORT_V2_QUEUE).name).toBe("maintenance"); + expect(groupOf(APPLE_HEALTH_IMPORT_LEGACY_QUEUE).name).toBe("maintenance"); + // Only the current-parser queue runs in the heavy lane; the legacy + // bridge re-sends its backlog onto it. + expect(jobOf(APPLE_HEALTH_IMPORT_V2_QUEUE).lane).toBe(true); + expect(jobOf(APPLE_HEALTH_IMPORT_LEGACY_QUEUE).lane).toBeUndefined(); + expect(await loadedHandler(APPLE_HEALTH_IMPORT_V2_QUEUE)).toBe( + handleAppleHealthImportJobs, ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bAPPLE_HEALTH_IMPORT_V2_QUEUE\b/); - expect(allQueues![1]).toMatch(/\bAPPLE_HEALTH_IMPORT_LEGACY_QUEUE\b/); - // The v2 window is the wider of the two: its binding wraps the batch in - // a loop that keeps the first failed outcome, so the handler call sits - // further from the queue name than the legacy bridge's does. - expect(maintenanceSource).toMatch( - /createAndWork[\s\S]{0,200}APPLE_HEALTH_IMPORT_V2_QUEUE[\s\S]{0,600}handleAppleHealthImport/, - ); - expect(maintenanceSource).toMatch( - /createAndWork[\s\S]{0,200}APPLE_HEALTH_IMPORT_LEGACY_QUEUE[\s\S]{0,400}migrateLegacyAppleHealthImport/, + expect(await loadedHandler(APPLE_HEALTH_IMPORT_LEGACY_QUEUE)).toBe( + handleLegacyAppleHealthImportJobs, ); }); diff --git a/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts b/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts index bb6147eab..e9daf5438 100644 --- a/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts +++ b/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts @@ -1,49 +1,35 @@ /** - * v1.11.1 — queue-registration guard for the combined Coach memory-refresh - * job (rolling conversation summary + durable fact extraction). + * Queue-registration guard for the combined Coach memory-refresh job + * (rolling conversation summary + durable fact extraction). * - * Same source-text-grep approach as the period-narrative / stress-strain - * guards: assert the queue is imported, registered in `allQueues`, and wired - * to a `createAndWork` handler that runs the refresh pipeline — without booting - * pg-boss + Prisma. An unregistered queue silently never runs (the recurring - * past bug this guards against; v1.4.37 W10 caught exactly this class). + * Reads the worker registry: the queue is declared in the status group and + * its loader resolves to the handler that runs the refresh pipeline. An + * unregistered queue silently never runs (the dead-queue class). */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -// v1.18.1 — the coach-memory-refresh wiring moved out of the 2143-LOC -// reminder-worker boot file into the status registrar. The dead-queue guard -// follows the wiring there. -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-status.ts"); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { COACH_MEMORY_REFRESH_QUEUE } from "@/lib/ai/coach/coach-memory-shared"; +import { handleCoachMemoryRefresh } from "@/lib/jobs/reminder/status-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -function allQueuesBlock(): string { - const m = source.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(m).not.toBeNull(); - return m![1]; -} - -describe("reminder-worker — coach-memory-refresh wiring", () => { - it("imports the queue symbol + worker from the coach-memory modules", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/ai\/coach\/coach-memory-shared["']/, - ); - expect(source).toMatch(/\bCOACH_MEMORY_REFRESH_QUEUE\b/); - expect(source).toMatch( - /from\s*["']@\/lib\/ai\/coach\/coach-memory-refresh-worker["']/, - ); - expect(source).toMatch(/\brunCoachMemoryRefresh\b/); +describe("coach-memory-refresh registration", () => { + it("declares the coach-memory-refresh queue in the status group", () => { + expect(groupOf(COACH_MEMORY_REFRESH_QUEUE).name).toBe("status"); }); - it("registers the coach-memory-refresh queue in allQueues", () => { - expect(allQueuesBlock()).toMatch(/\bCOACH_MEMORY_REFRESH_QUEUE\b/); + it("runs one refresh at a time and has no cron", () => { + const job = jobOf(COACH_MEMORY_REFRESH_QUEUE); + expect(job.work).toEqual({ localConcurrency: 1 }); + expect(job.cron).toBeUndefined(); }); - it("registers a createAndWork handler that runs the refresh pipeline", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}COACH_MEMORY_REFRESH_QUEUE[\s\S]{0,600}runCoachMemoryRefresh/, + it("binds the handler that runs the refresh pipeline", async () => { + expect(await loadedHandler(COACH_MEMORY_REFRESH_QUEUE)).toBe( + handleCoachMemoryRefresh, ); }); }); diff --git a/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts b/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts index 0567b4e40..1999316f6 100644 --- a/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts +++ b/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts @@ -1,58 +1,50 @@ /** - * v1.18.7 — coach-message-cleanup queue registration guard. + * Coach-message-cleanup queue registration guard. * - * Same source-text-grep approach as the other queue-wiring guards: assert the - * queue is registered in `allQueues`, scheduled, and wired to a `createAndWork` - * handler — without booting pg-boss + Prisma. An unregistered queue silently - * never drains (the recurring v1.4.37 dead-queue bug), which would leave the - * encrypted Coach history growing unbounded with the retention job dark. + * Reads the worker registry: the queue is declared in the maintenance group, + * scheduled with the cron as its retry, and bound to the cleanup handler. An + * unregistered queue silently never drains (the dead-queue class), which + * would leave the encrypted Coach history growing unbounded with the + * retention job dark. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-maintenance.ts", +import { handleCoachMessageCleanup } from "@/lib/jobs/reminder/cleanup-handlers"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; + +const QUEUE = "coach-message-cleanup"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "cleanup-handlers.ts"), + "utf8", ); -const workerSource = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync( - join(__dirname, "..", "reminder", "cleanup-handlers.ts"), - "utf8", - ); -describe("reminder-worker — coach-message-cleanup wiring", () => { - it("imports the cleanup handler from the cleanup-handlers module", () => { - expect(workerSource).toMatch(/\bhandleCoachMessageCleanup\b/); - expect(workerSource).toMatch(/\bCoachMessageCleanupPayload\b/); +describe("coach-message-cleanup registration", () => { + it("declares the queue in the maintenance group", () => { + expect(groupOf(QUEUE).name).toBe("maintenance"); }); - it("registers the coach-message-cleanup queue in the allQueues loop", () => { - const allQueuesMatch = workerSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bCOACH_MESSAGE_CLEANUP_QUEUE\b/); + it("schedules the nightly cron with the cron as the retry", () => { + const job = jobOf(QUEUE); + expect(job.cron).toBe("50 3 * * *"); + expect(job.scheduleOptions).toBe(cronIsTheRetry); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("schedules the coach-message-cleanup cron", () => { - expect(workerSource).toMatch( - /\[COACH_MESSAGE_CLEANUP_QUEUE,\s*COACH_MESSAGE_CLEANUP_CRON,\s*cronIsTheRetry\]/, - ); - }); - - it("registers a createAndWork handler for the queue", () => { - expect(workerSource).toMatch( - /createAndWork[\s\S]{0,200}COACH_MESSAGE_CLEANUP_QUEUE[\s\S]{0,200}handleCoachMessageCleanup/, - ); + it("binds the cleanup handler", async () => { + expect(await loadedHandler(QUEUE)).toBe(handleCoachMessageCleanup); }); it("prunes coach messages inside the handler", () => { - expect(workerSource).toMatch( + expect(handlerSource).toMatch( /handleCoachMessageCleanup[\s\S]{0,400}cleanupOldCoachMessages/, ); }); diff --git a/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts b/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts index b93282312..96bb7da83 100644 --- a/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts +++ b/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts @@ -1,48 +1,46 @@ /** - * v1.30.3 — cumulative-PersonalRecord rederivation queue - * registration guard. + * Cumulative-PersonalRecord rederivation queue registration guard. * - * Same source-text-grep approach as the step-consolidation-repair guard: - * assert the queue is registered in `allQueues`, a `createAndWork` handler is - * wired against it, and the boot-discovery enqueue helper fires — without - * booting pg-boss + Prisma. An unregistered queue silently never drains - * (the v1.4.37 dead-queue class). + * Reads the worker registry: the queue is declared in the rollup group, + * bound to the rederivation handler, and offered again by the rollup boot + * discovery. An unregistered queue silently never drains (the dead-queue + * class). */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-rollup.ts"); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { handleCumulativePrRederive } from "@/lib/jobs/reminder/rollup-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { + CUMULATIVE_PR_REDERIVE_CONCURRENCY, + CUMULATIVE_PR_REDERIVE_QUEUE, +} from "@/lib/personal-records/cumulative-pr-rederivation"; -describe("reminder-worker — cumulative-pr-rederive wiring", () => { - it("imports the queue symbols from the rederivation module", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/personal-records\/cumulative-pr-rederivation["']/, - ); - expect(source).toMatch(/\bCUMULATIVE_PR_REDERIVE_QUEUE\b/); - expect(source).toMatch(/\brunCumulativePrRederivationForUser\b/); - expect(source).toMatch(/\benqueueBootTimeCumulativePrRederivation\b/); +describe("cumulative-pr-rederive registration", () => { + it("declares the queue in the rollup group", () => { + expect(groupOf(CUMULATIVE_PR_REDERIVE_QUEUE).name).toBe("rollup"); }); - it("registers the queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bCUMULATIVE_PR_REDERIVE_QUEUE\b/); + it("is a per-user exclusive one-shot at the module's concurrency", () => { + const job = jobOf(CUMULATIVE_PR_REDERIVE_QUEUE); + expect(job.policy?.policy).toBe("exclusive"); + expect(job.work).toEqual({ + localConcurrency: CUMULATIVE_PR_REDERIVE_CONCURRENCY, + }); + expect(job.cron).toBeUndefined(); }); - it("registers a createAndWork handler against the queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}CUMULATIVE_PR_REDERIVE_QUEUE[\s\S]{0,400}runCumulativePrRederivationForUser/, + it("binds the rederivation handler", async () => { + expect(await loadedHandler(CUMULATIVE_PR_REDERIVE_QUEUE)).toBe( + handleCumulativePrRederive, ); }); - it("fires the boot-discovery enqueue helper", () => { - expect(source).toMatch( - /await enqueueBootTimeCumulativePrRederivation\([^)]*\)/, - ); + it("runs the boot discovery pass", () => { + expect(discoveryOf("rollup")).toContain("cumulative-pr-rederive"); }); }); diff --git a/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts b/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts index 8c9158af2..8be8a1395 100644 --- a/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts +++ b/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts @@ -1,67 +1,55 @@ /** - * v1.15 — cycle-reminder queue registration guard. + * Cycle-reminder queue registration guard. * - * Same source-text-grep approach as the mood-reminder / drain / mean- - * consolidation registration guards: assert the cycle-reminder queue is - * declared, listed in `allQueues` (the v1.4.37 dead-queue lesson — an - * unregistered queue is never provisioned and the schedule silently - * no-ops), scheduled on its cron, and wired to a `createAndWork` handler that - * delegates to `runCycleReminderTick`. No pg-boss / Prisma boot. + * Reads the worker registry: the cycle-reminder queue is declared in the + * reminders group (an unregistered queue is never provisioned and the + * schedule silently no-ops), scheduled on its 15-minute cron, and bound to + * the handler that delegates to `runCycleReminderTick`. No pg-boss / Prisma + * boot. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the reminder-dispatch wiring moved out of the 2143-LOC -// reminder-worker boot file into the reminders registrar. The dead-queue guard -// follows the wiring there (the handler module is still concatenated for the -// handler-symbol assertions). -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-reminders.ts", +import { handleCycleReminderCheck } from "@/lib/jobs/reminder/mood-cycle-checks"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; + +const CYCLE_REMINDER_QUEUE = "cycle-reminder-check"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "mood-cycle-checks.ts"), + "utf8", ); -const source = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync( - join(__dirname, "..", "reminder", "mood-cycle-checks.ts"), - "utf8", - ); -describe("reminder-worker — cycle-reminder wiring", () => { +describe("cycle-reminder registration", () => { it("imports the cycle-reminder tick runner", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/cycle-reminder["']/); - expect(source).toMatch(/\brunCycleReminderTick\b/); + expect(handlerSource).toMatch( + /from\s*["']@\/lib\/jobs\/cycle-reminder["']/, + ); + expect(handlerSource).toMatch(/\brunCycleReminderTick\b/); }); - it("declares the queue + cron constants", () => { - expect(source).toMatch( - /CYCLE_REMINDER_QUEUE\s*=\s*["']cycle-reminder-check["']/, - ); - expect(source).toMatch( - /CYCLE_REMINDER_CRON\s*=\s*["']\*\/15 \* \* \* \*["']/, - ); + it("declares the cycle-reminder queue in the reminders group", () => { + expect(groupOf(CYCLE_REMINDER_QUEUE).name).toBe("reminders"); }); - it("registers the cycle-reminder queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bCYCLE_REMINDER_QUEUE\b/); + it("schedules the cycle-reminder cron, single-flight", () => { + const job = jobOf(CYCLE_REMINDER_QUEUE); + expect(job.cron).toBe("*/15 * * * *"); + expect(job.scheduleOptions).toBeUndefined(); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("schedules the cycle-reminder cron", () => { - expect(source).toMatch(/\[CYCLE_REMINDER_QUEUE,\s*CYCLE_REMINDER_CRON\]/); - }); - - it("registers a createAndWork handler that runs the cycle-reminder tick", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}CYCLE_REMINDER_QUEUE[\s\S]{0,200}handleCycleReminderCheck/, + it("binds the handler that runs the cycle-reminder tick", async () => { + expect(await loadedHandler(CYCLE_REMINDER_QUEUE)).toBe( + handleCycleReminderCheck, ); - expect(source).toMatch( + expect(handlerSource).toMatch( /handleCycleReminderCheck[\s\S]{0,400}runCycleReminderTick/, ); }); diff --git a/src/lib/jobs/__tests__/daily-briefing.test.ts b/src/lib/jobs/__tests__/daily-briefing.test.ts index 8c7a5ca43..67c01ed9e 100644 --- a/src/lib/jobs/__tests__/daily-briefing.test.ts +++ b/src/lib/jobs/__tests__/daily-briefing.test.ts @@ -1,19 +1,47 @@ /** * Daily-briefing fallback cron: tz slot-gating + result accounting, plus - * the dead-queue wiring guard (imported / in allQueues / scheduled / worked). + * the dead-queue wiring guard read off the worker registry (declared, + * scheduled, bound) and the morning push the finalised refresh fires. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, it, expect, vi } from "vitest"; + +const { runMorningDigestRefresh, maybeDispatchPush } = vi.hoisted(() => ({ + runMorningDigestRefresh: vi.fn(), + maybeDispatchPush: vi.fn(async () => "sent"), +})); +vi.mock("@/lib/jobs/morning-digest-refresh", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/morning-digest-refresh") + >()), + runMorningDigestRefresh, +})); +vi.mock("@/lib/daily/daily-briefing-push", async (importOriginal) => ({ + ...(await importOriginal()), + maybeDispatchDailyBriefing: maybeDispatchPush, +})); +vi.mock("@/lib/jobs/reminder/shared", async (importOriginal) => ({ + ...(await importOriginal()), + getWorkerPrisma: () => ({}), +})); import type { PrismaClient } from "@/generated/prisma/client"; +import { runDailyBriefingTick } from "@/lib/jobs/daily-briefing"; import { - runDailyBriefingTick, DAILY_BRIEFING_QUEUE, DAILY_BRIEFING_CRON, -} from "@/lib/jobs/daily-briefing"; +} from "@/lib/jobs/daily-briefing-queue"; import type { maybeDispatchDailyBriefing } from "@/lib/daily/daily-briefing-push"; +import type { MorningDigestRefreshPayload } from "@/lib/jobs/morning-digest-refresh-shared"; +import { + handleDailyBriefing, + handleMorningDigestRefresh, +} from "@/lib/jobs/reminder/status-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import type { Job } from "pg-boss"; // 06:00Z → 08:00 Berlin (the fallback hour) but 02:00 in New York (not the // slot), so a mixed-tz cohort exercises the per-user local-hour gate. @@ -76,38 +104,44 @@ describe("runDailyBriefingTick — slot gating", () => { }); }); -describe("reminder-worker — daily-briefing wiring", () => { - const registrar = readFileSync( - join(__dirname, "..", "reminder", "register-status.ts"), - "utf8", - ); +describe("daily-briefing registration", () => { + function refreshJob(): Job { + return { + id: "j1", + name: "morning-digest-refresh", + data: { userId: "u1", localDate: "2026-07-16" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }; + } - it("imports the queue symbols from the daily-briefing module", () => { - expect(registrar).toMatch(/from\s*["']@\/lib\/jobs\/daily-briefing["']/); - expect(registrar).toMatch(/\bDAILY_BRIEFING_QUEUE\b/); - expect(registrar).toMatch(/\bDAILY_BRIEFING_CRON\b/); - expect(registrar).toMatch(/\brunDailyBriefingTick\b/); + it("declares the queue in the status group and schedules the cron", () => { + expect(groupOf(DAILY_BRIEFING_QUEUE).name).toBe("status"); + const job = jobOf(DAILY_BRIEFING_QUEUE); + expect(job.cron).toBe(DAILY_BRIEFING_CRON); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("registers the queue in allQueues and schedules the cron", () => { - const allQueues = registrar.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bDAILY_BRIEFING_QUEUE\b/); - expect(registrar).toMatch( - /\[DAILY_BRIEFING_QUEUE,\s*DAILY_BRIEFING_CRON\]/, - ); + it("binds the handler that runs the tick", async () => { + expect(await loadedHandler(DAILY_BRIEFING_QUEUE)).toBe(handleDailyBriefing); }); - it("wires a createAndWork handler that runs the tick", () => { - expect(registrar).toMatch( - /createAndWork[\s\S]{0,200}DAILY_BRIEFING_QUEUE[\s\S]{0,400}runDailyBriefingTick/, + it("fires the finalisation-hook push after a finalised morning refresh", async () => { + runMorningDigestRefresh.mockResolvedValueOnce({ status: "finalised" }); + await handleMorningDigestRefresh([refreshJob()]); + expect(maybeDispatchPush).toHaveBeenCalledTimes(1); + expect(maybeDispatchPush).toHaveBeenCalledWith( + expect.anything(), + "u1", + expect.any(Date), ); - }); - it("fires the finalisation-hook push after a finalised morning refresh", () => { - expect(registrar).toMatch( - /result\.status\s*===\s*["']finalised["'][\s\S]{0,300}maybeDispatchDailyBriefing/, - ); + maybeDispatchPush.mockClear(); + runMorningDigestRefresh.mockResolvedValueOnce({ status: "already-final" }); + await handleMorningDigestRefresh([refreshJob()]); + expect(maybeDispatchPush).not.toHaveBeenCalled(); }); it("uses an every-15-min cron for the fallback slot", () => { diff --git a/src/lib/jobs/__tests__/data-arrival-queue.test.ts b/src/lib/jobs/__tests__/data-arrival-queue.test.ts index a86f8057f..b97123667 100644 --- a/src/lib/jobs/__tests__/data-arrival-queue.test.ts +++ b/src/lib/jobs/__tests__/data-arrival-queue.test.ts @@ -1,5 +1,5 @@ /** - * v1.4.37 dead-queue guard for the `data-arrival` queue. + * Dead-queue guard for the `data-arrival` queue. * * Four facts have to hold together or the spine is silently inert: the queue * name is provisioned, a handler drains it, the queue carries a policy that @@ -7,83 +7,65 @@ * is itself registered. Any one of them missing fails open in the worst way — * the emits succeed, the jobs vanish, and nothing anywhere reports a problem. * - * Source-text assertions over the registrar modules, matching the shape of the - * sibling `*-queue.test.ts` guards. + * Reads the worker registry for the queue facts; the retention handler's body + * is read as text. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -const statusRegistrar = readFileSync( - join(__dirname, "..", "reminder", "register-status.ts"), +import { DATA_ARRIVAL_QUEUE } from "@/lib/arrivals/emit-shared"; +import { handleDataArrival } from "@/lib/jobs/data-arrival"; +import { handleArrivalReactionCleanup } from "@/lib/jobs/reminder/cleanup-handlers"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; + +const ARRIVAL_REACTION_CLEANUP_QUEUE = "arrival-reaction-cleanup"; + +const cleanupHandlers = readFileSync( + join(__dirname, "..", "reminder", "cleanup-handlers.ts"), "utf8", ); -const maintenanceSource = - readFileSync( - join(__dirname, "..", "reminder", "register-maintenance.ts"), - "utf8", - ) + - readFileSync( - join(__dirname, "..", "reminder", "cleanup-handlers.ts"), - "utf8", - ); - describe("data-arrival queue wiring", () => { - it("provisions the queue in the allQueues list", () => { - const allQueues = statusRegistrar.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bDATA_ARRIVAL_QUEUE\b/); + it("provisions the queue in the status group", () => { + expect(groupOf(DATA_ARRIVAL_QUEUE).name).toBe("status"); }); - it("binds a createAndWork handler that drains it", () => { - expect(statusRegistrar).toMatch( - /createAndWork[\s\S]{0,200}DATA_ARRIVAL_QUEUE[\s\S]{0,200}handleDataArrival/, - ); + it("binds the handler that drains it", async () => { + expect(await loadedHandler(DATA_ARRIVAL_QUEUE)).toBe(handleDataArrival); }); it("carries the exclusive policy its singleton keys depend on", () => { // Under pg-boss's default `standard` policy NO index covers // `singleton_key`, so the day-scoped keys would coalesce nothing at all and - // a chatty ingest would queue one job per batch. This assertion is anchored - // INSIDE the extracted policy table so an unrelated mention of the queue - // name elsewhere in the file cannot satisfy it. - const table = statusRegistrar.match( - /const queuePolicies: QueuePolicyTable\s*=\s*\{([\s\S]*?)\n\};/, - ); - expect(table).not.toBeNull(); - expect(table![1]).toMatch( - /\[DATA_ARRIVAL_QUEUE\][\s\S]{0,200}policy:\s*"exclusive"/, - ); + // a chatty ingest would queue one job per batch. + expect(jobOf(DATA_ARRIVAL_QUEUE).policy?.policy).toBe("exclusive"); }); it("has no cron schedule — it is send-only, driven by ingest", () => { - // Tolerates the optional third schedule element, so a cron sneaking in - // with send options still trips this. - expect(statusRegistrar).not.toMatch( - /\[\s*DATA_ARRIVAL_QUEUE,\s*\w+_CRON\s*[,\]]/, - ); + const job = jobOf(DATA_ARRIVAL_QUEUE); + expect(job.cron).toBeUndefined(); + expect(job.scheduleOptions).toBeUndefined(); }); - it("registers the retention job that prunes the markers", () => { - const allQueues = maintenanceSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bARRIVAL_REACTION_CLEANUP_QUEUE\b/); - expect(maintenanceSource).toMatch( - /\[\s*ARRIVAL_REACTION_CLEANUP_QUEUE,\s*ARRIVAL_REACTION_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, - ); - expect(maintenanceSource).toMatch( - /createAndWork[\s\S]{0,200}ARRIVAL_REACTION_CLEANUP_QUEUE[\s\S]{0,200}handleArrivalReactionCleanup/, + it("registers the retention job that prunes the markers", async () => { + expect(groupOf(ARRIVAL_REACTION_CLEANUP_QUEUE).name).toBe("maintenance"); + const job = jobOf(ARRIVAL_REACTION_CLEANUP_QUEUE); + expect(job.cron).toBe("10 4 * * *"); + expect(job.scheduleOptions).toBe(cronIsTheRetry); + expect(await loadedHandler(ARRIVAL_REACTION_CLEANUP_QUEUE)).toBe( + handleArrivalReactionCleanup, ); }); it("the retention handler actually deletes arrival reactions by age", () => { - expect(maintenanceSource).toMatch( + expect(cleanupHandlers).toMatch( /handleArrivalReactionCleanup[\s\S]{0,600}arrivalReaction\.deleteMany/, ); }); diff --git a/src/lib/jobs/__tests__/data-backup-send-policy.test.ts b/src/lib/jobs/__tests__/data-backup-send-policy.test.ts index c7e470c91..a24ba53ce 100644 --- a/src/lib/jobs/__tests__/data-backup-send-policy.test.ts +++ b/src/lib/jobs/__tests__/data-backup-send-policy.test.ts @@ -29,7 +29,8 @@ import { import { OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_SEND_OPTIONS, -} from "../offhost-backup"; +} from "@/lib/jobs/offhost-backup-queue"; +import { jobOf } from "../registry/__tests__/registry-helpers"; const read = (file: string) => readFileSync(join(process.cwd(), file), "utf8"); @@ -59,13 +60,12 @@ describe("data-backup — pg-boss send policy", () => { }); it("the weekly and nightly schedules carry the policy", () => { - const source = read("src/lib/jobs/reminder/register-maintenance.ts"); - expect(source).toMatch( - /\[\s*DATA_BACKUP_QUEUE,\s*DATA_BACKUP_CRON,\s*DATA_BACKUP_SEND_OPTIONS,?\s*\]/, - ); - expect(source).toMatch( - /\[\s*OFFHOST_BACKUP_QUEUE,\s*OFFHOST_BACKUP_CRON,\s*OFFHOST_BACKUP_SEND_OPTIONS,?\s*\]/, - ); + const weekly = jobOf(DATA_BACKUP_QUEUE); + expect(weekly.cron).toBe("0 3 * * 0"); + expect(weekly.scheduleOptions).toBe(DATA_BACKUP_SEND_OPTIONS); + const nightly = jobOf(OFFHOST_BACKUP_QUEUE); + expect(nightly.cron).toBe("30 2 * * *"); + expect(nightly.scheduleOptions).toBe(OFFHOST_BACKUP_SEND_OPTIONS); }); it("the admin run-now route sends with the same policy", () => { diff --git a/src/lib/jobs/__tests__/document-ai-run.test.ts b/src/lib/jobs/__tests__/document-ai-run.test.ts index 5fc7878f1..3f8bf1757 100644 --- a/src/lib/jobs/__tests__/document-ai-run.test.ts +++ b/src/lib/jobs/__tests__/document-ai-run.test.ts @@ -112,11 +112,11 @@ vi.mock("@/lib/jobs/boss-instance", () => ({ import { AiUnavailableError } from "@/lib/ai/capabilities/refusal"; import { DOCUMENT_AI_RUN_EXPIRE_SECONDS, - DOCUMENT_AI_RUN_QUEUE, enqueueDocumentAiRun, handleDocumentAiRunJobs, runDocumentAiRun, } from "@/lib/jobs/document-ai-run"; +import { DOCUMENT_AI_RUN_QUEUE } from "@/lib/jobs/document-ai-run-queue"; const NOW = 1_800_000_000_000; const now = () => NOW; diff --git a/src/lib/jobs/__tests__/document-index.test.ts b/src/lib/jobs/__tests__/document-index.test.ts index 3d7652ae9..456005ac2 100644 --- a/src/lib/jobs/__tests__/document-index.test.ts +++ b/src/lib/jobs/__tests__/document-index.test.ts @@ -15,11 +15,8 @@ vi.mock("@/lib/documents/auto-stage-labs", () => ({ maybeAutoStageLabFacts: vi.fn().mockResolvedValue(undefined), })); -import { - DOCUMENT_INDEX_QUEUE, - enqueueDocumentIndex, - runDocumentIndex, -} from "../document-index"; +import { enqueueDocumentIndex, runDocumentIndex } from "../document-index"; +import { DOCUMENT_INDEX_QUEUE } from "../document-index-queue"; import { indexDocumentContent } from "@/lib/documents/index-document"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { maybeAutoStageLabFacts } from "@/lib/documents/auto-stage-labs"; diff --git a/src/lib/jobs/__tests__/document-summary-catchup.test.ts b/src/lib/jobs/__tests__/document-summary-catchup.test.ts index 96c7dd4da..111a6afb7 100644 --- a/src/lib/jobs/__tests__/document-summary-catchup.test.ts +++ b/src/lib/jobs/__tests__/document-summary-catchup.test.ts @@ -32,8 +32,8 @@ import { enqueueSummaryCatchUp, runSummaryCatchUpForUser, MAX_ENQUEUES_PER_RUN, - DOCUMENT_SUMMARY_CATCHUP_QUEUE, } from "../document-summary-catchup"; +import { DOCUMENT_SUMMARY_CATCHUP_QUEUE } from "../document-summary-catchup-queue"; import { prisma } from "@/lib/db"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; diff --git a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts index 412d147fd..65c0ecb63 100644 --- a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts +++ b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts @@ -1,75 +1,136 @@ /** - * v1.4.37 — pg-boss queue registration for the nightly drain of - * per-sample APPLE_HEALTH cumulative rows. The drain helper exists - * since v1.4.30 but only ran via the admin endpoint / CLI; the - * scheduled wrapper closes the loop so the list view stops painting - * hundreds of step chunks per day. + * pg-boss queue registration for the nightly drain of per-sample + * APPLE_HEALTH cumulative rows. The drain helper once ran only via the admin + * endpoint / CLI; the scheduled wrapper closes the loop so the list view + * stops painting hundreds of step chunks per day. * - * Same source-text-grep approach as the Withings queue regression - * guard so we never have to boot pg-boss + Prisma to assert the - * scheduler wiring stays intact. + * Reads the worker registry (the queue, its cron and its binding) and runs + * the bound handler against mocked passes, so pg-boss and Prisma never boot. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; -import { describe, expect, it } from "vitest"; +const drainPerSampleCumulative = vi.hoisted(() => + vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + bucketsCollapsed: 0, + perSampleRowsDeleted: 0, + dailyRowsUpserted: 0, + }, + })), +); +vi.mock( + "@/lib/measurements/drain-per-sample-cumulative", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/drain-per-sample-cumulative") + >()), + drainPerSampleCumulative, + }), +); +vi.mock( + "@/lib/measurements/consolidate-daily-mean", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/consolidate-daily-mean") + >()), + consolidateDailyMean: vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + daysConsolidated: 0, + perSampleRowsSoftDeleted: 0, + dailyRowsUpserted: 0, + daysFailed: 0, + }, + })), + }), +); +vi.mock( + "@/lib/measurements/dense-intraday-retention", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/dense-intraday-retention") + >()), + runDenseIntradayRetention: vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + daysConsolidated: 0, + perSampleRowsSoftDeleted: 0, + hourlyRowsUpserted: 0, + dailyRowsRetired: 0, + }, + })), + }), +); +vi.mock("@/lib/jobs/reminder/shared", async (importOriginal) => ({ + ...(await importOriginal()), + getWorkerPrisma: () => ({}), +})); -// v1.18.1 — the drain-cumulative wiring moved out of the 2143-LOC -// reminder-worker boot file into the rollup registrar. The dead-queue guard -// follows the wiring there. -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-rollup.ts"); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { + DRAIN_CUMULATIVE_CRON, + DRAIN_CUMULATIVE_QUEUE, +} from "@/lib/jobs/drain-cumulative-queue"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + groupOf, + jobOf, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { DRAIN_CUMULATIVE_CUTOFF_HOURS } from "@/lib/measurements/drain-per-sample-cumulative"; -describe("reminder-worker — drainPerSampleCumulative nightly schedule", () => { +describe("drainPerSampleCumulative nightly schedule", () => { it("declares the drain queue at the documented Berlin cadence", () => { - expect(source).toMatch( - /DRAIN_CUMULATIVE_QUEUE\s*=\s*["']drain-per-sample-cumulative["']/, - ); + expect(DRAIN_CUMULATIVE_QUEUE).toBe("drain-per-sample-cumulative"); // 03:45 Europe/Berlin — between audit-log cleanup (03:15) and the // feedback aggregator (04:00). Keep the slot stable so an operator // staring at the cron table can find the job easily. - expect(source).toMatch(/DRAIN_CUMULATIVE_CRON\s*=\s*["']45 3 \* \* \*["']/); + expect(DRAIN_CUMULATIVE_CRON).toBe("45 3 * * *"); }); - it("passes the 36 hour grace window into the drain helper", () => { - // v1.4.38 — the literal `36` lives in the helper module so the - // worker, the admin route, and the CLI all read one source of - // truth. The worker still references the constant by name in the - // cutoffHours slot so the cron's behaviour is unchanged. - expect(source).toMatch(/cutoffHours:\s*DRAIN_CUMULATIVE_CUTOFF_HOURS/); - }); - - it("imports the cutoff constant and the helper from the drain module", () => { - // v1.4.38 — the import is now a multi-symbol form. Match the helper - // and the constant separately so a future contributor who adds a - // third symbol does not break the assertion. - expect(source).toMatch( - /from\s*["']@\/lib\/measurements\/drain-per-sample-cumulative["']/, + it("passes the 36 hour grace window into the drain helper", async () => { + // The literal `36` lives in the helper module so the worker, the admin + // route, and the CLI all read one source of truth. + expect(DRAIN_CUMULATIVE_CUTOFF_HOURS).toBe(36); + const { boss, sends } = recordingBoss(); + const handler = await jobOf(DRAIN_CUMULATIVE_QUEUE).load({ boss }); + await handler([ + { + id: "j1", + name: DRAIN_CUMULATIVE_QUEUE, + data: { triggeredAt: "2026-01-01T03:45:00.000Z" }, + expireInSeconds: 900, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + expect(drainPerSampleCumulative).toHaveBeenCalledTimes(1); + expect(drainPerSampleCumulative).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + dryRun: false, + cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS, + }), ); - expect(source).toMatch(/\bdrainPerSampleCumulative\b/); - expect(source).toMatch(/\bDRAIN_CUMULATIVE_CUTOFF_HOURS\b/); + // A tick that finished inside its budget hands nothing on. + expect(sends).toEqual([]); }); - it("registers a createAndWork handler against the drain queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,1100}drainPerSampleCumulative/, - ); + it("schedules the drain cron with the cron as the retry", () => { + const job = jobOf(DRAIN_CUMULATIVE_QUEUE); + expect(job.cron).toBe(DRAIN_CUMULATIVE_CRON); + expect(job.scheduleOptions).toBe(cronIsTheRetry); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("schedules the drain cron via boss.schedule (allQueues + schedules)", () => { - expect(source).toMatch( - /\[DRAIN_CUMULATIVE_QUEUE,\s*DRAIN_CUMULATIVE_CRON,\s*cronIsTheRetry\]/, - ); - }); - - it("registers the drain queue in the allQueues createQueue loop", () => { - // pg-boss v12 requires explicit createQueue before scheduling. The - // allQueues array drives the boot-time `for (const q of allQueues)` - // loop; missing the entry silently no-ops the schedule below. - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bDRAIN_CUMULATIVE_QUEUE\b/); + it("declares the drain queue in the rollup group", () => { + // pg-boss v12 requires explicit createQueue before scheduling; a queue + // the registry does not declare is never created and its schedule + // silently no-ops. + expect(groupOf(DRAIN_CUMULATIVE_QUEUE).name).toBe("rollup"); }); }); diff --git a/src/lib/jobs/__tests__/fitbit-queues.test.ts b/src/lib/jobs/__tests__/fitbit-queues.test.ts index a9e9fbbc6..988c8fcf8 100644 --- a/src/lib/jobs/__tests__/fitbit-queues.test.ts +++ b/src/lib/jobs/__tests__/fitbit-queues.test.ts @@ -1,12 +1,12 @@ /** - * v1.12.0 — pg-boss queue registration for the Fitbit / Google Health sync layer. + * pg-boss queue registration for the Fitbit sync layer. * - * The reminder-worker imports heavy infrastructure (pg-boss, Prisma adapter, - * notification dispatchers, …) so we don't boot the whole worker here. Instead we - * read the source as text and assert the queue constants + cron schedules + - * handler wiring + allQueues + boot-enqueue are present — the fast guard that - * catches the v1.4.37 dead-queue class (a queue declared but never registered in - * `allQueues`, which silently never drains). + * Reads the worker registry instead of booting the worker: every Fitbit queue + * is declared in the integration-sync group with its cron and handler, the + * full-history queues forward to the shared admission lane, and the boot + * discovery offers the backfill and the sleep repair again. This is the fast + * guard for the dead-queue class (a queue declared but never registered, + * which silently never drains). * * Fitbit is poll-only at launch (no webhook — Pub/Sub deferred), so a single * hourly `fitbit-sync` queue drives the per-user `syncUserFitbit` driver across @@ -17,81 +17,101 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring + boot discovery moved out of the 2143-LOC -// reminder-worker boot file into domain registrars; the Fitbit queues live in -// the integration-sync registrar. The dead-queue guard follows the wiring there -// (the handler module is still concatenated for the import-routine assertions). -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-integration-sync.ts", -); -const source = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync(join(__dirname, "..", "reminder", "fitbit-sync.ts"), "utf8"); +import { + FITBIT_BACKFILL_QUEUE, + FITBIT_OAUTH_STATE_CLEANUP_CRON, + FITBIT_OAUTH_STATE_CLEANUP_QUEUE, + FITBIT_SLEEP_REPAIR_QUEUE, + FITBIT_SYNC_CRON, + FITBIT_SYNC_QUEUE, +} from "@/lib/fitbit/jobs/queues"; +import { INTEGRATION_BACKFILL_ADMISSION_QUEUE } from "@/lib/jobs/integration-backfill-admission"; +import { + handleFitbitOAuthStateCleanup, + handleFitbitSync, +} from "@/lib/jobs/reminder/fitbit-sync"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -const FITBIT_QUEUE_CONSTS = [ - "FITBIT_SYNC_QUEUE", - "FITBIT_BACKFILL_QUEUE", - "FITBIT_SLEEP_REPAIR_QUEUE", - "FITBIT_OAUTH_STATE_CLEANUP_QUEUE", +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "fitbit-sync.ts"), + "utf8", +); + +const FITBIT_QUEUES = [ + FITBIT_SYNC_QUEUE, + FITBIT_BACKFILL_QUEUE, + FITBIT_SLEEP_REPAIR_QUEUE, + FITBIT_OAUTH_STATE_CLEANUP_QUEUE, ] as const; -describe("reminder-worker — Fitbit sync queues", () => { +/** The admission sends the source queue's handler makes for one job. */ +async function forwardOneJob(queue: string) { + const { boss, sends } = recordingBoss(); + const handler = await jobOf(queue).load({ boss }); + await handler([ + { + id: "j1", + name: queue, + data: { userId: "u1" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + return sends; +} + +describe("Fitbit sync queues", () => { it("declares the Fitbit sync + oauth-state-cleanup queue constants", () => { - expect(source).toMatch(/FITBIT_SYNC_QUEUE\s*=\s*["']fitbit-sync["']/); - expect(source).toMatch( - /FITBIT_OAUTH_STATE_CLEANUP_QUEUE\s*=\s*["']fitbit-oauth-state-cleanup["']/, - ); + expect(FITBIT_SYNC_QUEUE).toBe("fitbit-sync"); + expect(FITBIT_OAUTH_STATE_CLEANUP_QUEUE).toBe("fitbit-oauth-state-cleanup"); }); - it("registers EVERY Fitbit queue in allQueues (v1.4.37 dead-queue guard)", () => { - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - const body = block![1]!; - for (const q of FITBIT_QUEUE_CONSTS) { - expect(body).toContain(q); + it("declares EVERY Fitbit queue in the integration-sync group", () => { + for (const q of FITBIT_QUEUES) { + expect(groupOf(q).name).toBe("integration_sync"); } }); it("schedules the hourly poll cron + the daily oauth-state sweep", () => { - expect(source).toMatch(/\[FITBIT_SYNC_QUEUE,\s*FITBIT_SYNC_CRON\]/); - expect(source).toMatch( - /\[\s*FITBIT_OAUTH_STATE_CLEANUP_QUEUE,\s*FITBIT_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, - ); + expect(jobOf(FITBIT_SYNC_QUEUE).cron).toBe(FITBIT_SYNC_CRON); + expect(jobOf(FITBIT_SYNC_QUEUE).scheduleOptions).toBeUndefined(); + const sweep = jobOf(FITBIT_OAUTH_STATE_CLEANUP_QUEUE); + expect(sweep.cron).toBe(FITBIT_OAUTH_STATE_CLEANUP_CRON); + expect(sweep.scheduleOptions).toBe(cronIsTheRetry); }); - it("registers a createAndWork handler for every Fitbit queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,160}FITBIT_SYNC_QUEUE[\s\S]{0,160}handleFitbitSync/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}FITBIT_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}FITBIT_SLEEP_REPAIR_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,160}FITBIT_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,160}handleFitbitOAuthStateCleanup/, + it("binds a handler for every Fitbit queue", async () => { + expect(await loadedHandler(FITBIT_SYNC_QUEUE)).toBe(handleFitbitSync); + expect(await loadedHandler(FITBIT_OAUTH_STATE_CLEANUP_QUEUE)).toBe( + handleFitbitOAuthStateCleanup, ); + for (const [queue, kind] of [ + [FITBIT_BACKFILL_QUEUE, "fitbit-backfill"], + [FITBIT_SLEEP_REPAIR_QUEUE, "fitbit-sleep-repair"], + ] as const) { + const sends = await forwardOneJob(queue); + expect(sends).toHaveLength(1); + expect(sends[0]!.queue).toBe(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(sends[0]!.data).toMatchObject({ kind, data: { userId: "u1" } }); + } }); - it("wires the self-converging Fitbit backfill boot discovery", () => { - expect(source).toMatch( - /enqueueBootTimeFitbitBackfill\(\s*bootStaggerSecondsFor\("fitbit-backfill"\)/, - ); + it("wires the self-converging backfill and the one-shot sleep-repair boot discovery", () => { + const passes = discoveryOf("integration_sync"); + expect(passes).toContain("fitbit-backfill"); + expect(passes).toContain("fitbit-sleep-repair"); }); - it("wires the one-shot Fitbit sleep-repair boot discovery", () => { - expect(source).toMatch( - /enqueueBootTimeFitbitSleepRepair\(\s*bootStaggerSecondsFor\("fitbit-sleep-repair"\)/, - ); - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/fitbit-sleep-repair["']/); - }); - - it("imports the Fitbit sync driver + backfill exports", () => { - expect(source).toMatch(/from\s*["']@\/lib\/fitbit\/sync["']/); - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/fitbit-backfill["']/); + it("imports the Fitbit sync driver into the handler module", () => { + expect(handlerSource).toMatch(/from\s*["']@\/lib\/fitbit\/sync["']/); }); }); diff --git a/src/lib/jobs/__tests__/geo-backfill.test.ts b/src/lib/jobs/__tests__/geo-backfill.test.ts index 8a6b4a835..036401d48 100644 --- a/src/lib/jobs/__tests__/geo-backfill.test.ts +++ b/src/lib/jobs/__tests__/geo-backfill.test.ts @@ -283,13 +283,13 @@ describe("runGeoBackfill", () => { }); /** - * v1.4.37 — the helper is now scheduled via pg-boss at the hourly - * :40 slot. The schedule constants live in the helper module so the - * reminder-worker boot pulls them by import; this test pins both - * shapes so a regression that drops the cadence or drifts the queue - * name lands here, not in production. + * The helper is scheduled via pg-boss at the hourly :40 slot. The + * schedule constants live in the helper module so the worker registry + * pulls them by import; this test pins both shapes so a regression that + * drops the cadence or drifts the queue name lands here, not in + * production. */ -describe("geo-backfill scheduling contract (v1.4.37)", () => { +describe("geo-backfill scheduling contract", () => { it("exports the pg-boss queue name", () => { expect(GEO_BACKFILL_QUEUE).toBe("geo-backfill"); }); @@ -298,20 +298,11 @@ describe("geo-backfill scheduling contract (v1.4.37)", () => { expect(GEO_BACKFILL_CRON).toBe("40 * * * *"); }); - it("reminder-worker imports both the runner and the schedule constants", async () => { - // Source-text probe avoids importing the worker module (which - // would drag pg-boss + a live Postgres connection). The contract - // is binary: the worker either registers the queue + cron or it - // does not; both presence checks pin the v1.4.37 scheduling - // shape end-to-end. + it("the worker registry schedules the queue and binds the ops handler", async () => { + // The handler module's import is read as text; the queue facts come from + // the registry, so neither pg-boss nor a live Postgres connection boots. const fs = await import("node:fs/promises"); const path = await import("node:path"); - // v1.18.1 — the geo-backfill wiring moved out of the 2143-LOC - // reminder-worker boot file into the maintenance registrar. - const workerSrc = await fs.readFile( - path.resolve(__dirname, "..", "reminder", "register-maintenance.ts"), - "utf-8", - ); const handlerSrc = await fs.readFile( path.resolve(__dirname, "..", "reminder", "ops-handlers.ts"), "utf-8", @@ -319,7 +310,11 @@ describe("geo-backfill scheduling contract (v1.4.37)", () => { expect(handlerSrc).toMatch( /import \{[^}]*runGeoBackfill[^}]*\} from "@\/lib\/jobs\/geo-backfill";/, ); - expect(workerSrc).toContain("GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON"); - expect(workerSrc).toContain("handleGeoBackfill"); + const { groupOf, jobOf, loadedHandler } = + await import("../registry/__tests__/registry-helpers"); + const { handleGeoBackfill } = await import("../reminder/ops-handlers"); + expect(groupOf(GEO_BACKFILL_QUEUE).name).toBe("maintenance"); + expect(jobOf(GEO_BACKFILL_QUEUE).cron).toBe(GEO_BACKFILL_CRON); + expect(await loadedHandler(GEO_BACKFILL_QUEUE)).toBe(handleGeoBackfill); }); }); diff --git a/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts b/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts index 7b65fe798..52362b370 100644 --- a/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts +++ b/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts @@ -1,10 +1,10 @@ /** * Runtime GeoLite2 fetch (issue #659) — queue-wiring guard + boot gating. * - * The source-text-grep half mirrors the other dead-queue guards: assert the - * queue is imported, in `allQueues`, scheduled, and bound to a `createAndWork` - * handler that runs the fetch — without booting pg-boss. An unregistered queue - * silently never drains (the recurring v1.4.37 dead-queue bug), which here + * The first half reads the worker registry: the queue is declared in the + * maintenance group, scheduled monthly, and bound to the handler that runs + * the fetch, and the maintenance boot discovery carries the fetch pass. An + * unregistered queue silently never drains (the dead-queue class), which here * would mean a keyed self-host never gets its databases fetched. * * The second half exercises the boot-discovery gate directly: it must send a @@ -15,54 +15,46 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-maintenance.ts", +import { + GEOLITE2_FETCH_CRON, + GEOLITE2_FETCH_QUEUE, +} from "@/lib/jobs/geolite2-fetch"; +import { handleGeolite2Fetch } from "@/lib/jobs/reminder/ops-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "ops-handlers.ts"), + "utf8", ); -const workerSource = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync(join(__dirname, "..", "reminder", "ops-handlers.ts"), "utf8"); -describe("reminder-worker — geolite2-fetch wiring", () => { - it("imports the queue symbols from the geolite2-fetch module", () => { - expect(workerSource).toMatch(/from\s*["']@\/lib\/jobs\/geolite2-fetch["']/); - expect(workerSource).toMatch(/\bGEOLITE2_FETCH_QUEUE\b/); - expect(workerSource).toMatch(/\bGEOLITE2_FETCH_CRON\b/); - expect(workerSource).toMatch(/\benqueueGeolite2FetchBootDiscovery\b/); +describe("geolite2-fetch registration", () => { + it("declares the queue in the maintenance group", () => { + expect(groupOf(GEOLITE2_FETCH_QUEUE).name).toBe("maintenance"); }); - it("registers the queue in the allQueues loop", () => { - const allQueuesMatch = workerSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bGEOLITE2_FETCH_QUEUE\b/); + it("schedules the monthly cron with the default retries", () => { + const job = jobOf(GEOLITE2_FETCH_QUEUE); + expect(job.cron).toBe(GEOLITE2_FETCH_CRON); + expect(job.scheduleOptions).toBeUndefined(); }); - it("schedules the monthly cron", () => { - expect(workerSource).toMatch( - /\[GEOLITE2_FETCH_QUEUE,\s*GEOLITE2_FETCH_CRON\]/, - ); - }); - - it("registers a createAndWork handler for the queue", () => { - expect(workerSource).toMatch( - /createAndWork[\s\S]{0,200}GEOLITE2_FETCH_QUEUE[\s\S]{0,200}handleGeolite2Fetch/, - ); + it("binds the fetch handler", async () => { + expect(await loadedHandler(GEOLITE2_FETCH_QUEUE)).toBe(handleGeolite2Fetch); }); it("runs the fetch inside the handler", () => { - expect(workerSource).toMatch( + expect(handlerSource).toMatch( /handleGeolite2Fetch[\s\S]{0,500}fetchGeoLite2Databases/, ); }); - it("fires the boot discovery from enqueueMaintenanceBootDiscovery", () => { - expect(workerSource).toMatch( - /enqueueMaintenanceBootDiscovery[\s\S]*enqueueGeolite2FetchBootDiscovery/, - ); + it("fires the boot discovery from the maintenance group", () => { + expect(discoveryOf("maintenance")).toContain("geolite2-fetch"); }); }); diff --git a/src/lib/jobs/__tests__/google-health-queues.test.ts b/src/lib/jobs/__tests__/google-health-queues.test.ts index 783593da4..b0c76de06 100644 --- a/src/lib/jobs/__tests__/google-health-queues.test.ts +++ b/src/lib/jobs/__tests__/google-health-queues.test.ts @@ -1,12 +1,11 @@ /** - * v1.26.0 — pg-boss queue registration for the Google Health sync layer. + * pg-boss queue registration for the Google Health sync layer. * - * The reminder-worker imports heavy infrastructure (pg-boss, Prisma adapter, - * notification dispatchers, …) so we don't boot the whole worker here. Instead - * we read the source as text and assert the queue constants + cron schedules + - * handler wiring + allQueues + boot-enqueue are present — the fast guard that - * catches the v1.4.37 dead-queue class (a queue declared but never registered - * in `allQueues`, which silently never drains). + * Reads the worker registry instead of booting the worker: every Google + * Health queue is declared in the integration-sync group with its cron and + * handler, the backfill forwards to the shared admission lane, and the boot + * discovery offers it again. This is the fast guard for the dead-queue class + * (a queue declared but never registered, which silently never drains). * * Google Health is poll-only at launch (no webhook — Pub/Sub deferred), so a * single hourly `google-health-sync` queue drives the per-user @@ -18,75 +17,90 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-integration-sync.ts", -); -const source = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync( - join(__dirname, "..", "reminder", "google-health-sync.ts"), - "utf8", - ); +import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "@/lib/jobs/google-health-backfill-queue"; +import { INTEGRATION_BACKFILL_ADMISSION_QUEUE } from "@/lib/jobs/integration-backfill-admission"; +import { + GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON, + GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, + GOOGLE_HEALTH_SYNC_CRON, + GOOGLE_HEALTH_SYNC_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; +import { + handleGoogleHealthOAuthStateCleanup, + handleGoogleHealthSync, +} from "@/lib/jobs/reminder/google-health-sync"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -const GOOGLE_HEALTH_QUEUE_CONSTS = [ - "GOOGLE_HEALTH_SYNC_QUEUE", - "GOOGLE_HEALTH_BACKFILL_QUEUE", - "GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE", +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "google-health-sync.ts"), + "utf8", +); + +const GOOGLE_HEALTH_QUEUES = [ + GOOGLE_HEALTH_SYNC_QUEUE, + GOOGLE_HEALTH_BACKFILL_QUEUE, + GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, ] as const; -describe("reminder-worker — Google Health sync queues", () => { +describe("Google Health sync queues", () => { it("declares the Google Health sync + oauth-state-cleanup queue constants", () => { - expect(source).toMatch( - /GOOGLE_HEALTH_SYNC_QUEUE\s*=\s*["']google-health-sync["']/, - ); - expect(source).toMatch( - /GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE\s*=\s*["']google-health-oauth-state-cleanup["']/, + expect(GOOGLE_HEALTH_SYNC_QUEUE).toBe("google-health-sync"); + expect(GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE).toBe( + "google-health-oauth-state-cleanup", ); }); - it("registers EVERY Google Health queue in allQueues (v1.4.37 dead-queue guard)", () => { - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - const body = block![1]!; - for (const q of GOOGLE_HEALTH_QUEUE_CONSTS) { - expect(body).toContain(q); + it("declares EVERY Google Health queue in the integration-sync group", () => { + for (const q of GOOGLE_HEALTH_QUEUES) { + expect(groupOf(q).name).toBe("integration_sync"); } }); it("schedules the hourly poll cron + the daily oauth-state sweep", () => { - expect(source).toMatch( - /\[GOOGLE_HEALTH_SYNC_QUEUE,\s*GOOGLE_HEALTH_SYNC_CRON\]/, - ); - expect(source).toMatch( - /\[\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE,\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, - ); + expect(jobOf(GOOGLE_HEALTH_SYNC_QUEUE).cron).toBe(GOOGLE_HEALTH_SYNC_CRON); + expect(jobOf(GOOGLE_HEALTH_SYNC_QUEUE).scheduleOptions).toBeUndefined(); + const sweep = jobOf(GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE); + expect(sweep.cron).toBe(GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON); + expect(sweep.scheduleOptions).toBe(cronIsTheRetry); }); - it("registers a createAndWork handler for every Google Health queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}GOOGLE_HEALTH_SYNC_QUEUE[\s\S]{0,200}handleGoogleHealthSync/, + it("binds a handler for every Google Health queue", async () => { + expect(await loadedHandler(GOOGLE_HEALTH_SYNC_QUEUE)).toBe( + handleGoogleHealthSync, ); - expect(source).toMatch( - /createAndWork[\s\S]{0,320}GOOGLE_HEALTH_BACKFILL_QUEUE[\s\S]{0,320}enqueueIntegrationBackfillAdmission/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,200}handleGoogleHealthOAuthStateCleanup/, + expect(await loadedHandler(GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE)).toBe( + handleGoogleHealthOAuthStateCleanup, ); + const { boss, sends } = recordingBoss(); + const backfill = await jobOf(GOOGLE_HEALTH_BACKFILL_QUEUE).load({ boss }); + await backfill([ + { + id: "j1", + name: GOOGLE_HEALTH_BACKFILL_QUEUE, + data: { userId: "u1" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + expect(sends).toHaveLength(1); + expect(sends[0]!.queue).toBe(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(sends[0]!.data).toMatchObject({ kind: "google-health-backfill" }); }); it("wires the self-converging Google Health backfill boot discovery", () => { - expect(source).toMatch( - /enqueueBootTimeGoogleHealthBackfill\(\s*bootStaggerSecondsFor\("google-health-backfill"\)/, - ); + expect(discoveryOf("integration_sync")).toContain("google-health-backfill"); }); - it("imports the Google Health sync driver + backfill exports", () => { - expect(source).toMatch(/from\s*["']@\/lib\/google-health\/sync["']/); - expect(source).toMatch( - /from\s*["']@\/lib\/jobs\/google-health-backfill["']/, - ); + it("imports the Google Health sync driver into the handler module", () => { + expect(handlerSource).toMatch(/from\s*["']@\/lib\/google-health\/sync["']/); }); }); diff --git a/src/lib/jobs/__tests__/insight-pregenerate.test.ts b/src/lib/jobs/__tests__/insight-pregenerate.test.ts index fd2cda222..f413497c1 100644 --- a/src/lib/jobs/__tests__/insight-pregenerate.test.ts +++ b/src/lib/jobs/__tests__/insight-pregenerate.test.ts @@ -1,5 +1,5 @@ /** - * v1.7.0 — unit tests for the nightly insight pre-generation cron. + * Unit tests for the nightly insight pre-generation cron. * * Covers: * - the discovery query selects stale-cache users who kept AI analysis @@ -14,12 +14,10 @@ * - the warm pass is single-locale + refill-only (v1.16.8); * - the forced single-user warm is idempotent (freshness re-check), * backs off after a failure, and is capped per day (v1.16.8); - * - the queue is registered in `allQueues` AND scheduled in - * reminder-worker.ts (the v1.4.37 W10 unregistered-queue catch). + * - the worker registry declares the queue, schedules its cron and binds + * its handler (the unregistered-queue catch). */ import { describe, it, expect, vi, beforeEach } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; const checkRateLimit = vi.fn(); const getAssistantFlags = vi.fn(); @@ -128,8 +126,18 @@ import { PREGENERATE_STALE_MS, FORCE_WARM_DAILY_LIMIT, INSIGHT_PREGENERATE_QUEUE, - INSIGHT_PREGENERATE_CRON, } from "../insight-pregenerate"; +import { + INSIGHT_PREGENERATE_CRON, + INSIGHT_PREGENERATE_EXPIRE_SECONDS, +} from "../insight-pregenerate-shared"; +import { handleInsightPregenerateJob } from "../reminder/insights-handlers"; +import { insightRetryOptions } from "../reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; function makePrisma( @@ -1402,29 +1410,22 @@ describe("forceWarmUser — on-demand single-user warm (v1.8.7.1)", () => { }); describe("queue registration", () => { - // v1.18.1 — the insight-pregenerate wiring moved out of the 2143-LOC - // reminder-worker boot file into the status registrar. The dead-queue guard - // follows the wiring there. - const workerSrc = fs.readFileSync( - path.resolve(__dirname, "../reminder/register-status.ts"), - "utf8", - ); - - it("registers the queue name in the allQueues createQueue loop", () => { - const match = workerSrc.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(match).not.toBeNull(); - expect(match![1]).toMatch(/\bINSIGHT_PREGENERATE_QUEUE\b/); + it("declares the queue in the status group", () => { + expect(groupOf(INSIGHT_PREGENERATE_QUEUE).name).toBe("status"); }); - it("schedules the cron in the schedules table (with retry policy)", () => { - expect(workerSrc).toMatch( - /\[\s*INSIGHT_PREGENERATE_QUEUE\s*,\s*INSIGHT_PREGENERATE_CRON\s*,\s*\{\s*\.\.\.insightRetryOptions\s*,\s*expireInSeconds:\s*INSIGHT_PREGENERATE_EXPIRE_SECONDS\s*,?\s*\}\s*,?\s*\]/, - ); + it("schedules the cron with the retry policy", () => { + const job = jobOf(INSIGHT_PREGENERATE_QUEUE); + expect(job.cron).toBe(INSIGHT_PREGENERATE_CRON); + expect(job.scheduleOptions).toEqual({ + ...insightRetryOptions, + expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS, + }); }); - it("registers a createAndWork handler for the queue", () => { - expect(workerSrc).toMatch( - /createAndWork[\s\S]{0,80}INSIGHT_PREGENERATE_QUEUE/, + it("binds the pre-generation handler to the queue", async () => { + expect(await loadedHandler(INSIGHT_PREGENERATE_QUEUE)).toBe( + handleInsightPregenerateJob, ); }); diff --git a/src/lib/jobs/__tests__/insight-status-generate.test.ts b/src/lib/jobs/__tests__/insight-status-generate.test.ts index da40217a7..cfd66efa4 100644 --- a/src/lib/jobs/__tests__/insight-status-generate.test.ts +++ b/src/lib/jobs/__tests__/insight-status-generate.test.ts @@ -1,18 +1,16 @@ /** - * v1.8.3 — unit tests for the on-demand per-metric status generation queue. + * Unit tests for the on-demand per-metric status generation queue. * * Covers: * - the dispatch runs the matching generator with force:true; * - an unknown metric is annotated and skipped, not thrown; * - the enqueue helper no-ops without a global boss instance and de-dupes * with a singletonKey when one is present; - * - the queue is registered in `allQueues` AND wired to a `createAndWork` - * handler in reminder-worker.ts (the v1.4.37 W10 unregistered-queue - * catch — a queue built but never registered silently never drains). + * - the worker registry declares the queue and binds its handler (the + * unregistered-queue catch — a queue built but never registered silently + * never drains). */ import { describe, it, expect, vi, beforeEach } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; const getGlobalBoss = vi.fn(); vi.mock("@/lib/jobs/boss-instance", () => ({ @@ -47,6 +45,12 @@ vi.mock("@/lib/ai/capabilities/gate", () => ({ aiCapabilityForJob: (...a: unknown[]) => aiCapabilityForJob(...a), aiCapabilityForRecord: vi.fn(), })); +// The worker's insights handlers (loaded to check the queue's binding) also +// import the nightly status batch; stub it so its card table never reads the +// stubbed generators above. +vi.mock("@/lib/insights/status-batch", () => ({ + generateStatusBatchForUser: vi.fn(), +})); const annotate = vi.fn(); vi.mock("@/lib/logging/context", async (importOriginal) => ({ ...(await importOriginal()), @@ -59,6 +63,13 @@ import { INSIGHT_STATUS_GENERATE_QUEUE, INSIGHT_STATUS_METRICS, } from "../insight-status-generate"; +import { INSIGHT_STATUS_GENERATE_CONCURRENCY } from "../insight-status-generate-queue"; +import { handleInsightStatusGenerate } from "../reminder/insights-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; beforeEach(() => { vi.clearAllMocks(); @@ -259,23 +270,18 @@ describe("enqueueStatusGeneration", () => { }); describe("queue registration", () => { - // v1.18.1 — the insight-status-generate wiring moved out of the 2143-LOC - // reminder-worker boot file into the status registrar. The dead-queue guard - // follows the wiring there. - const workerSrc = fs.readFileSync( - path.resolve(__dirname, "../reminder/register-status.ts"), - "utf8", - ); - - it("registers the queue name in the allQueues createQueue loop", () => { - const match = workerSrc.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(match).not.toBeNull(); - expect(match![1]).toMatch(/\bINSIGHT_STATUS_GENERATE_QUEUE\b/); + it("declares the queue in the status group, low-concurrency and cron-free", () => { + expect(groupOf(INSIGHT_STATUS_GENERATE_QUEUE).name).toBe("status"); + const job = jobOf(INSIGHT_STATUS_GENERATE_QUEUE); + expect(job.work).toEqual({ + localConcurrency: INSIGHT_STATUS_GENERATE_CONCURRENCY, + }); + expect(job.cron).toBeUndefined(); }); - it("registers a createAndWork handler for the queue", () => { - expect(workerSrc).toMatch( - /createAndWork[\s\S]{0,120}INSIGHT_STATUS_GENERATE_QUEUE/, + it("binds the status handler to the queue", async () => { + expect(await loadedHandler(INSIGHT_STATUS_GENERATE_QUEUE)).toBe( + handleInsightStatusGenerate, ); }); diff --git a/src/lib/jobs/__tests__/intake-auto-skip.test.ts b/src/lib/jobs/__tests__/intake-auto-skip.test.ts index 4073e96d1..05ad666a0 100644 --- a/src/lib/jobs/__tests__/intake-auto-skip.test.ts +++ b/src/lib/jobs/__tests__/intake-auto-skip.test.ts @@ -27,12 +27,14 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { caches, __resetAllCachesForTests } from "@/lib/cache/server-cache"; import { - INTAKE_AUTO_SKIP_CRON, INTAKE_AUTO_SKIP_GRACE_HOURS, - INTAKE_AUTO_SKIP_QUEUE, medicationAutoMissDelayMs, runIntakeAutoSkipPass, } from "../intake-auto-skip"; +import { + INTAKE_AUTO_SKIP_CRON, + INTAKE_AUTO_SKIP_QUEUE, +} from "../intake-auto-skip-queue"; interface FakeIntakeRow { id: string; diff --git a/src/lib/jobs/__tests__/intake-auto-take.test.ts b/src/lib/jobs/__tests__/intake-auto-take.test.ts index db93d7c97..bee2b010c 100644 --- a/src/lib/jobs/__tests__/intake-auto-take.test.ts +++ b/src/lib/jobs/__tests__/intake-auto-take.test.ts @@ -39,15 +39,17 @@ import { consumeForIntake } from "@/lib/medications/inventory/consumption"; import { refreshDoseSlot } from "@/lib/medications/dose-day-refresh"; import { queueMedicationIntakeSync } from "@/lib/notifications/medication-intake-sync"; import { - INTAKE_AUTO_TAKE_CRON, INTAKE_AUTO_TAKE_LOOKBACK_MS, - INTAKE_AUTO_TAKE_QUEUE, autoTakeFloor, decideSlot, dueAutoTakeSlots, isInPauseEra, runIntakeAutoTakePass, } from "../intake-auto-take"; +import { + INTAKE_AUTO_TAKE_CRON, + INTAKE_AUTO_TAKE_QUEUE, +} from "../intake-auto-take-queue"; import { autoTakeOwnedFrom } from "../intake-auto-skip"; import { autoTakeRollingAnchor } from "@/lib/medications/auto-take"; import { computeDisplayDue } from "@/lib/medications/scheduling/next-due"; diff --git a/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts b/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts index a99672474..3f5f2dc6b 100644 --- a/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts +++ b/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts @@ -1,95 +1,102 @@ /** - * v1.8.2 — duplicate dose-slot dedup queue registration guard. + * Duplicate dose-slot dedup queue registration guard. * - * Same source-text-grep approach as the drain + step-consolidation + - * mean-consolidation regression guards: assert the queue is registered - * in `allQueues`, a `createAndWork` handler is wired against it, and the - * boot-discovery enqueue helper is called — without booting pg-boss + - * Prisma. An unregistered queue silently never drains (the past incident - * this guard exists to prevent). + * Reads the worker registry: the queue is declared in the maintenance group + * with its daily cron, bound to the dedup handler, and offered again by the + * maintenance boot discovery. The bound handler runs against a mocked dedup + * module, so pg-boss and Prisma never boot. An unregistered queue silently + * never drains (the past incident this guard exists to prevent). */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; -import { describe, expect, it } from "vitest"; - -// v1.18.1 — the intake-slot-dedup wiring + boot discovery moved out of the -// 2143-LOC reminder-worker boot file into the maintenance registrar. The -// dead-queue guard follows the wiring there. -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-maintenance.ts", +const { dedupeUserIntakeSlots, enqueueBootTimeIntakeSlotDedup } = vi.hoisted( + () => ({ + dedupeUserIntakeSlots: vi.fn(async () => ({ + slotsCollapsed: 0, + rowsSoftDeleted: 0, + rowsNormalised: 0, + daysRecomputed: 0, + })), + enqueueBootTimeIntakeSlotDedup: vi.fn(async () => ({ + enqueued: 0, + skipped: 0, + error: null, + })), + }), ); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +vi.mock("@/lib/medications/intake-slot-dedup", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/medications/intake-slot-dedup") + >()), + dedupeUserIntakeSlots, + enqueueBootTimeIntakeSlotDedup, +})); -describe("reminder-worker — intake-slot-dedup wiring", () => { - it("imports the queue symbols from the intake-slot-dedup module", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/medications\/intake-slot-dedup["']/, +import type { Job } from "pg-boss"; + +import { handleIntakeSlotDedup } from "@/lib/jobs/reminder/maintenance-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { + INTAKE_SLOT_DEDUP_CONCURRENCY, + INTAKE_SLOT_DEDUP_QUEUE, +} from "@/lib/medications/intake-slot-dedup"; + +function job(data: { userId?: string }): Job<{ userId?: string }> { + return { + id: "j1", + name: INTAKE_SLOT_DEDUP_QUEUE, + data, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("intake-slot-dedup registration", () => { + it("declares the intake-slot-dedup queue in the maintenance group", () => { + expect(groupOf(INTAKE_SLOT_DEDUP_QUEUE).name).toBe("maintenance"); + const definition = jobOf(INTAKE_SLOT_DEDUP_QUEUE); + expect(definition.policy?.policy).toBe("exclusive"); + expect(definition.work).toEqual({ + localConcurrency: INTAKE_SLOT_DEDUP_CONCURRENCY, + }); + }); + + it("binds the dedup handler, which runs the per-user dedup", async () => { + expect(await loadedHandler(INTAKE_SLOT_DEDUP_QUEUE)).toBe( + handleIntakeSlotDedup, ); - expect(source).toMatch(/\bINTAKE_SLOT_DEDUP_QUEUE\b/); - expect(source).toMatch(/\bdedupeUserIntakeSlots\b/); - expect(source).toMatch(/\benqueueBootTimeIntakeSlotDedup\b/); + await handleIntakeSlotDedup([job({ userId: "u1" })]); + expect(dedupeUserIntakeSlots).toHaveBeenCalledWith("u1"); + expect(enqueueBootTimeIntakeSlotDedup).not.toHaveBeenCalled(); }); - it("registers the intake-slot-dedup queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bINTAKE_SLOT_DEDUP_QUEUE\b/); + it("fires the boot-discovery pass", () => { + expect(discoveryOf("maintenance")).toContain("intake-slot-dedup"); }); - it("registers a createAndWork handler against the intake-slot-dedup queue", () => { - // v1.15.19 — span widened from {0,400}: the handler gained the daily - // cron-tick dispatch branch (no `userId` → discovery fan-out) between - // the queue name and the per-user dedup call. - expect(source).toMatch( - /createAndWork[\s\S]{0,200}INTAKE_SLOT_DEDUP_QUEUE[\s\S]{0,1400}dedupeUserIntakeSlots/, - ); - }); - - it("fires the boot-discovery enqueue helper", () => { - expect(source).toMatch(/await enqueueBootTimeIntakeSlotDedup\(\)/); - }); - - // v1.15.19 — the boot-only pass left duplicates created between deploys - // standing until the next worker restart. The queue now also carries a - // daily cron tick whose empty payload (no `userId`) dispatches the same - // discovery fan-out. + // The boot-only pass left duplicates created between deploys standing + // until the next worker restart. The queue also carries a daily cron tick + // whose empty payload (no `userId`) dispatches the same discovery fan-out. it("declares a daily cron inside the 03:xx maintenance window", () => { - const cronMatch = source.match( - /const INTAKE_SLOT_DEDUP_CRON\s*=\s*["']([^"']+)["']/, - ); - expect(cronMatch).not.toBeNull(); // Five-field cron, daily (day-of-month/month/day-of-week all `*`), // inside the 03:xx maintenance window. - expect(cronMatch![1]).toMatch(/^\d{1,2} 3 \* \* \*$/); + expect(jobOf(INTAKE_SLOT_DEDUP_QUEUE).cron).toMatch(/^\d{1,2} 3 \* \* \*$/); }); - it("registers the cron in the schedules table", () => { - // v1.15.20 — the schedules tuple grew an optional send-options slot - // (retry policy for the insight queues); match on the assignment, not - // the exact tuple type. - // v1.18.1 — `schedules` is now a module-level const in the registrar, so it - // closes at column 0 (`\n];`) rather than the monolith's function-body - // two-space indent. - const schedulesMatch = source.match( - /const schedules:[^=]*=\s*\[([\s\S]*?)\n\];/, - ); - expect(schedulesMatch).not.toBeNull(); - expect(schedulesMatch![1]).toMatch( - /\[INTAKE_SLOT_DEDUP_QUEUE,\s*INTAKE_SLOT_DEDUP_CRON\]/, - ); - }); - - it("dispatches a payload without userId to the discovery fan-out", () => { - // The handler must branch on the missing `userId` BEFORE the per-user - // dedup call, and the branch must run the discovery helper. - expect(source).toMatch( - /INTAKE_SLOT_DEDUP_QUEUE[\s\S]{0,800}if \(!userId\) \{[\s\S]{0,400}enqueueBootTimeIntakeSlotDedup\(\)/, - ); + it("dispatches a payload without userId to the discovery fan-out", async () => { + await handleIntakeSlotDedup([job({})]); + expect(enqueueBootTimeIntakeSlotDedup).toHaveBeenCalledTimes(1); + expect(dedupeUserIntakeSlots).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/jobs/__tests__/integration-backfill-admission.test.ts b/src/lib/jobs/__tests__/integration-backfill-admission.test.ts index a26bd6561..69ac042a1 100644 --- a/src/lib/jobs/__tests__/integration-backfill-admission.test.ts +++ b/src/lib/jobs/__tests__/integration-backfill-admission.test.ts @@ -1,7 +1,74 @@ +/** + * The shared full-history backfill admission: every provider backfill and + * repair forwards into one database-coordinated drain. The boot discovery, + * the drain's binding and the source queues' forwarding are read off the + * worker registry; the boot discovery runs against stubbed enqueue helpers + * that record the stagger each pass hands them. + */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { SendOptions } from "pg-boss"; + +const staggers = vi.hoisted(() => new Map()); +const recordStagger = vi.hoisted(() => (kind: string, stagger: number) => { + staggers.set(kind, stagger); + return { enqueued: 0, skipped: 0, error: null }; +}); +vi.mock("@/lib/jobs/whoop-backfill", async (importOriginal) => ({ + ...(await importOriginal()), + enqueueBootTimeWhoopBackfill: async (stagger: number) => + recordStagger("whoop-backfill", stagger), +})); +vi.mock("@/lib/jobs/fitbit-backfill", async (importOriginal) => ({ + ...(await importOriginal()), + enqueueBootTimeFitbitBackfill: async (stagger: number) => + recordStagger("fitbit-backfill", stagger), +})); +vi.mock("@/lib/jobs/google-health-backfill", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/google-health-backfill") + >()), + enqueueBootTimeGoogleHealthBackfill: async (stagger: number) => + recordStagger("google-health-backfill", stagger), +})); +vi.mock("@/lib/jobs/google-health-sleep-repair", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/google-health-sleep-repair") + >()), + enqueueBootTimeGoogleHealthSleepRepair: async (stagger: number) => + recordStagger("google-health-sleep-repair", stagger), +})); +vi.mock("@/lib/jobs/sleep-timeline-backfill", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/sleep-timeline-backfill") + >()), + enqueueBootTimeSleepTimelineBackfill: async (stagger: number) => + recordStagger("sleep-timeline-backfill", stagger), +})); +vi.mock("@/lib/jobs/lab-biomarker-backfill", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/lab-biomarker-backfill") + >()), + enqueueBootTimeLabBiomarkerBackfill: async (stagger: number) => + recordStagger("lab-biomarker-backfill", stagger), +})); +vi.mock("@/lib/jobs/strava-backfill", async (importOriginal) => ({ + ...(await importOriginal()), + enqueueBootTimeStravaBackfill: async (stagger: number) => + recordStagger("strava-backfill", stagger), +})); +vi.mock("@/lib/jobs/fitbit-sleep-repair", async (importOriginal) => ({ + ...(await importOriginal()), + enqueueBootTimeFitbitSleepRepair: async (stagger: number) => + recordStagger("fitbit-sleep-repair", stagger), +})); +vi.mock("@/lib/jobs/oura-backfill", async (importOriginal) => ({ + ...(await importOriginal()), + enqueueBootTimeOuraBackfill: async (stagger: number) => + recordStagger("oura-backfill", stagger), +})); + import { INTEGRATION_BACKFILL_ADMISSION_GROUP, INTEGRATION_BACKFILL_ADMISSION_QUEUE, @@ -12,12 +79,29 @@ import { bootStaggerSecondsFor, enqueueIntegrationBackfillAdmission, integrationBackfillSourceOptions, + type IntegrationBackfillKind, } from "../integration-backfill-admission"; - -const registrarSource = readFileSync( - join(__dirname, "../reminder/register-integration-sync.ts"), - "utf8", -); +import { + FITBIT_BACKFILL_QUEUE, + FITBIT_SLEEP_REPAIR_QUEUE, + FITBIT_SYNC_QUEUE, +} from "@/lib/fitbit/jobs/queues"; +import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "../google-health-backfill-queue"; +import { GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE } from "../google-health-sleep-repair-queue"; +import { + GOOGLE_HEALTH_SYNC_QUEUE, + STRAVA_SYNC_QUEUE, +} from "../integration-poll-queues"; +import { LAB_BIOMARKER_BACKFILL_QUEUE } from "../lab-biomarker-backfill"; +import { OURA_BACKFILL_QUEUE } from "../oura-backfill-queue"; +import { + groupNamed, + jobOf, + recordingBoss, +} from "../registry/__tests__/registry-helpers"; +import { SLEEP_TIMELINE_BACKFILL_QUEUE } from "../sleep-timeline-backfill-queue"; +import { STRAVA_BACKFILL_QUEUE } from "../strava-backfill-queue"; +import { WHOOP_BACKFILL_QUEUE } from "../whoop-backfill-queue"; const providerSources = [ "whoop-backfill.ts", @@ -32,7 +116,7 @@ const providerSources = [ ].map((file) => readFileSync(join(__dirname, "..", file), "utf8")); describe("shared full-history backfill admission", () => { - it("assigns every integration backfill a deterministic durable boot stagger", () => { + it("assigns every integration backfill a deterministic durable boot stagger", async () => { expect(INTEGRATION_BACKFILL_BOOT_ORDER).toEqual([ "whoop-backfill", "fitbit-backfill", @@ -54,23 +138,14 @@ describe("shared full-history backfill admission", () => { expect(source).toMatch(/startAfterSeconds:\s*number\s*=\s*0/); } - const discoveryCalls = { - "whoop-backfill": "enqueueBootTimeWhoopBackfill", - "fitbit-backfill": "enqueueBootTimeFitbitBackfill", - "google-health-backfill": "enqueueBootTimeGoogleHealthBackfill", - "google-health-sleep-repair": "enqueueBootTimeGoogleHealthSleepRepair", - "sleep-timeline-backfill": "enqueueBootTimeSleepTimelineBackfill", - "lab-biomarker-backfill": "enqueueBootTimeLabBiomarkerBackfill", - "strava-backfill": "enqueueBootTimeStravaBackfill", - "fitbit-sleep-repair": "enqueueBootTimeFitbitSleepRepair", - "oura-backfill": "enqueueBootTimeOuraBackfill", - } as const; + // Every kind has a boot discovery pass, and each pass hands its helper + // the kind's own stagger. + const passes = groupNamed("integration_sync").discovery ?? []; for (const kind of INTEGRATION_BACKFILL_BOOT_ORDER) { - expect(registrarSource).toMatch( - new RegExp( - `${discoveryCalls[kind]}\\(\\s*bootStaggerSecondsFor\\("${kind}"\\)`, - ), - ); + const pass = passes.find((p) => p.name === kind); + expect(pass, kind).toBeDefined(); + await pass!.run(); + expect(staggers.get(kind), kind).toBe(bootStaggerSecondsFor(kind)); } }); @@ -139,40 +214,51 @@ describe("shared full-history backfill admission", () => { ).not.toHaveProperty("startAfter"); }); - it("registers one shared globally bounded drain and leaves incremental workers outside it", () => { - expect(registrarSource).toMatch( - /INTEGRATION_BACKFILL_ADMISSION_QUEUE[\s\S]{0,220}localConcurrency:\s*INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY[\s\S]{0,120}groupConcurrency:\s*INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY/, - ); - expect(registrarSource).toMatch( - /\[INTEGRATION_BACKFILL_ADMISSION_QUEUE\]:\s*\{[\s\S]{0,100}policy:\s*"exclusive"/, - ); + it("registers one shared globally bounded drain and leaves incremental workers outside it", async () => { + const drain = jobOf(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(drain.work).toEqual({ + localConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, + groupConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, + }); + expect(drain.policy?.policy).toBe("exclusive"); - for (const queue of [ - "WHOOP_BACKFILL_QUEUE", - "FITBIT_BACKFILL_QUEUE", - "GOOGLE_HEALTH_BACKFILL_QUEUE", - "GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE", - "SLEEP_TIMELINE_BACKFILL_QUEUE", - "LAB_BIOMARKER_BACKFILL_QUEUE", - "STRAVA_BACKFILL_QUEUE", - "FITBIT_SLEEP_REPAIR_QUEUE", - "OURA_BACKFILL_QUEUE", - ]) { - expect(registrarSource).toMatch( - new RegExp( - `${queue}[\\s\\S]{0,500}enqueueIntegrationBackfillAdmission`, - ), - ); + const sources: ReadonlyArray<[string, IntegrationBackfillKind]> = [ + [WHOOP_BACKFILL_QUEUE, "whoop-backfill"], + [FITBIT_BACKFILL_QUEUE, "fitbit-backfill"], + [GOOGLE_HEALTH_BACKFILL_QUEUE, "google-health-backfill"], + [GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, "google-health-sleep-repair"], + [SLEEP_TIMELINE_BACKFILL_QUEUE, "sleep-timeline-backfill"], + [LAB_BIOMARKER_BACKFILL_QUEUE, "lab-biomarker-backfill"], + [STRAVA_BACKFILL_QUEUE, "strava-backfill"], + [FITBIT_SLEEP_REPAIR_QUEUE, "fitbit-sleep-repair"], + [OURA_BACKFILL_QUEUE, "oura-backfill"], + ]; + for (const [queue, kind] of sources) { + const { boss, sends } = recordingBoss(); + const handler = await jobOf(queue).load({ boss }); + await handler([ + { + id: "j1", + name: queue, + data: { userId: "u1", provider: "WHOOP" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + expect(sends, queue).toHaveLength(1); + expect(sends[0]!.queue).toBe(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(sends[0]!.data).toMatchObject({ kind }); } for (const queue of [ - "FITBIT_SYNC_QUEUE", - "GOOGLE_HEALTH_SYNC_QUEUE", - "STRAVA_SYNC_QUEUE", + FITBIT_SYNC_QUEUE, + GOOGLE_HEALTH_SYNC_QUEUE, + STRAVA_SYNC_QUEUE, ]) { - expect(registrarSource).toMatch( - new RegExp(`${queue}[\\s\\S]{0,100}localConcurrency:\\s*1`), - ); + expect(jobOf(queue).work, queue).toEqual({ localConcurrency: 1 }); + expect(jobOf(queue).lane, queue).toBeUndefined(); } }); }); diff --git a/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts b/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts index 7f42958a7..85babde37 100644 --- a/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts +++ b/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts @@ -1,57 +1,49 @@ /** - * MCP Phase 3 (M2) — mcp-token-cleanup queue registration guard. + * Mcp-token-cleanup queue registration guard. * - * Source-text-grep guard like the other queue-wiring checks: assert the queue - * is in `allQueues`, scheduled, and wired to a `createAndWork` handler. An - * unregistered queue silently never drains (the recurring v1.4.37 dead-queue - * bug), which would leave expired MCP connector access tokens growing unbounded. + * Reads the worker registry: the queue is declared in the maintenance group, + * scheduled with the cron as its retry, and bound to the cleanup handler. An + * unregistered queue silently never drains (the dead-queue class), which + * would leave expired MCP connector access tokens growing unbounded. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-maintenance.ts", +import { handleMcpTokenCleanup } from "@/lib/jobs/reminder/cleanup-handlers"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; + +const QUEUE = "mcp-token-cleanup"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "cleanup-handlers.ts"), + "utf8", ); -const workerSource = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync( - join(__dirname, "..", "reminder", "cleanup-handlers.ts"), - "utf8", - ); -describe("reminder-worker — mcp-token-cleanup wiring", () => { - it("imports the cleanup handler from the cleanup-handlers module", () => { - expect(workerSource).toMatch(/\bhandleMcpTokenCleanup\b/); - expect(workerSource).toMatch(/\bMcpTokenCleanupPayload\b/); +describe("mcp-token-cleanup registration", () => { + it("declares the queue in the maintenance group", () => { + expect(groupOf(QUEUE).name).toBe("maintenance"); }); - it("registers the mcp-token-cleanup queue in the allQueues loop", () => { - const allQueuesMatch = workerSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bMCP_TOKEN_CLEANUP_QUEUE\b/); + it("schedules the daily cron with the cron as the retry", () => { + const job = jobOf(QUEUE); + expect(job.cron).toBe("0 4 * * *"); + expect(job.scheduleOptions).toBe(cronIsTheRetry); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("schedules the mcp-token-cleanup cron", () => { - expect(workerSource).toMatch( - /\[MCP_TOKEN_CLEANUP_QUEUE,\s*MCP_TOKEN_CLEANUP_CRON,\s*cronIsTheRetry\]/, - ); - }); - - it("registers a createAndWork handler for the queue", () => { - expect(workerSource).toMatch( - /createAndWork[\s\S]{0,200}MCP_TOKEN_CLEANUP_QUEUE[\s\S]{0,200}handleMcpTokenCleanup/, - ); + it("binds the cleanup handler", async () => { + expect(await loadedHandler(QUEUE)).toBe(handleMcpTokenCleanup); }); it("prunes MCP tokens inside the handler", () => { - expect(workerSource).toMatch( + expect(handlerSource).toMatch( /handleMcpTokenCleanup[\s\S]{0,400}cleanupExpiredMcpTokens/, ); }); diff --git a/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts b/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts index 4dbfd6d11..a2076591a 100644 --- a/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts +++ b/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts @@ -1,48 +1,44 @@ /** - * v1.7.0 — daily-mean consolidation queue registration guard. + * Daily-mean consolidation queue registration guard. * - * Same source-text-grep approach as the drain + step-consolidation - * regression guards: assert the queue is registered in `allQueues`, a - * `createAndWork` handler is wired against it, and the boot-discovery - * enqueue helper is called — without booting pg-boss + Prisma. An - * unregistered queue silently never drains. + * Reads the worker registry: the queue is declared in the rollup group, + * bound to the consolidation handler, and offered again by the rollup boot + * discovery. An unregistered queue silently never drains. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -// v1.18.1 — the mean-consolidation wiring + boot discovery moved out of the -// 2143-LOC reminder-worker boot file into the rollup registrar. The dead-queue -// guard follows the wiring there. -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-rollup.ts"); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { + MEAN_CONSOLIDATION_CONCURRENCY, + MEAN_CONSOLIDATION_QUEUE, +} from "@/lib/jobs/mean-consolidation"; +import { handleMeanConsolidation } from "@/lib/jobs/reminder/rollup-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -describe("reminder-worker — mean-consolidation wiring", () => { - it("imports the queue symbols from the mean-consolidation module", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/mean-consolidation["']/); - expect(source).toMatch(/\bMEAN_CONSOLIDATION_QUEUE\b/); - expect(source).toMatch(/\brunMeanConsolidationForUser\b/); - expect(source).toMatch(/\benqueueBootTimeMeanConsolidation\b/); +describe("mean-consolidation registration", () => { + it("declares the mean-consolidation queue in the rollup group", () => { + expect(groupOf(MEAN_CONSOLIDATION_QUEUE).name).toBe("rollup"); }); - it("registers the mean-consolidation queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bMEAN_CONSOLIDATION_QUEUE\b/); + it("is a per-user exclusive pass at the module's concurrency", () => { + const job = jobOf(MEAN_CONSOLIDATION_QUEUE); + expect(job.policy?.policy).toBe("exclusive"); + expect(job.work).toEqual({ + localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY, + }); }); - it("registers a createAndWork handler against the mean-consolidation queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}MEAN_CONSOLIDATION_QUEUE[\s\S]{0,400}runMeanConsolidationForUser/, + it("binds the consolidation handler", async () => { + expect(await loadedHandler(MEAN_CONSOLIDATION_QUEUE)).toBe( + handleMeanConsolidation, ); }); - it("fires the boot-discovery enqueue helper", () => { - // The boot wiring passes a stagger offset so the heavy backfills don't all - // drain in parallel onto one tenant; allow an optional argument here. - expect(source).toMatch(/await enqueueBootTimeMeanConsolidation\([^)]*\)/); + it("runs the boot discovery pass", () => { + expect(discoveryOf("rollup")).toContain("mean-consolidation"); }); }); diff --git a/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts b/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts index 564b9cb95..98f87af95 100644 --- a/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts +++ b/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts @@ -1,90 +1,81 @@ /** - * v1.17.1 — measurement-reminder queue registration guard. + * Measurement-reminder queue registration guard. * - * Same source-text-grep approach as the cycle / mood / drain registration - * guards: assert the Vorsorge-reminder queue is declared, listed in - * `allQueues` (the v1.4.37 dead-queue lesson — an unregistered queue is - * never provisioned and the schedule silently no-ops), scheduled on its - * cron, and wired to a `createAndWork` handler that delegates to - * `runMeasurementReminderTick`. No pg-boss / Prisma boot. + * Reads the worker registry: the Vorsorge-reminder queue is declared in the + * reminders group (an unregistered queue is never provisioned and the + * schedule silently no-ops), scheduled on its cron, and bound to the handler + * that delegates to `runMeasurementReminderTick`; the eventful + * reminder-satisfy queue sits beside it. No pg-boss / Prisma boot. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the reminder-dispatch wiring (measurement reminder + -// reminder-satisfy) moved out of the 2143-LOC reminder-worker boot file into -// the reminders registrar. The dead-queue guard follows the wiring there (the -// handler module is still concatenated for the handler-symbol assertions). -const source = - readFileSync( - join(__dirname, "..", "reminder", "register-reminders.ts"), - "utf8", - ) + - readFileSync( - join(__dirname, "..", "reminder", "mood-cycle-checks.ts"), - "utf8", - ); +import { + REMINDER_SATISFY_CONCURRENCY, + REMINDER_SATISFY_QUEUE, +} from "@/lib/jobs/reminder-satisfy"; +import { + handleMeasurementReminderCheck, + handleReminderSatisfy, +} from "@/lib/jobs/reminder/mood-cycle-checks"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -describe("reminder-worker — measurement-reminder wiring", () => { +const MEASUREMENT_REMINDER_QUEUE = "measurement-reminder-check"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "mood-cycle-checks.ts"), + "utf8", +); + +describe("measurement-reminder registration", () => { it("imports the measurement-reminder tick runner", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/measurement-reminder["']/); - expect(source).toMatch(/\brunMeasurementReminderTick\b/); + expect(handlerSource).toMatch( + /from\s*["']@\/lib\/jobs\/measurement-reminder["']/, + ); + expect(handlerSource).toMatch(/\brunMeasurementReminderTick\b/); }); - it("declares the queue + cron constants", () => { - expect(source).toMatch( - /MEASUREMENT_REMINDER_QUEUE\s*=\s*["']measurement-reminder-check["']/, - ); - expect(source).toMatch( - /MEASUREMENT_REMINDER_CRON\s*=\s*["']\*\/15 \* \* \* \*["']/, - ); + it("declares the queue in the reminders group", () => { + expect(groupOf(MEASUREMENT_REMINDER_QUEUE).name).toBe("reminders"); }); - it("registers the queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bMEASUREMENT_REMINDER_QUEUE\b/); + it("schedules the measurement-reminder cron, single-flight", () => { + const job = jobOf(MEASUREMENT_REMINDER_QUEUE); + expect(job.cron).toBe("*/15 * * * *"); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("schedules the measurement-reminder cron", () => { - expect(source).toMatch( - /\[MEASUREMENT_REMINDER_QUEUE,\s*MEASUREMENT_REMINDER_CRON\]/, + it("binds the handler that runs the tick", async () => { + expect(await loadedHandler(MEASUREMENT_REMINDER_QUEUE)).toBe( + handleMeasurementReminderCheck, ); - }); - - it("registers a createAndWork handler that runs the tick", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}MEASUREMENT_REMINDER_QUEUE[\s\S]{0,200}handleMeasurementReminderCheck/, - ); - expect(source).toMatch( + expect(handlerSource).toMatch( /handleMeasurementReminderCheck[\s\S]{0,400}runMeasurementReminderTick/, ); }); }); -describe("reminder-worker — eventful reminder-satisfy wiring (v1.18.1)", () => { - it("imports the satisfy queue constants", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/reminder-satisfy["']/); - expect(source).toMatch(/\bREMINDER_SATISFY_QUEUE\b/); +describe("eventful reminder-satisfy registration", () => { + it("declares the satisfy queue in the reminders group, with no cron", () => { + expect(groupOf(REMINDER_SATISFY_QUEUE).name).toBe("reminders"); + const job = jobOf(REMINDER_SATISFY_QUEUE); + expect(job.cron).toBeUndefined(); + expect(job.work).toEqual({ + localConcurrency: REMINDER_SATISFY_CONCURRENCY, + }); }); - it("registers the satisfy queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, + it("binds the satisfy handler", async () => { + expect(await loadedHandler(REMINDER_SATISFY_QUEUE)).toBe( + handleReminderSatisfy, ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bREMINDER_SATISFY_QUEUE\b/); - }); - - it("binds a createAndWork handler for the satisfy queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}REMINDER_SATISFY_QUEUE[\s\S]{0,200}handleReminderSatisfy/, - ); - expect(source).toMatch( + expect(handlerSource).toMatch( /handleReminderSatisfy[\s\S]{0,800}runReminderSatisfyForUser/, ); }); diff --git a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts index f27b28739..9defa2f68 100644 --- a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts +++ b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts @@ -1,23 +1,26 @@ /** - * v1.7.0 — measurement-tombstone retention cleanup. + * Measurement-tombstone retention cleanup. * * Two guards: * 1. Behavioural: the helper hard-deletes only soft-deleted rows whose * `deletedAt` predates the retention horizon, leaving live rows and * recently-tombstoned rows untouched. - * 2. Source-grep wiring (same approach as the drain-cumulative guard): - * the queue is registered in `allQueues`, scheduled, and bound to a - * `createAndWork` handler — a missing `allQueues` entry silently no-ops - * the schedule under pg-boss v12. + * 2. Registry wiring: the queue is declared in the maintenance group, + * scheduled, and bound to its handler — a queue the registry does not + * declare is never created, and its schedule silently no-ops under + * pg-boss v12. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it, vi } from "vitest"; import { cleanupExpiredMeasurementTombstones } from "../measurement-tombstone-cleanup"; import { PURGE_BATCH_SIZE, PURGE_MAX_BATCHES } from "../purge-batch"; import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client"; +import { handleMeasurementTombstoneCleanup } from "../reminder/cleanup-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; const DAY_MS = 86_400_000; @@ -170,42 +173,24 @@ describe("cleanupExpiredMeasurementTombstones — accounts under restore", () => }); }); -describe("reminder-worker — measurement-tombstone-cleanup schedule", () => { - // v1.18.1 — the cleanup wiring moved out of the 2143-LOC reminder-worker - // boot file into the maintenance registrar. The dead-queue guard follows it. - const source = readFileSync( - join(__dirname, "..", "reminder", "register-maintenance.ts"), - "utf8", - ); +describe("measurement-tombstone-cleanup schedule", () => { + const QUEUE = "measurement-tombstone-cleanup"; it("declares the queue at the documented Berlin cadence", () => { - expect(source).toMatch( - /MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE\s*=\s*["']measurement-tombstone-cleanup["']/, - ); - expect(source).toMatch( - /MEASUREMENT_TOMBSTONE_CLEANUP_CRON\s*=\s*["']40 3 \* \* \*["']/, - ); + expect(jobOf(QUEUE).cron).toBe("40 3 * * *"); }); - it("registers the queue in the allQueues createQueue loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch( - /\bMEASUREMENT_TOMBSTONE_CLEANUP_QUEUE\b/, - ); + it("declares the queue in the maintenance group", () => { + expect(groupOf(QUEUE).name).toBe("maintenance"); }); - it("schedules the cron via boss.schedule (allQueues + schedules)", () => { - expect(source).toMatch( - /\[MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE,\s*MEASUREMENT_TOMBSTONE_CLEANUP_CRON\]/, - ); + it("keeps the default retries on the schedule", () => { + const job = jobOf(QUEUE); + expect(job.scheduleOptions).toBeUndefined(); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("binds a createAndWork handler to the queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE[\s\S]{0,200}handleMeasurementTombstoneCleanup/, - ); + it("binds the cleanup handler to the queue", async () => { + expect(await loadedHandler(QUEUE)).toBe(handleMeasurementTombstoneCleanup); }); }); diff --git a/src/lib/jobs/__tests__/mood-prognosis-refresh.test.ts b/src/lib/jobs/__tests__/mood-prognosis-refresh.test.ts index b3c77861d..8f6e7bd6a 100644 --- a/src/lib/jobs/__tests__/mood-prognosis-refresh.test.ts +++ b/src/lib/jobs/__tests__/mood-prognosis-refresh.test.ts @@ -46,11 +46,13 @@ const predictionFindManyMock = vi.fn(); const deleteManyMock = vi.fn(); import { - MOOD_PROGNOSIS_REFRESH_CRON, - MOOD_PROGNOSIS_REFRESH_QUEUE, handleMoodPrognosisRefresh, runMoodPrognosisRefresh, } from "@/lib/jobs/mood-prognosis-refresh"; +import { + MOOD_PROGNOSIS_REFRESH_CRON, + MOOD_PROGNOSIS_REFRESH_QUEUE, +} from "@/lib/jobs/mood-prognosis-refresh-queue"; import { MIN_ENTRIES_FOR_FORECAST } from "@/lib/analytics/mood-prognosis/thresholds"; const NOW = new Date("2026-08-08T02:35:00.000Z"); diff --git a/src/lib/jobs/__tests__/nightscout-queues.test.ts b/src/lib/jobs/__tests__/nightscout-queues.test.ts index 6c994d55a..fa2cf8173 100644 --- a/src/lib/jobs/__tests__/nightscout-queues.test.ts +++ b/src/lib/jobs/__tests__/nightscout-queues.test.ts @@ -1,96 +1,78 @@ /** - * v1.17.0 — pg-boss queue registration for the three new poll-only sync layers: - * Nightscout CGM, Polar, and Oura. + * pg-boss queue registration for the three poll-only sync layers: Nightscout + * CGM, Polar, and Oura. * - * Source-as-text guard (the reminder-worker pulls heavy infra we don't boot - * here): assert each queue constant + cron + handler wiring + allQueues - * membership are present — the fast guard against the v1.4.37 dead-queue class - * (a queue declared but never registered in `allQueues`, which silently never - * drains). All three syncs are poll-only (no webhook, no backfill queue): a - * single hourly tick walks every configured account. + * Reads the worker registry (the worker pulls heavy infra we don't boot + * here): each queue is declared in the integration-sync group with its cron + * and handler — the fast guard against the dead-queue class (a queue declared + * but never registered, which silently never drains). All three syncs are + * poll-only (no webhook): a single hourly tick walks every configured + * account. The queue names live in `integration-poll-queues.ts`, so an OAuth + * callback can enqueue a connect-time first sync without the handler graph. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring moved out of the 2143-LOC reminder-worker boot -// file into domain registrars; the Nightscout / Polar / Oura queues live in the -// integration-sync registrar. The dead-queue guard follows the wiring there -// (the handler module is still concatenated for the handler-symbol assertions). -// -// v1.32.28 — the four poll-queue NAMES moved again, into -// `integration-poll-queues.ts`, so an OAuth callback can enqueue a connect-time -// first sync without importing the registrar's handler graph. The guard follows -// them: the name declarations are asserted against that module, everything else -// (allQueues membership, cron schedule, handler binding) still against the -// registrar. -const source = - readFileSync(join(__dirname, "..", "integration-poll-queues.ts"), "utf8") + - readFileSync( - join(__dirname, "..", "reminder", "register-integration-sync.ts"), - "utf8", - ) + - readFileSync(join(__dirname, "..", "reminder", "nightscout-sync.ts"), "utf8"); +import { + NIGHTSCOUT_SYNC_CRON, + NIGHTSCOUT_SYNC_QUEUE, + OURA_SYNC_CRON, + OURA_SYNC_QUEUE, + POLAR_SYNC_CRON, + POLAR_SYNC_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; +import { handleNightscoutSync } from "@/lib/jobs/reminder/nightscout-sync"; +import { handleOuraSync } from "@/lib/jobs/reminder/oura-sync"; +import { handlePolarSync } from "@/lib/jobs/reminder/polar-sync"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -describe("reminder-worker — Nightscout sync queue", () => { +describe("Nightscout sync queue", () => { it("declares the nightscout-sync queue constant", () => { - expect(source).toMatch( - /NIGHTSCOUT_SYNC_QUEUE\s*=\s*["']nightscout-sync["']/, - ); + expect(NIGHTSCOUT_SYNC_QUEUE).toBe("nightscout-sync"); }); - it("registers the queue in allQueues (v1.4.37 dead-queue guard)", () => { - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - expect(block![1]!).toContain("NIGHTSCOUT_SYNC_QUEUE"); + it("declares the queue in the integration-sync group", () => { + expect(groupOf(NIGHTSCOUT_SYNC_QUEUE).name).toBe("integration_sync"); }); it("schedules the hourly poll cron", () => { - expect(source).toMatch(/\[NIGHTSCOUT_SYNC_QUEUE,\s*NIGHTSCOUT_SYNC_CRON\]/); + expect(jobOf(NIGHTSCOUT_SYNC_QUEUE).cron).toBe(NIGHTSCOUT_SYNC_CRON); }); - it("registers a createAndWork handler for the queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,160}NIGHTSCOUT_SYNC_QUEUE[\s\S]{0,160}handleNightscoutSync/, + it("binds the handler for the queue", async () => { + expect(await loadedHandler(NIGHTSCOUT_SYNC_QUEUE)).toBe( + handleNightscoutSync, ); }); }); -describe("reminder-worker — Polar / Oura sync queues", () => { - // The same v1.4.37 dead-queue class applies to the two OAuth-wearable polls - // shipped alongside Nightscout in v1.17.0 — assert their allQueues membership - // so a future refactor can't silently drop one and starve the sync. - const allQueuesBlock = () => { - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - return block![1]!; - }; - +describe("Polar / Oura sync queues", () => { + // The same dead-queue class applies to the two OAuth-wearable polls shipped + // alongside Nightscout — assert their registry membership so a future + // refactor can't silently drop one and starve the sync. it("declares the polar-sync and oura-sync queue constants", () => { - expect(source).toMatch(/POLAR_SYNC_QUEUE\s*=\s*["']polar-sync["']/); - expect(source).toMatch(/OURA_SYNC_QUEUE\s*=\s*["']oura-sync["']/); + expect(POLAR_SYNC_QUEUE).toBe("polar-sync"); + expect(OURA_SYNC_QUEUE).toBe("oura-sync"); }); - it("registers POLAR_SYNC_QUEUE in allQueues (v1.4.37 dead-queue guard)", () => { - expect(allQueuesBlock()).toContain("POLAR_SYNC_QUEUE"); + it("declares POLAR_SYNC_QUEUE in the integration-sync group", () => { + expect(groupOf(POLAR_SYNC_QUEUE).name).toBe("integration_sync"); }); - it("registers OURA_SYNC_QUEUE in allQueues (v1.4.37 dead-queue guard)", () => { - expect(allQueuesBlock()).toContain("OURA_SYNC_QUEUE"); + it("declares OURA_SYNC_QUEUE in the integration-sync group", () => { + expect(groupOf(OURA_SYNC_QUEUE).name).toBe("integration_sync"); }); it("schedules the hourly poll cron for both", () => { - expect(source).toMatch(/\[POLAR_SYNC_QUEUE,\s*POLAR_SYNC_CRON\]/); - expect(source).toMatch(/\[OURA_SYNC_QUEUE,\s*OURA_SYNC_CRON\]/); + expect(jobOf(POLAR_SYNC_QUEUE).cron).toBe(POLAR_SYNC_CRON); + expect(jobOf(OURA_SYNC_QUEUE).cron).toBe(OURA_SYNC_CRON); }); - it("registers a createAndWork handler for each queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,160}POLAR_SYNC_QUEUE[\s\S]{0,160}handlePolarSync/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,160}OURA_SYNC_QUEUE[\s\S]{0,160}handleOuraSync/, - ); + it("binds a handler for each queue", async () => { + expect(await loadedHandler(POLAR_SYNC_QUEUE)).toBe(handlePolarSync); + expect(await loadedHandler(OURA_SYNC_QUEUE)).toBe(handleOuraSync); }); }); diff --git a/src/lib/jobs/__tests__/period-narrative-warm.test.ts b/src/lib/jobs/__tests__/period-narrative-warm.test.ts index a41071ad5..8bdf1bb09 100644 --- a/src/lib/jobs/__tests__/period-narrative-warm.test.ts +++ b/src/lib/jobs/__tests__/period-narrative-warm.test.ts @@ -1,6 +1,4 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; const checkRateLimit = vi.fn(); const getAssistantFlags = vi.fn(); @@ -34,6 +32,14 @@ import { PERIOD_NARRATIVE_QUEUE, PERIOD_NARRATIVE_CRON, } from "../period-narrative-warm"; +import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "../period-narrative-queue"; +import { handlePeriodNarrative } from "../reminder/status-handlers"; +import { insightRetryOptions } from "../reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; function makePrisma( users: Array<{ id: string; locale: string | null }>, @@ -234,29 +240,22 @@ describe("runPeriodNarrativeWarm", () => { }); describe("queue registration", () => { - // v1.18.1 — the period-narrative wiring moved out of the 2143-LOC - // reminder-worker boot file into the status registrar. The dead-queue guard - // follows the wiring there. - const workerSrc = fs.readFileSync( - path.resolve(__dirname, "../reminder/register-status.ts"), - "utf8", - ); - - it("registers the queue in the allQueues createQueue loop", () => { - const match = workerSrc.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(match).not.toBeNull(); - expect(match![1]).toMatch(/\bPERIOD_NARRATIVE_QUEUE\b/); + it("declares the queue in the status group", () => { + expect(groupOf(PERIOD_NARRATIVE_QUEUE).name).toBe("status"); }); - it("schedules the cron in the schedules table (with retry policy)", () => { - expect(workerSrc).toMatch( - /\[\s*PERIOD_NARRATIVE_QUEUE\s*,\s*PERIOD_NARRATIVE_CRON\s*,\s*\{\s*\.\.\.insightRetryOptions\s*,\s*expireInSeconds:\s*PERIOD_NARRATIVE_EXPIRE_SECONDS\s*,?\s*\}\s*,?\s*\]/, - ); + it("schedules the cron with the retry policy", () => { + const job = jobOf(PERIOD_NARRATIVE_QUEUE); + expect(job.cron).toBe(PERIOD_NARRATIVE_CRON); + expect(job.scheduleOptions).toEqual({ + ...insightRetryOptions, + expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS, + }); }); - it("registers a createAndWork handler for the queue", () => { - expect(workerSrc).toMatch( - /createAndWork[\s\S]{0,120}PERIOD_NARRATIVE_QUEUE/, + it("binds the warm handler to the queue", async () => { + expect(await loadedHandler(PERIOD_NARRATIVE_QUEUE)).toBe( + handlePeriodNarrative, ); }); diff --git a/src/lib/jobs/__tests__/queue-policy-tables.test.ts b/src/lib/jobs/__tests__/queue-policy-tables.test.ts index bc4b5ea04..7a728b6fb 100644 --- a/src/lib/jobs/__tests__/queue-policy-tables.test.ts +++ b/src/lib/jobs/__tests__/queue-policy-tables.test.ts @@ -1,160 +1,138 @@ /** - * Dead-policy guard, in the same spirit as the v1.4.37 dead-queue guards. + * Dead-policy guard, in the same spirit as the dead-queue guards. * - * A queue's `singletonKey` only de-duplicates if its registrar names a policy - * for it: pg-boss constrains `singleton_key` through partial unique indexes - * scoped by queue policy, and the default `standard` policy has none. So the - * enqueue site and the registrar entry are two halves of one guarantee that - * live in different files, and nothing in the type system couples them — - * deleting the registrar entry leaves the `singletonKey` at the call site - * looking entirely correct while it silently stops doing anything. + * A queue's `singletonKey` only de-duplicates if its registry declaration + * names a policy for it: pg-boss constrains `singleton_key` through partial + * unique indexes scoped by queue policy, and the default `standard` policy + * has none. So the enqueue site and the declaration are two halves of one + * guarantee that live in different files, and nothing in the type system + * couples them — dropping the declaration's policy leaves the `singletonKey` + * at the call site looking entirely correct while it silently stops doing + * anything. * - * These assertions are that coupling. Each queue below is listed with the - * policy its call site depends on; dropping or weakening an entry turns this - * suite red instead of quietly restoring the storm. + * These assertions are that coupling. Each queue below is listed by its wire + * name with the policy its call site depends on, and must be declared in the + * named registry group; dropping or weakening an entry turns this suite red + * instead of quietly restoring the storm. * * The behavioural proof that the policies mean what the names say lives in * `tests/integration/queue-singleton-policy.integration.test.ts`, which runs a * real pg-boss against a real Postgres. */ -import fs from "node:fs"; -import path from "node:path"; - import { describe, expect, it } from "vitest"; -const read = (file: string) => - fs.readFileSync(path.resolve(__dirname, "../reminder", file), "utf8"); +import type { JobGroupName } from "@/lib/jobs/registry/define-job"; +import { + groupNamed, + groupOf, + jobOf, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -/** - * Assert that `queue` appears in the registrar's `queuePolicies` table with - * `policy`. Anchored to the table so an unrelated mention of the constant - * elsewhere in the file cannot satisfy it. - */ -function expectPolicy(source: string, queue: string, policy: string) { - const table = source.match( - /const queuePolicies: QueuePolicyTable = \{([\s\S]*?)\n\};/, - ); - expect(table, "registrar has no queuePolicies table").not.toBeNull(); - const entry = new RegExp(`\\[${queue}\\]:\\s*\\{\\s*policy:\\s*"${policy}"`); +/** Assert that `queue` is declared in `group` with `policy`. */ +function expectPolicy(group: JobGroupName, queue: string, policy: string) { + expect(groupOf(queue).name).toBe(group); expect( - table![1], + jobOf(queue).policy?.policy, `${queue} must carry the "${policy}" policy — without it the singletonKey at its enqueue site is inert`, - ).toMatch(entry); + ).toBe(policy); } -describe("rollup registrar queue policies", () => { - const source = read("register-rollup.ts"); - +describe("rollup group queue policies", () => { // `short`, not `exclusive`: the handler re-reads the bucket when it starts, // so only QUEUED duplicates may be collapsed. Promoting either of these to // `exclusive` would strand a measurement written while a recompute ran. it.each([ - ["ROLLUP_RECOMPUTE_QUEUE", "short"], - ["MOOD_ROLLUP_RECOMPUTE_QUEUE", "short"], + ["rollup-recompute", "short"], + ["mood-rollup-recompute", "short"], ])("%s is %s", (queue, policy) => { - expectPolicy(source, queue, policy); + expectPolicy("rollup", queue, policy); }); // Per-user, self-converging backfills: a second concurrent run is pure // duplicated work and discovery re-offers the job while it is outstanding. it.each([ - ["ROLLUP_FULL_BACKFILL_QUEUE", "exclusive"], - ["MOOD_ROLLUP_FULL_BACKFILL_QUEUE", "exclusive"], - ["STEP_CONSOLIDATION_QUEUE", "exclusive"], - ["STEP_CONSOLIDATION_REPAIR_QUEUE", "exclusive"], - ["CUMULATIVE_PR_REDERIVE_QUEUE", "exclusive"], - ["MEAN_CONSOLIDATION_QUEUE", "exclusive"], - ["DENSE_INTRADAY_RETENTION_QUEUE", "exclusive"], - ["DENSE_INTRADAY_HOURLY_REBUILD_QUEUE", "exclusive"], + ["rollup-full-backfill", "exclusive"], + ["mood-rollup-full-backfill", "exclusive"], + ["step-consolidation", "exclusive"], + ["step-consolidation-repair", "exclusive"], + ["cumulative-pr-rederive", "exclusive"], + ["mean-consolidation", "exclusive"], + ["dense-intraday-retention", "exclusive"], + ["dense-intraday-hourly-rebuild", "exclusive"], ])("%s is %s", (queue, policy) => { - expectPolicy(source, queue, policy); + expectPolicy("rollup", queue, policy); }); }); -describe("status registrar queue policies", () => { - const source = read("register-status.ts"); - - it("MORNING_DIGEST_REFRESH_QUEUE is exclusive", () => { +describe("status group queue policies", () => { + it("morning-digest-refresh is exclusive", () => { // Seven sleep-write seams enqueue the same user+localDate key. Without // this the at-most-once-per-morning contract does not hold and the failure // path re-triggers forced comprehensive generations against the provider. - expectPolicy(source, "MORNING_DIGEST_REFRESH_QUEUE", "exclusive"); + expectPolicy("status", "morning-digest-refresh", "exclusive"); }); }); -describe("integration-sync registrar queue policies", () => { - const source = read("register-integration-sync.ts"); - +describe("integration-sync group queue policies", () => { it.each([ - ["WHOOP_BACKFILL_QUEUE", "exclusive"], - ["FITBIT_BACKFILL_QUEUE", "exclusive"], - ["GOOGLE_HEALTH_BACKFILL_QUEUE", "exclusive"], - ["STRAVA_BACKFILL_QUEUE", "exclusive"], - ["OURA_BACKFILL_QUEUE", "exclusive"], - ["GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE", "exclusive"], - ["FITBIT_SLEEP_REPAIR_QUEUE", "exclusive"], - ["SLEEP_TIMELINE_BACKFILL_QUEUE", "exclusive"], - ["LAB_BIOMARKER_BACKFILL_QUEUE", "exclusive"], + ["whoop-backfill", "exclusive"], + ["fitbit-backfill", "exclusive"], + ["google-health-backfill", "exclusive"], + ["strava-backfill", "exclusive"], + ["oura-backfill", "exclusive"], + ["google-health-sleep-repair", "exclusive"], + ["fitbit-sleep-repair", "exclusive"], + ["sleep-timeline-backfill", "exclusive"], + ["lab-biomarker-backfill", "exclusive"], ])("%s is %s", (queue, policy) => { - expectPolicy(source, queue, policy); + expectPolicy("integration_sync", queue, policy); }); }); -describe("maintenance registrar queue policies", () => { - const source = read("register-maintenance.ts"); - +describe("maintenance group queue policies", () => { it.each([ - ["INTAKE_SLOT_DEDUP_QUEUE", "exclusive"], - ["NOTE_ENCRYPTION_BACKFILL_QUEUE", "exclusive"], - ["MED_NOTES_ENCRYPTION_BACKFILL_QUEUE", "exclusive"], - ["FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE", "exclusive"], - ["DOCUMENT_THUMBNAIL_BACKFILL_QUEUE", "exclusive"], - ["CONTENT_INDEX_BACKFILL_QUEUE", "exclusive"], - ["ENCRYPTION_KEY_ROTATE_QUEUE", "exclusive"], + ["intake-slot-dedup", "exclusive"], + ["note-encryption-backfill", "exclusive"], + ["med-notes-encryption-backfill", "exclusive"], + ["free-text-encryption-backfill", "exclusive"], + ["document-thumbnail-backfill", "exclusive"], + ["document-content-index-backfill", "exclusive"], + ["encryption-key-rotate", "exclusive"], ])("%s is %s", (queue, policy) => { - expectPolicy(source, queue, policy); + expectPolicy("maintenance", queue, policy); }); // Per-document queues have no discovery pass to re-converge, so a re-process // requested after the running job read the old bytes must be admitted. it.each([ - ["DOCUMENT_INDEX_QUEUE", "short"], - ["DOCUMENT_THUMBNAIL_QUEUE", "short"], - ["DOCUMENT_SUMMARY_QUEUE", "short"], + ["document-index", "short"], + ["document-thumbnail", "short"], + ["document-summary", "short"], ])("%s is %s", (queue, policy) => { - expectPolicy(source, queue, policy); + expectPolicy("maintenance", queue, policy); }); - it("ENVIRONMENT_FETCH_QUEUE is deliberately left on standard", () => { + it("environment-fetch is deliberately left on standard", () => { // Its explicit-range backfill sends with no key at all, on purpose, so any // policy would collapse two different requested date ranges onto the shared - // empty key. This asserts the omission is intentional and documented, so a - // future reader does not "complete" the table and silently merge them. - const table = source.match( - /const queuePolicies: QueuePolicyTable = \{([\s\S]*?)\n\};/, - ); - expect(table![1]).not.toMatch(/\[ENVIRONMENT_FETCH_QUEUE\]/); - expect(source).toMatch(/ENVIRONMENT_FETCH_QUEUE is LEFT ALONE/); + // empty key. A future reader must not "complete" the declaration and + // silently merge them. + expect(groupOf("environment-fetch").name).toBe("maintenance"); + expect(jobOf("environment-fetch").policy).toBeUndefined(); }); }); describe("every policy decision carries a reason", () => { - it.each([ - "register-rollup.ts", - "register-status.ts", - "register-integration-sync.ts", - "register-maintenance.ts", - ])("%s", (file) => { - const table = read(file).match( - /const queuePolicies: QueuePolicyTable = \{([\s\S]*?)\n\};/, - ); - expect(table).not.toBeNull(); - - const policies = table![1].match(/policy:\s*"(?:short|exclusive)"/g) ?? []; - const reasons = table![1].match(/reason:\s*\n?\s*"/g) ?? []; - expect(policies.length).toBeGreaterThan(0); - // A policy without a stated reason is the thing this whole change exists - // to prevent: the next person re-deriving intent from the call sites. - expect(reasons.length).toBe(policies.length); - }); + it.each(["rollup", "status", "integration_sync", "maintenance"] as const)( + "%s", + (name) => { + const decided = groupNamed(name).jobs.filter((job) => job.policy); + expect(decided.length).toBeGreaterThan(0); + // A policy without a stated reason is the thing this whole change exists + // to prevent: the next person re-deriving intent from the call sites. + for (const job of decided) { + expect(job.policy!.reason.trim(), job.queue).not.toBe(""); + } + }, + ); }); diff --git a/src/lib/jobs/__tests__/read-side-effect-queues.test.ts b/src/lib/jobs/__tests__/read-side-effect-queues.test.ts index 71c952909..d956f6d95 100644 --- a/src/lib/jobs/__tests__/read-side-effect-queues.test.ts +++ b/src/lib/jobs/__tests__/read-side-effect-queues.test.ts @@ -2,72 +2,65 @@ * Dead-queue guard for the two queues that took the writes off the read paths. * * Moving a write off a GET only works if the new owner actually runs. A queue - * that is not in `allQueues` is never provisioned, a schedule without a - * `createAndWork` binding drains into nothing, and either failure is silent: + * the worker registry does not declare is never provisioned, a schedule + * without a bound handler drains into nothing, and either failure is silent: * the cron fires, the job vanishes, and the only symptom is a forecast that * stops moving or an unlock date that is never pinned. Both look exactly like * "nothing to do". * - * Source-text assertions over the registrar, matching the sibling - * `*-queue.test.ts` guards. + * Reads the registry declarations, matching the sibling `*-queue.test.ts` + * guards. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; +import { handleAchievementUnlockSweep } from "@/lib/jobs/achievement-unlock-sweep"; +import { + ACHIEVEMENT_UNLOCK_SWEEP_CRON, + ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, +} from "@/lib/jobs/achievement-unlock-sweep-queue"; +import { + CYCLE_PREDICTION_REFRESH_CRON, + CYCLE_PREDICTION_REFRESH_QUEUE, + handleCyclePredictionRefresh, +} from "@/lib/jobs/cycle-prediction-refresh"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; import { SUBSYSTEM_SURFACE } from "@/lib/jobs/subsystem-surface"; -const registrar = readFileSync( - join(__dirname, "..", "reminder", "register-maintenance.ts"), - "utf8", -); - -const allQueues = registrar.match(/const allQueues\s*=\s*\[([\s\S]*?)\n\];/); - const QUEUES = [ { label: "cycle-prediction-refresh", - binding: "CYCLE_PREDICTION_REFRESH_QUEUE", - cron: "CYCLE_PREDICTION_REFRESH_CRON", - handler: "handleCyclePredictionRefresh", + queue: CYCLE_PREDICTION_REFRESH_QUEUE, + cron: CYCLE_PREDICTION_REFRESH_CRON, + handler: handleCyclePredictionRefresh, name: "cycle-prediction-refresh", }, { label: "achievement-unlock-sweep", - binding: "ACHIEVEMENT_UNLOCK_SWEEP_QUEUE", - cron: "ACHIEVEMENT_UNLOCK_SWEEP_CRON", - handler: "handleAchievementUnlockSweep", + queue: ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, + cron: ACHIEVEMENT_UNLOCK_SWEEP_CRON, + handler: handleAchievementUnlockSweep, name: "achievement-unlock-sweep", }, ]; describe("queues that own a write moved off a GET", () => { - it("extracted an allQueues list to assert against", () => { - // The regexes below all read from this capture; if it ever stops - // matching, every assertion in this file would pass on an empty string. - expect(allQueues).not.toBeNull(); - expect(allQueues![1].length).toBeGreaterThan(0); - }); - for (const queue of QUEUES) { describe(queue.label, () => { - it("is provisioned in the allQueues list", () => { - expect(allQueues![1]).toMatch(new RegExp(`\\b${queue.binding}\\b`)); + it("is declared in the maintenance group", () => { + expect(queue.queue).toBe(queue.name); + expect(groupOf(queue.queue).name).toBe("maintenance"); }); it("carries a cron schedule", () => { - expect(registrar).toMatch( - new RegExp(`\\[\\s*${queue.binding},\\s*${queue.cron}`), - ); + expect(jobOf(queue.queue).cron).toBe(queue.cron); }); - it("binds a createAndWork handler that drains it", () => { - expect(registrar).toMatch( - new RegExp( - `createAndWork[\\s\\S]{0,200}${queue.binding}[\\s\\S]{0,200}${queue.handler}`, - ), - ); + it("binds the handler that drains it", async () => { + expect(await loadedHandler(queue.queue)).toBe(queue.handler); }); it("declares a failure audience", () => { diff --git a/src/lib/jobs/__tests__/recovery-score-queue.test.ts b/src/lib/jobs/__tests__/recovery-score-queue.test.ts index 65628b9e9..d97c088ff 100644 --- a/src/lib/jobs/__tests__/recovery-score-queue.test.ts +++ b/src/lib/jobs/__tests__/recovery-score-queue.test.ts @@ -1,10 +1,10 @@ /** - * v1.10.0 — computed scores (WX-C). Recovery-score queue registration guard. + * Computed scores: recovery-score queue registration guard. * - * Same source-text-grep approach as the other queue-wiring guards: assert - * the queue is imported, registered in `allQueues`, scheduled, and wired to - * a `createAndWork` handler — without booting pg-boss + Prisma. An unregistered - * queue silently never drains (the recurring past bug this guards against). + * Reads the worker registry: the queue is declared in the status group, + * scheduled on its cron, and bound to the handler that runs the pass. An + * unregistered queue silently never drains (the recurring past bug this + * guards against). * * Also pins the iOS-write rejection of the server-owned COMPUTED source on * the batch ingest route: the `batchSourceEnum` allowlist must stay exactly @@ -16,11 +16,16 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring moved out of the 2143-LOC reminder-worker boot -// file into domain registrars; the recovery-score queue lives in the status -// registrar. The dead-queue guard follows the wiring there. -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-status.ts"); -const workerSource = readFileSync(REGISTRAR_PATH, "utf8"); +import { + RECOVERY_SCORE_CRON, + RECOVERY_SCORE_QUEUE, +} from "@/lib/jobs/recovery-score-queue"; +import { handleRecoveryScore } from "@/lib/jobs/reminder/status-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; const BATCH_ROUTE_PATH = join( __dirname, @@ -35,32 +40,19 @@ const BATCH_ROUTE_PATH = join( ); const batchSource = readFileSync(BATCH_ROUTE_PATH, "utf8"); -describe("reminder-worker — recovery-score wiring", () => { - it("imports the queue symbols from the recovery-score module", () => { - expect(workerSource).toMatch(/from\s*["']@\/lib\/jobs\/recovery-score["']/); - expect(workerSource).toMatch(/\bRECOVERY_SCORE_QUEUE\b/); - expect(workerSource).toMatch(/\bRECOVERY_SCORE_CRON\b/); - expect(workerSource).toMatch(/\brunRecoveryScore\b/); - }); - - it("registers the recovery-score queue in the allQueues loop", () => { - const allQueuesMatch = workerSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bRECOVERY_SCORE_QUEUE\b/); +describe("recovery-score registration", () => { + it("declares the recovery-score queue in the status group", () => { + expect(groupOf(RECOVERY_SCORE_QUEUE).name).toBe("status"); }); it("schedules the recovery-score cron", () => { - expect(workerSource).toMatch( - /\[RECOVERY_SCORE_QUEUE,\s*RECOVERY_SCORE_CRON\]/, - ); + const job = jobOf(RECOVERY_SCORE_QUEUE); + expect(job.cron).toBe(RECOVERY_SCORE_CRON); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("registers a createAndWork handler that runs the recovery-score pass", () => { - expect(workerSource).toMatch( - /createAndWork[\s\S]{0,200}RECOVERY_SCORE_QUEUE[\s\S]{0,400}runRecoveryScore/, - ); + it("binds the handler that runs the recovery-score pass", async () => { + expect(await loadedHandler(RECOVERY_SCORE_QUEUE)).toBe(handleRecoveryScore); }); }); diff --git a/src/lib/jobs/__tests__/restore-drill.test.ts b/src/lib/jobs/__tests__/restore-drill.test.ts index 81eb035ef..07136af8d 100644 --- a/src/lib/jobs/__tests__/restore-drill.test.ts +++ b/src/lib/jobs/__tests__/restore-drill.test.ts @@ -2,13 +2,12 @@ import type { Readable } from "node:stream"; import { describe, it, expect, beforeEach, vi } from "vitest"; import { _resetCryptoCacheForTests, encrypt } from "@/lib/crypto"; import { encryptBackup } from "../offhost-backup"; +import { handleRestoreDrill, runRestoreDrill } from "../restore-drill"; import { - handleRestoreDrill, - runRestoreDrill, RESTORE_DRILL_CRON, RESTORE_DRILL_QUEUE, RESTORE_DRILL_SEND_OPTIONS, -} from "../restore-drill"; +} from "@/lib/jobs/restore-drill-queue"; import { QUEUE_RUNTIME } from "../queue-runtime"; vi.mock("../report-worker-error", () => ({ diff --git a/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts b/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts index cc803ecc5..f689739ec 100644 --- a/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts +++ b/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts @@ -1,50 +1,60 @@ /** - * v1.17.1 — pg-boss queue registration for the one-shot sleep-timeline backfill. + * pg-boss queue registration for the one-shot sleep-timeline backfill. * - * The reminder-worker imports heavy infrastructure, so we read the source as - * text and assert the queue constant is declared, registered in `allQueues`, - * wired to a `createAndWork` handler, and enqueued at boot — the fast guard that - * catches the v1.4.37 dead-queue class (a queue declared but never registered, - * which silently never drains). + * Reads the worker registry: the source queue is declared in the + * integration-sync group, forwards each job to the shared admission queue, + * and is offered again by the boot discovery. This is the fast guard for the + * dead-queue class (a queue declared but never registered, which silently + * never drains). */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring + boot discovery moved out of the 2143-LOC -// reminder-worker boot file into domain registrars; the sleep-timeline backfill -// lives in the integration-sync registrar. The dead-queue guard follows it. -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-integration-sync.ts", -); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { INTEGRATION_BACKFILL_ADMISSION_QUEUE } from "@/lib/jobs/integration-backfill-admission"; +import { + discoveryOf, + groupOf, + jobOf, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { + SLEEP_TIMELINE_BACKFILL_CONCURRENCY, + SLEEP_TIMELINE_BACKFILL_QUEUE, +} from "@/lib/jobs/sleep-timeline-backfill-queue"; -describe("reminder-worker — sleep-timeline backfill queue", () => { - it("registers the queue in allQueues (v1.4.37 dead-queue guard)", () => { - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - expect(block![1]!).toContain("SLEEP_TIMELINE_BACKFILL_QUEUE"); +describe("sleep-timeline backfill registration", () => { + it("declares the queue in the integration-sync group", () => { + expect(groupOf(SLEEP_TIMELINE_BACKFILL_QUEUE).name).toBe( + "integration_sync", + ); + const job = jobOf(SLEEP_TIMELINE_BACKFILL_QUEUE); + expect(job.policy?.policy).toBe("exclusive"); + expect(job.work).toEqual({ + localConcurrency: SLEEP_TIMELINE_BACKFILL_CONCURRENCY, + }); }); - it("wires a createAndWork handler through the shared admission queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,260}SLEEP_TIMELINE_BACKFILL_QUEUE[\s\S]{0,300}enqueueIntegrationBackfillAdmission/, - ); + it("forwards each job through the shared admission queue", async () => { + const { boss, sends } = recordingBoss(); + const handler = await jobOf(SLEEP_TIMELINE_BACKFILL_QUEUE).load({ boss }); + await handler([ + { + id: "j1", + name: SLEEP_TIMELINE_BACKFILL_QUEUE, + data: { userId: "u1", provider: "WHOOP" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + expect(sends).toHaveLength(1); + expect(sends[0]!.queue).toBe(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(sends[0]!.data).toMatchObject({ kind: "sleep-timeline-backfill" }); }); it("wires the boot discovery", () => { - expect(source).toMatch( - /enqueueBootTimeSleepTimelineBackfill\(\s*bootStaggerSecondsFor\("sleep-timeline-backfill"\)/, - ); - }); - - it("imports the backfill exports", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/jobs\/sleep-timeline-backfill["']/, + expect(discoveryOf("integration_sync")).toContain( + "sleep-timeline-backfill", ); }); }); diff --git a/src/lib/jobs/__tests__/status-cron-candidates.test.ts b/src/lib/jobs/__tests__/status-cron-candidates.test.ts index 331de5d62..b912a4c27 100644 --- a/src/lib/jobs/__tests__/status-cron-candidates.test.ts +++ b/src/lib/jobs/__tests__/status-cron-candidates.test.ts @@ -1,5 +1,5 @@ /** - * v1.15.20 — unit tests for the shared 02:xx status-cron discovery. + * Unit tests for the shared 02:xx status-cron discovery. * * Pins the three gates the discovery applies: * 1. the operator assistant kill-switch (`insightStatus` flag), @@ -9,8 +9,8 @@ * 3. the pregenerate-candidate skip (a stale or missing comprehensive * cache → the 04:30 pass owns the user; the 02:xx crons keep the * fresh-cache accounts), applied only while `briefing` is on, - * plus the mood-status queue registration in the worker source (the - * v1.4.37 dead-queue class guard). + * plus the nightly status batch's declaration in the worker registry (the + * dead-queue class guard). */ import { describe, it, expect, vi, beforeEach } from "vitest"; import fs from "node:fs"; @@ -25,6 +25,15 @@ vi.mock("@/lib/feature-flags", async () => import { findStatusCronCandidates } from "../status-cron-candidates"; import { PREGENERATE_STALE_MS } from "../insight-pregenerate"; +import { handleGeneralStatusGenerate } from "../reminder/insights-handlers"; +import { nightlyInsightPassOptions } from "../reminder/registrar-shared"; +import { allJobDefinitions } from "../registry"; +import { + groupNamed, + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; const NOW = new Date("2026-06-10T02:00:00.000Z"); const STALE_AT = new Date(NOW.getTime() - PREGENERATE_STALE_MS - 60_000); @@ -176,10 +185,7 @@ describe("findStatusCronCandidates — gates", () => { }); describe("the nightly status batch is the one status cron", () => { - const workerSrc = fs.readFileSync( - path.resolve(__dirname, "../reminder/register-status.ts"), - "utf8", - ); + const GENERAL_STATUS_QUEUE = "insights-general-status"; const RETIRED = [ "insights-blood-pressure-status", "insights-weight-status", @@ -189,27 +195,22 @@ describe("the nightly status batch is the one status cron", () => { "insights-medication-compliance-status", ]; - it("registers and schedules the 02:00 batch with the nightly pass options", () => { - const match = workerSrc.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(match).not.toBeNull(); - expect(match![1]).toMatch(/\bGENERAL_STATUS_QUEUE\b/); - expect(workerSrc).toMatch( - /\[\s*GENERAL_STATUS_QUEUE\s*,\s*GENERAL_STATUS_CRON\s*,\s*nightlyInsightPassOptions\s*\]/, + it("registers and schedules the 02:00 batch with the nightly pass options", async () => { + expect(groupOf(GENERAL_STATUS_QUEUE).name).toBe("status"); + const job = jobOf(GENERAL_STATUS_QUEUE); + expect(job.cron).toBe("0 2 * * *"); + expect(job.scheduleOptions).toBe(nightlyInsightPassOptions); + expect(await loadedHandler(GENERAL_STATUS_QUEUE)).toBe( + handleGeneralStatusGenerate, ); - expect(workerSrc).toMatch(/GENERAL_STATUS_CRON\s*=\s*"0 2 \* \* \*"/); }); it("retires the six per-metric crons on an instance that still has them", () => { - const retired = workerSrc.match( - /const RETIRED_STATUS_QUEUES\s*=\s*\[([\s\S]*?)\]/, + expect([...(groupNamed("status").retired ?? [])].sort()).toEqual( + [...RETIRED].sort(), ); - expect(retired).not.toBeNull(); - for (const name of RETIRED) expect(retired![1]).toContain(`"${name}"`); - expect(workerSrc).toMatch( - /await retireQueues\(boss,\s*RETIRED_STATUS_QUEUES\)/, - ); - const live = workerSrc.replace(retired![0], ""); - for (const name of RETIRED) expect(live).not.toContain(`"${name}"`); + const live = allJobDefinitions().map((job) => job.queue); + for (const name of RETIRED) expect(live).not.toContain(name); }); it("drives the batch through the shared discovery", () => { diff --git a/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts b/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts index e34c295cd..2f679fb63 100644 --- a/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts +++ b/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts @@ -1,51 +1,46 @@ /** - * v1.28.37 — step-consolidation repair queue registration guard + - * `extractStatsDay` unit coverage. + * Step-consolidation repair queue registration guard + `extractStatsDay` + * unit coverage. * - * Same source-text-grep approach as the drain + step-consolidation + - * mean-consolidation guards: assert the repair queue is registered in - * `allQueues`, a `createAndWork` handler is wired against it, and the - * boot-discovery enqueue helper fires — without booting pg-boss + Prisma. - * An unregistered queue silently never drains (the dead-queue class). + * Reads the worker registry: the repair queue is declared in the rollup + * group, bound to the repair handler, and offered again by the rollup boot + * discovery. An unregistered queue silently never drains (the dead-queue + * class). */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -import { extractStatsDay } from "../step-consolidation-repair"; +import { handleStepConsolidationRepair } from "@/lib/jobs/reminder/rollup-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-rollup.ts"); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { + STEP_CONSOLIDATION_REPAIR_CONCURRENCY, + STEP_CONSOLIDATION_REPAIR_QUEUE, + extractStatsDay, +} from "../step-consolidation-repair"; -describe("reminder-worker — step-consolidation-repair wiring", () => { - it("imports the queue symbols from the repair module", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/jobs\/step-consolidation-repair["']/, - ); - expect(source).toMatch(/\bSTEP_CONSOLIDATION_REPAIR_QUEUE\b/); - expect(source).toMatch(/\brunStepConsolidationRepairForUser\b/); - expect(source).toMatch(/\benqueueBootTimeStepConsolidationRepair\b/); +describe("step-consolidation-repair registration", () => { + it("declares the repair queue in the rollup group", () => { + expect(groupOf(STEP_CONSOLIDATION_REPAIR_QUEUE).name).toBe("rollup"); + const job = jobOf(STEP_CONSOLIDATION_REPAIR_QUEUE); + expect(job.policy?.policy).toBe("exclusive"); + expect(job.work).toEqual({ + localConcurrency: STEP_CONSOLIDATION_REPAIR_CONCURRENCY, + }); }); - it("registers the repair queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bSTEP_CONSOLIDATION_REPAIR_QUEUE\b/); - }); - - it("registers a createAndWork handler against the repair queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}STEP_CONSOLIDATION_REPAIR_QUEUE[\s\S]{0,400}runStepConsolidationRepairForUser/, + it("binds the repair handler", async () => { + expect(await loadedHandler(STEP_CONSOLIDATION_REPAIR_QUEUE)).toBe( + handleStepConsolidationRepair, ); }); - it("fires the boot-discovery enqueue helper", () => { - expect(source).toMatch( - /await enqueueBootTimeStepConsolidationRepair\([^)]*\)/, - ); + it("runs the boot discovery pass", () => { + expect(discoveryOf("rollup")).toContain("step-consolidation-repair"); }); }); diff --git a/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts b/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts index 969d80fba..2595ac000 100644 --- a/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts +++ b/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts @@ -1,39 +1,125 @@ /** - * v1.10.0 — computed scores (WX-E). Queue-registration guards for the - * Stress + Strain nightly score jobs and the dense intra-day retention - * drain. + * Computed scores: queue-registration guards for the Stress + Strain nightly + * score jobs and the dense intra-day retention drain. * - * Same source-text-grep approach as the WX-C recovery-score guard: assert - * each queue is imported, registered in `allQueues`, scheduled (the two - * cron-driven score jobs), and wired to a `createAndWork` handler — without - * booting pg-boss + Prisma. An unregistered queue silently never drains - * (the recurring past bug this guards against). + * Reads the worker registry: each queue is declared in its group, scheduled + * (the two cron-driven score jobs), bound to its handler, and offered again by + * boot discovery where it has one — without booting pg-boss + Prisma. An + * unregistered queue silently never drains (the recurring past bug this + * guards against). The retention handler and the nightly drain run against + * mocked passes. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import type { Job } from "pg-boss"; +import { describe, expect, it, vi } from "vitest"; -// v1.18.1 — the queue wiring moved out of the 2143-LOC reminder-worker boot -// file into domain registrars. Stress + Strain live in the status registrar; -// the dense intra-day retention queue (and the nightly fold onto the -// drain-cumulative tick) lives in the rollup registrar. Concatenate both so the -// dead-queue guard follows the wiring into each. -const STATUS_REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-status.ts", +const { + runDenseIntradayRetention, + runDenseIntradayRetentionForUser, + workerLog, +} = vi.hoisted(() => ({ + runDenseIntradayRetention: vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + daysConsolidated: 0, + perSampleRowsSoftDeleted: 0, + hourlyRowsUpserted: 0, + dailyRowsRetired: 0, + }, + })), + runDenseIntradayRetentionForUser: vi.fn(async () => ({ + daysConsolidated: 1, + perSampleRowsSoftDeleted: 2, + derivedRestingRowsUpserted: 3, + stoppedEarly: false, + })), + workerLog: vi.fn(), +})); +vi.mock( + "@/lib/measurements/dense-intraday-retention", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/dense-intraday-retention") + >()), + runDenseIntradayRetention, + }), ); -const ROLLUP_REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-rollup.ts", +vi.mock("@/lib/jobs/dense-intraday-retention", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/dense-intraday-retention") + >()), + runDenseIntradayRetentionForUser, +})); +vi.mock( + "@/lib/measurements/drain-per-sample-cumulative", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/drain-per-sample-cumulative") + >()), + drainPerSampleCumulative: vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + bucketsCollapsed: 0, + perSampleRowsDeleted: 0, + dailyRowsUpserted: 0, + }, + })), + }), ); -const source = - readFileSync(STATUS_REGISTRAR_PATH, "utf8") + - readFileSync(ROLLUP_REGISTRAR_PATH, "utf8"); +vi.mock( + "@/lib/measurements/consolidate-daily-mean", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/consolidate-daily-mean") + >()), + consolidateDailyMean: vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + daysConsolidated: 0, + perSampleRowsSoftDeleted: 0, + dailyRowsUpserted: 0, + daysFailed: 0, + }, + })), + }), +); +vi.mock("@/lib/jobs/reminder/shared", async (importOriginal) => ({ + ...(await importOriginal()), + getWorkerPrisma: () => ({}), + workerLog, +})); + +import { DENSE_INTRADAY_HOURLY_REBUILD_QUEUE } from "@/lib/jobs/dense-intraday-hourly-rebuild-queue"; +import { DENSE_INTRADAY_RETENTION_QUEUE } from "@/lib/jobs/dense-intraday-retention-queue"; +import { DRAIN_CUMULATIVE_QUEUE } from "@/lib/jobs/drain-cumulative-queue"; +import { + handleDenseIntradayHourlyRebuild, + handleDenseIntradayRetention, +} from "@/lib/jobs/reminder/rollup-handlers"; +import { + handleStrainScore, + handleStressScore, +} from "@/lib/jobs/reminder/status-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { + STRAIN_SCORE_CRON, + STRAIN_SCORE_QUEUE, +} from "@/lib/jobs/strain-score-queue"; +import { + STRESS_SCORE_CRON, + STRESS_SCORE_QUEUE, +} from "@/lib/jobs/stress-score-queue"; const DENSE_RETENTION_PATH = join( __dirname, @@ -42,91 +128,70 @@ const DENSE_RETENTION_PATH = join( ); const denseRetentionSource = readFileSync(DENSE_RETENTION_PATH, "utf8"); -function allQueuesBlock(): string { - // Each registrar owns its own `const allQueues = [...]`; union every block so - // a queue registered in either the status or rollup registrar satisfies the - // membership assertion. - const blocks = [...source.matchAll(/const allQueues\s*=\s*\[([\s\S]*?)\];/g)]; - expect(blocks.length).toBeGreaterThan(0); - return blocks.map((m) => m[1]).join("\n"); +function job(name: string, data: T): Job { + return { + id: "j1", + name, + data, + expireInSeconds: 900, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }; } -describe("reminder-worker — stress-score wiring", () => { - it("imports the queue symbols from the stress-score module", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/stress-score["']/); - expect(source).toMatch(/\bSTRESS_SCORE_QUEUE\b/); - expect(source).toMatch(/\bSTRESS_SCORE_CRON\b/); - expect(source).toMatch(/\brunStressScore\b/); - }); - - it("registers the stress-score queue in allQueues", () => { - expect(allQueuesBlock()).toMatch(/\bSTRESS_SCORE_QUEUE\b/); +describe("stress-score registration", () => { + it("declares the stress-score queue in the status group", () => { + expect(groupOf(STRESS_SCORE_QUEUE).name).toBe("status"); }); it("schedules the stress-score cron", () => { - expect(source).toMatch(/\[STRESS_SCORE_QUEUE,\s*STRESS_SCORE_CRON\]/); + expect(jobOf(STRESS_SCORE_QUEUE).cron).toBe(STRESS_SCORE_CRON); }); - it("registers a createAndWork handler that runs the stress-score pass", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}STRESS_SCORE_QUEUE[\s\S]{0,400}runStressScore/, - ); + it("binds the handler that runs the stress-score pass", async () => { + expect(await loadedHandler(STRESS_SCORE_QUEUE)).toBe(handleStressScore); }); }); -describe("reminder-worker — strain-score wiring", () => { - it("imports the queue symbols from the strain-score module", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/strain-score["']/); - expect(source).toMatch(/\bSTRAIN_SCORE_QUEUE\b/); - expect(source).toMatch(/\bSTRAIN_SCORE_CRON\b/); - expect(source).toMatch(/\brunStrainScore\b/); - }); - - it("registers the strain-score queue in allQueues", () => { - expect(allQueuesBlock()).toMatch(/\bSTRAIN_SCORE_QUEUE\b/); +describe("strain-score registration", () => { + it("declares the strain-score queue in the status group", () => { + expect(groupOf(STRAIN_SCORE_QUEUE).name).toBe("status"); }); it("schedules the strain-score cron", () => { - expect(source).toMatch(/\[STRAIN_SCORE_QUEUE,\s*STRAIN_SCORE_CRON\]/); + expect(jobOf(STRAIN_SCORE_QUEUE).cron).toBe(STRAIN_SCORE_CRON); }); - it("registers a createAndWork handler that runs the strain-score pass", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}STRAIN_SCORE_QUEUE[\s\S]{0,400}runStrainScore/, - ); + it("binds the handler that runs the strain-score pass", async () => { + expect(await loadedHandler(STRAIN_SCORE_QUEUE)).toBe(handleStrainScore); }); }); -describe("reminder-worker — dense intra-day retention wiring", () => { - it("imports the queue symbols from the dense-intraday-retention module", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/jobs\/dense-intraday-retention["']/, - ); - expect(source).toMatch(/\bDENSE_INTRADAY_RETENTION_QUEUE\b/); - expect(source).toMatch(/\brunDenseIntradayRetentionForUser\b/); - expect(source).toMatch(/\benqueueBootTimeDenseIntradayRetention\b/); - }); - - it("registers the dense-intraday-retention queue in allQueues", () => { - expect(allQueuesBlock()).toMatch(/\bDENSE_INTRADAY_RETENTION_QUEUE\b/); - }); - - it("registers a createAndWork handler against the dense-intraday-retention queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,250}DENSE_INTRADAY_RETENTION_QUEUE[\s\S]{0,500}runDenseIntradayRetentionForUser/, +describe("dense intra-day retention registration", () => { + it("declares the dense-intraday-retention queue in the rollup group", () => { + expect(groupOf(DENSE_INTRADAY_RETENTION_QUEUE).name).toBe("rollup"); + expect(jobOf(DENSE_INTRADAY_RETENTION_QUEUE).policy?.policy).toBe( + "exclusive", ); }); - it("fires the boot-discovery enqueue helper", () => { - expect(source).toMatch(/await enqueueBootTimeDenseIntradayRetention\(\)/); + it("binds the per-user retention handler", async () => { + expect(await loadedHandler(DENSE_INTRADAY_RETENTION_QUEUE)).toBe( + handleDenseIntradayRetention, + ); + }); + + it("fires the boot-discovery pass", () => { + expect(discoveryOf("rollup")).toContain("dense-intraday-retention"); }); it("defers the boot-discovery drain past the startup storm (P2028 guard)", () => { // The boot-time per-user retention jobs MUST carry a startAfter delay so // the transaction-per-day drain never contends with the deploy storm + // migration + other boot backfills + foreground health-checks for the - // shared connection pool. Dropping this re-opens the v1.10.0 pool- - // exhaustion (P2028) restart loop on a data-heavy tenant. + // shared connection pool. Dropping this re-opens the pool-exhaustion + // (P2028) restart loop on a data-heavy tenant. expect(denseRetentionSource).toMatch( /DENSE_INTRADAY_RETENTION_BOOT_DELAY_SECONDS\s*=\s*\d+/, ); @@ -135,44 +200,52 @@ describe("reminder-worker — dense intra-day retention wiring", () => { ); }); - it("folds the nightly dense-retention walk onto the drain-cumulative tick", () => { + it("folds the nightly dense-retention walk onto the drain-cumulative tick", async () => { // The steady-state nightly walk runs the global (no-userId) pass on the // same tick as the cumulative + mean drains. - expect(source).toMatch(/runDenseIntradayRetention\(\s*getWorkerPrisma\(\)/); + const { boss } = recordingBoss(); + const drain = await jobOf(DRAIN_CUMULATIVE_QUEUE).load({ boss }); + await drain([job(DRAIN_CUMULATIVE_QUEUE, {})]); + expect(runDenseIntradayRetention).toHaveBeenCalledTimes(1); + expect(runDenseIntradayRetention).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ dryRun: false }), + ); }); - it("surfaces the iOS#34 derived-resting count in the per-user worker log", () => { + it("surfaces the iOS#34 derived-resting count in the per-user worker log", async () => { // The PULSE facet mints derived RESTING_HEART_RATE rows for proxy users; // the per-user handler returns the count and the worker logs it so the // signal-preservation is observable in production. - expect(source).toMatch(/derivedRestingRowsUpserted/); + const outcome = await handleDenseIntradayRetention([ + job(DENSE_INTRADAY_RETENTION_QUEUE, { + userId: "u1", + enqueuedAt: "2026-01-01T00:00:00.000Z", + }), + ]); + expect(outcome).toMatchObject({ + did: { derived_resting_rows_upserted: 3 }, + }); + expect(workerLog).toHaveBeenCalledWith( + "info", + expect.stringContaining("derivedRestingRowsUpserted=3"), + ); }); }); -describe("reminder-worker — dense hourly-rebuild wiring", () => { - it("imports the queue symbols from the dense-intraday-hourly-rebuild module", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/jobs\/dense-intraday-hourly-rebuild["']/, - ); - expect(source).toMatch(/\bDENSE_INTRADAY_HOURLY_REBUILD_QUEUE\b/); - expect(source).toMatch(/\brunDenseIntradayHourlyRebuildForUser\b/); - expect(source).toMatch(/\benqueueBootTimeDenseIntradayHourlyRebuild\b/); +describe("dense hourly-rebuild registration", () => { + it("declares the hourly-rebuild queue in the rollup group", () => { + expect(groupOf(DENSE_INTRADAY_HOURLY_REBUILD_QUEUE).name).toBe("rollup"); }); - it("registers the hourly-rebuild queue in allQueues", () => { - expect(allQueuesBlock()).toMatch(/\bDENSE_INTRADAY_HOURLY_REBUILD_QUEUE\b/); - }); - - it("registers a createAndWork handler against the hourly-rebuild queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,250}DENSE_INTRADAY_HOURLY_REBUILD_QUEUE[\s\S]{0,500}runDenseIntradayHourlyRebuildForUser/, + it("binds the hourly-rebuild handler", async () => { + expect(await loadedHandler(DENSE_INTRADAY_HOURLY_REBUILD_QUEUE)).toBe( + handleDenseIntradayHourlyRebuild, ); }); - it("fires the boot-discovery enqueue helper", () => { - expect(source).toMatch( - /await enqueueBootTimeDenseIntradayHourlyRebuild\(\)/, - ); + it("fires the boot-discovery pass", () => { + expect(discoveryOf("rollup")).toContain("dense-intraday-hourly-rebuild"); }); it("defers the boot-discovery rebuild past the startup storm (P2028 guard)", () => { diff --git a/src/lib/jobs/__tests__/subsystem-surface.test.ts b/src/lib/jobs/__tests__/subsystem-surface.test.ts index bb32673db..d416cb006 100644 --- a/src/lib/jobs/__tests__/subsystem-surface.test.ts +++ b/src/lib/jobs/__tests__/subsystem-surface.test.ts @@ -1,92 +1,17 @@ -import { existsSync, readFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +/** + * Every queue the worker registry declares has a failure audience in + * `SUBSYSTEM_SURFACE`, and the table names no queue the registry has dropped. + */ import { describe, expect, it } from "vitest"; +import { allJobDefinitions } from "@/lib/jobs/registry"; import { SUBSYSTEM_SURFACE, type SubsystemSurface, } from "@/lib/jobs/subsystem-surface"; -const REGISTRARS = [ - "register-integration-sync.ts", - "register-status.ts", - "register-rollup.ts", - "register-reminders.ts", - "register-maintenance.ts", -].map((file) => join(process.cwd(), "src/lib/jobs/reminder", file)); - -function withoutComments(source: string): string { - return source.replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/.*$/gm, ""); -} - -export function extractAllQueueBindings(source: string): string[] { - const match = /const\s+allQueues\s*=\s*\[([\s\S]*?)\n\s*\];/.exec(source); - if (!match) throw new Error("Registrar has no allQueues array"); - - return withoutComments(match[1]) - .split(",") - .map((entry) => entry.trim()) - .filter(Boolean) - .map((entry) => { - if (!/^[A-Z][A-Z0-9_]*$/.test(entry)) { - throw new Error(`Unsupported allQueues entry: ${entry}`); - } - return entry; - }); -} - -function literalPattern(binding: string): RegExp { - return new RegExp( - `(?:export\\s+)?const\\s+${binding}(?:\\s*:[^=;]+)?\\s*=\\s*["']([^"']+)["']`, - ); -} - -function modulePath(fromFile: string, specifier: string): string { - const base = specifier.startsWith("@/") - ? resolve(process.cwd(), "src", specifier.slice(2)) - : resolve(dirname(fromFile), specifier); - const candidates = [`${base}.ts`, `${base}.tsx`, join(base, "index.ts")]; - const found = candidates.find((candidate) => existsSync(candidate)); - if (!found) throw new Error(`Cannot resolve ${specifier} from ${fromFile}`); - return found; -} - -function resolveQueueBinding( - binding: string, - file: string, - seen: Set = new Set(), -): string { - const visit = `${file}:${binding}`; - if (seen.has(visit)) throw new Error(`Cyclic queue binding: ${visit}`); - seen.add(visit); - - const source = withoutComments(readFileSync(file, "utf8")); - const local = literalPattern(binding).exec(source); - if (local) return local[1]; - - const imports = /import\s*{([\s\S]*?)}\s*from\s*["']([^"']+)["'];?/g; - for (const match of source.matchAll(imports)) { - for (const rawToken of match[1].split(",")) { - const token = rawToken.trim().replace(/^type\s+/, ""); - if (!token) continue; - const [imported, alias] = token - .split(/\s+as\s+/) - .map((part) => part.trim()); - if ((alias ?? imported) !== binding) continue; - return resolveQueueBinding(imported, modulePath(file, match[2]), seen); - } - } - - throw new Error(`Cannot resolve queue binding ${binding} from ${file}`); -} - function registeredQueueNames(): string[] { - return REGISTRARS.flatMap((file) => { - const source = readFileSync(file, "utf8"); - return extractAllQueueBindings(source).map((binding) => - resolveQueueBinding(binding, file), - ); - }); + return allJobDefinitions().map((job) => job.queue); } export function findSurfaceDrift( @@ -113,8 +38,9 @@ function assertSurfaceComplete( } describe("SUBSYSTEM_SURFACE", () => { - it("classifies every current registrar queue with no stale entries", () => { + it("classifies every current registry queue with no stale entries", () => { const names = registeredQueueNames(); + expect(names.length).toBeGreaterThan(0); expect(new Set(names).size).toBe(names.length); expect(() => assertSurfaceComplete(names, SUBSYSTEM_SURFACE)).not.toThrow(); }); @@ -126,24 +52,9 @@ describe("SUBSYSTEM_SURFACE", () => { } }); - it("catches a new allQueues binding without a surface entry", () => { - const fixture = ` - const EXISTING_QUEUE = "existing-queue"; - const NEW_QUEUE = "new-queue"; - const allQueues = [ - EXISTING_QUEUE, - NEW_QUEUE, - ]; - `; - const names = extractAllQueueBindings(fixture).map((binding) => { - const match = literalPattern(binding).exec(withoutComments(fixture)); - if (!match) - throw new Error(`Fixture binding did not resolve: ${binding}`); - return match[1]; - }); - + it("catches a new registry queue without a surface entry", () => { expect(() => - assertSurfaceComplete(names, { + assertSurfaceComplete(["existing-queue", "new-queue"], { "existing-queue": { audience: "system" }, }), ).toThrow(/missing=\[new-queue\]/); diff --git a/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts b/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts index 9fe67d0b0..544707125 100644 --- a/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts +++ b/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts @@ -1,63 +1,56 @@ /** - * v1.12.2 ios-coord — TLS-pin-monitor queue registration guard. + * TLS-pin-monitor queue registration guard. * - * Same source-text-grep approach as the other queue-wiring guards: assert - * the queue is imported, registered in `allQueues`, scheduled, and wired to - * a `createAndWork` handler — without booting pg-boss + Prisma. An unregistered - * queue silently never drains (the recurring v1.4.37 dead-queue bug), which - * would leave the pinned-leaf-rotation alarm dark. + * Reads the worker registry: the queue is declared in the maintenance group, + * scheduled on its cron with the default retries, and bound to the handler + * that runs the monitor pass. An unregistered queue silently never drains + * (the dead-queue class), which would leave the pinned-leaf-rotation alarm + * dark. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the TLS-pin-monitor wiring moved out of the 2143-LOC -// reminder-worker boot file into the maintenance registrar. The dead-queue -// guard follows the wiring there (the ops-handlers module is still concatenated -// for the handler-body `runTlsPinMonitor` assertion). -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-maintenance.ts", +import { handleTlsPinMonitor } from "@/lib/jobs/reminder/ops-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { + TLS_PIN_MONITOR_CRON, + TLS_PIN_MONITOR_QUEUE, +} from "@/lib/jobs/tls-pin-monitor-queue"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "ops-handlers.ts"), + "utf8", ); -const workerSource = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync(join(__dirname, "..", "reminder", "ops-handlers.ts"), "utf8"); -describe("reminder-worker — tls-pin-monitor wiring", () => { - it("imports the queue symbols from the tls-pin-monitor module", () => { - expect(workerSource).toMatch( - /from\s*["']@\/lib\/jobs\/tls-pin-monitor["']/, - ); - expect(workerSource).toMatch(/\bTLS_PIN_MONITOR_QUEUE\b/); - expect(workerSource).toMatch(/\bTLS_PIN_MONITOR_CRON\b/); - expect(workerSource).toMatch(/\brunTlsPinMonitor\b/); +describe("tls-pin-monitor registration", () => { + it("imports the monitor pass into the ops handlers", () => { + expect(handlerSource).toMatch(/\brunTlsPinMonitor\b/); }); - it("registers the tls-pin-monitor queue in the allQueues loop", () => { - const allQueuesMatch = workerSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bTLS_PIN_MONITOR_QUEUE\b/); + it("declares the tls-pin-monitor queue in the maintenance group", () => { + expect(groupOf(TLS_PIN_MONITOR_QUEUE).name).toBe("maintenance"); }); - it("schedules the tls-pin-monitor cron", () => { - expect(workerSource).toMatch( - /\[TLS_PIN_MONITOR_QUEUE,\s*TLS_PIN_MONITOR_CRON\]/, - ); + it("schedules the tls-pin-monitor cron with the default retries", () => { + const job = jobOf(TLS_PIN_MONITOR_QUEUE); + expect(job.cron).toBe(TLS_PIN_MONITOR_CRON); + expect(job.scheduleOptions).toBeUndefined(); }); - it("registers a createAndWork handler for the queue", () => { - expect(workerSource).toMatch( - /createAndWork[\s\S]{0,200}TLS_PIN_MONITOR_QUEUE[\s\S]{0,200}handleTlsPinMonitor/, + it("binds the monitor handler", async () => { + expect(await loadedHandler(TLS_PIN_MONITOR_QUEUE)).toBe( + handleTlsPinMonitor, ); }); it("runs the monitor pass inside the handler", () => { - expect(workerSource).toMatch( + expect(handlerSource).toMatch( /handleTlsPinMonitor[\s\S]{0,400}runTlsPinMonitor/, ); }); diff --git a/src/lib/jobs/__tests__/whoop-queues.test.ts b/src/lib/jobs/__tests__/whoop-queues.test.ts index a1ad468b4..cdea7c9f2 100644 --- a/src/lib/jobs/__tests__/whoop-queues.test.ts +++ b/src/lib/jobs/__tests__/whoop-queues.test.ts @@ -1,113 +1,126 @@ /** - * v1.11.0 — pg-boss queue registration for the WHOOP sync layer. + * pg-boss queue registration for the WHOOP sync layer. * - * The reminder-worker imports heavy infrastructure (pg-boss, Prisma adapter, - * notification dispatchers, …) so we don't boot the whole worker here. Instead - * we read the source as text and assert the queue constants + cron schedules + - * handler wiring + allQueues + boot-enqueue are present — the fast guard that - * catches the v1.4.37 dead-queue class (a queue declared but never registered - * in `allQueues`, which silently never drains). + * Reads the worker registry instead of booting the worker: every WHOOP queue + * is declared in the integration-sync group with its cron and handler, the + * backfill forwards to the shared admission lane, and the boot discovery + * offers it again. This is the fast guard for the dead-queue class (a queue + * declared but never registered, which silently never drains). */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring + boot discovery moved out of the 2143-LOC -// reminder-worker boot file into domain registrars; the WHOOP queues live in -// the integration-sync registrar. The dead-queue guard follows the wiring there -// (the handler module is still concatenated for the import-routine assertions). -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-integration-sync.ts", -); -const source = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync(join(__dirname, "..", "reminder", "whoop-sync.ts"), "utf8"); +import { INTEGRATION_BACKFILL_ADMISSION_QUEUE } from "@/lib/jobs/integration-backfill-admission"; +import { + WHOOP_CYCLE_SYNC_CRON, + WHOOP_CYCLE_SYNC_QUEUE, + WHOOP_OAUTH_STATE_CLEANUP_CRON, + WHOOP_OAUTH_STATE_CLEANUP_QUEUE, + WHOOP_RECOVERY_SYNC_CRON, + WHOOP_RECOVERY_SYNC_QUEUE, + WHOOP_SLEEP_SYNC_CRON, + WHOOP_SLEEP_SYNC_QUEUE, + WHOOP_WORKOUT_SYNC_CRON, + WHOOP_WORKOUT_SYNC_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + handleWhoopCycleSync, + handleWhoopRecoverySync, + handleWhoopSleepSync, + handleWhoopWorkoutSync, +} from "@/lib/jobs/reminder/whoop-sync"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { WHOOP_BACKFILL_QUEUE } from "@/lib/jobs/whoop-backfill-queue"; -const WHOOP_QUEUE_CONSTS = [ - "WHOOP_RECOVERY_SYNC_QUEUE", - "WHOOP_SLEEP_SYNC_QUEUE", - "WHOOP_WORKOUT_SYNC_QUEUE", - "WHOOP_CYCLE_SYNC_QUEUE", - "WHOOP_BACKFILL_QUEUE", - "WHOOP_OAUTH_STATE_CLEANUP_QUEUE", +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "whoop-sync.ts"), + "utf8", +); + +const WHOOP_QUEUES = [ + WHOOP_RECOVERY_SYNC_QUEUE, + WHOOP_SLEEP_SYNC_QUEUE, + WHOOP_WORKOUT_SYNC_QUEUE, + WHOOP_CYCLE_SYNC_QUEUE, + WHOOP_BACKFILL_QUEUE, + WHOOP_OAUTH_STATE_CLEANUP_QUEUE, ] as const; -describe("reminder-worker — WHOOP sync queues", () => { +describe("WHOOP sync queues", () => { it("declares every WHOOP sync queue constant", () => { - expect(source).toMatch( - /WHOOP_RECOVERY_SYNC_QUEUE\s*=\s*["']whoop-recovery-sync["']/, - ); - expect(source).toMatch( - /WHOOP_SLEEP_SYNC_QUEUE\s*=\s*["']whoop-sleep-sync["']/, - ); - expect(source).toMatch( - /WHOOP_WORKOUT_SYNC_QUEUE\s*=\s*["']whoop-workout-sync["']/, - ); - expect(source).toMatch( - /WHOOP_CYCLE_SYNC_QUEUE\s*=\s*["']whoop-cycle-sync["']/, - ); + expect(WHOOP_RECOVERY_SYNC_QUEUE).toBe("whoop-recovery-sync"); + expect(WHOOP_SLEEP_SYNC_QUEUE).toBe("whoop-sleep-sync"); + expect(WHOOP_WORKOUT_SYNC_QUEUE).toBe("whoop-workout-sync"); + expect(WHOOP_CYCLE_SYNC_QUEUE).toBe("whoop-cycle-sync"); }); - it("registers EVERY WHOOP queue in allQueues (v1.4.37 dead-queue guard)", () => { - // Isolate the allQueues array body so the assertion can't be satisfied by - // an incidental mention elsewhere in the file. - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - const body = block![1]!; - for (const q of WHOOP_QUEUE_CONSTS) { - expect(body).toContain(q); + it("declares EVERY WHOOP queue in the integration-sync group", () => { + for (const q of WHOOP_QUEUES) { + expect(groupOf(q).name).toBe("integration_sync"); } }); it("schedules the four poll crons + the daily oauth-state sweep", () => { - expect(source).toMatch( - /\[WHOOP_RECOVERY_SYNC_QUEUE,\s*WHOOP_RECOVERY_SYNC_CRON\]/, - ); - expect(source).toMatch( - /\[WHOOP_SLEEP_SYNC_QUEUE,\s*WHOOP_SLEEP_SYNC_CRON\]/, - ); - expect(source).toMatch( - /\[WHOOP_WORKOUT_SYNC_QUEUE,\s*WHOOP_WORKOUT_SYNC_CRON\]/, - ); - expect(source).toMatch( - /\[WHOOP_CYCLE_SYNC_QUEUE,\s*WHOOP_CYCLE_SYNC_CRON\]/, - ); - expect(source).toMatch( - /\[\s*WHOOP_OAUTH_STATE_CLEANUP_QUEUE,\s*WHOOP_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, + expect(jobOf(WHOOP_RECOVERY_SYNC_QUEUE).cron).toBe( + WHOOP_RECOVERY_SYNC_CRON, ); + expect(jobOf(WHOOP_SLEEP_SYNC_QUEUE).cron).toBe(WHOOP_SLEEP_SYNC_CRON); + expect(jobOf(WHOOP_WORKOUT_SYNC_QUEUE).cron).toBe(WHOOP_WORKOUT_SYNC_CRON); + expect(jobOf(WHOOP_CYCLE_SYNC_QUEUE).cron).toBe(WHOOP_CYCLE_SYNC_CRON); + const sweep = jobOf(WHOOP_OAUTH_STATE_CLEANUP_QUEUE); + expect(sweep.cron).toBe(WHOOP_OAUTH_STATE_CLEANUP_CRON); + expect(sweep.scheduleOptions).toBe(cronIsTheRetry); }); - it("registers a createAndWork handler for every WHOOP sync queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,160}WHOOP_RECOVERY_SYNC_QUEUE[\s\S]{0,160}handleWhoopRecoverySync/, + it("binds a handler for every WHOOP sync queue", async () => { + expect(await loadedHandler(WHOOP_RECOVERY_SYNC_QUEUE)).toBe( + handleWhoopRecoverySync, ); - expect(source).toMatch( - /createAndWork[\s\S]{0,160}WHOOP_SLEEP_SYNC_QUEUE[\s\S]{0,160}handleWhoopSleepSync/, + expect(await loadedHandler(WHOOP_SLEEP_SYNC_QUEUE)).toBe( + handleWhoopSleepSync, ); - expect(source).toMatch( - /createAndWork[\s\S]{0,160}WHOOP_WORKOUT_SYNC_QUEUE[\s\S]{0,160}handleWhoopWorkoutSync/, + expect(await loadedHandler(WHOOP_WORKOUT_SYNC_QUEUE)).toBe( + handleWhoopWorkoutSync, ); - expect(source).toMatch( - /createAndWork[\s\S]{0,160}WHOOP_CYCLE_SYNC_QUEUE[\s\S]{0,160}handleWhoopCycleSync/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}WHOOP_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, + expect(await loadedHandler(WHOOP_CYCLE_SYNC_QUEUE)).toBe( + handleWhoopCycleSync, ); + const { boss, sends } = recordingBoss(); + const backfill = await jobOf(WHOOP_BACKFILL_QUEUE).load({ boss }); + await backfill([ + { + id: "j1", + name: WHOOP_BACKFILL_QUEUE, + data: { userId: "u1" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + expect(sends).toHaveLength(1); + expect(sends[0]!.queue).toBe(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(sends[0]!.data).toMatchObject({ kind: "whoop-backfill" }); }); it("wires the self-converging WHOOP backfill boot discovery", () => { - expect(source).toMatch( - /enqueueBootTimeWhoopBackfill\(\s*bootStaggerSecondsFor\("whoop-backfill"\)/, - ); + expect(discoveryOf("integration_sync")).toContain("whoop-backfill"); }); it("imports the WHOOP per-resource sync routines", () => { - expect(source).toMatch(/from\s*["']@\/lib\/whoop\/sync-recovery["']/); - expect(source).toMatch(/from\s*["']@\/lib\/whoop\/sync-cycle["']/); - expect(source).toMatch(/from\s*["']@\/lib\/whoop\/sync-workout["']/); + expect(handlerSource).toMatch( + /from\s*["']@\/lib\/whoop\/sync-recovery["']/, + ); + expect(handlerSource).toMatch(/from\s*["']@\/lib\/whoop\/sync-cycle["']/); + expect(handlerSource).toMatch(/from\s*["']@\/lib\/whoop\/sync-workout["']/); }); }); diff --git a/src/lib/jobs/__tests__/withings-ecg-worker.test.ts b/src/lib/jobs/__tests__/withings-ecg-worker.test.ts index 457a24121..73a2358bd 100644 --- a/src/lib/jobs/__tests__/withings-ecg-worker.test.ts +++ b/src/lib/jobs/__tests__/withings-ecg-worker.test.ts @@ -1,7 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { Job } from "pg-boss"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; const { syncUserEcg } = vi.hoisted(() => ({ syncUserEcg: vi.fn< @@ -28,7 +26,8 @@ vi.mock("@/lib/withings/sync-activity", () => ({ vi.mock("@/lib/withings/sync-sleep", () => ({ syncUserSleep: vi.fn(async () => 0), })); -vi.mock("@/lib/jobs/reminder-satisfy", () => ({ +vi.mock("@/lib/jobs/reminder-satisfy", async (importOriginal) => ({ + ...(await importOriginal()), enqueueReminderSatisfy: vi.fn(async () => undefined), })); vi.mock("@/lib/jobs/worker-status", () => ({ @@ -72,6 +71,13 @@ import { handleWithingsEcgSync, type WithingsEcgSyncPayload, } from "../reminder/withings-sync"; +import { WITHINGS_ECG_SYNC_CRON } from "../integration-poll-queues"; +import { + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; +import { WITHINGS_ECG_SYNC_QUEUE } from "../withings-ecg-queue"; function job(data: WithingsEcgSyncPayload): Job { return { data } as Job; @@ -82,10 +88,6 @@ beforeEach(() => { ecgFindMany.mockResolvedValue([]); isReauthRequired.mockResolvedValue(false); }); -const registrarSource = readFileSync( - join(process.cwd(), "src/lib/jobs/reminder/register-integration-sync.ts"), - "utf8", -); describe("handleWithingsEcgSync", () => { it("a webhook job (userId + window) does a targeted date-windowed sync", async () => { @@ -176,17 +178,13 @@ describe("handleWithingsEcgSync", () => { }); describe("withings ECG queue registration", () => { - it("deduplicates only queued replays and admits a rescue after work starts", () => { - const allQueues = registrarSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bWITHINGS_ECG_SYNC_QUEUE\b/); - expect(registrarSource).toMatch( - /\[WITHINGS_ECG_SYNC_QUEUE\]:\s*\{[\s\S]*?policy:\s*"short"/, - ); - expect(registrarSource).toMatch( - /createAndWork\(\s*boss,\s*WITHINGS_ECG_SYNC_QUEUE[\s\S]{0,120}handleWithingsEcgSync/, + it("deduplicates only queued replays and admits a rescue after work starts", async () => { + expect(groupOf(WITHINGS_ECG_SYNC_QUEUE).name).toBe("integration_sync"); + const definition = jobOf(WITHINGS_ECG_SYNC_QUEUE); + expect(definition.policy?.policy).toBe("short"); + expect(definition.cron).toBe(WITHINGS_ECG_SYNC_CRON); + expect(await loadedHandler(WITHINGS_ECG_SYNC_QUEUE)).toBe( + handleWithingsEcgSync, ); }); }); diff --git a/src/lib/jobs/__tests__/withings-queues.test.ts b/src/lib/jobs/__tests__/withings-queues.test.ts index 8a5bd52ec..0cd5a286b 100644 --- a/src/lib/jobs/__tests__/withings-queues.test.ts +++ b/src/lib/jobs/__tests__/withings-queues.test.ts @@ -1,90 +1,83 @@ /** - * v1.4.25 — pg-boss queue registration for Withings activity + - * sleep v2 syncs. + * pg-boss queue registration for Withings activity + sleep v2 syncs. * - * The reminder-worker imports heavy infrastructure (pg-boss, Prisma - * adapter, notification dispatchers, …) so we don't boot the whole - * worker in this test. Instead we read the source file as text and - * assert the queue constants + cron schedules + handler wiring are - * present — a fast regression guard that catches accidental deletion - * of the W17b/c plumbing without spinning up Postgres. + * Reads the worker registry instead of booting the worker (pg-boss, Prisma + * adapter, notification dispatchers, …): the queues are declared with their + * crons and bound to their handlers, a fast regression guard against + * accidental deletion of the plumbing without spinning up Postgres. The + * import-routine assertions read the handler module. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring moved out of the 2143-LOC reminder-worker boot -// file into domain registrars; the Withings queues live in the integration-sync -// registrar. The dead-queue guard follows the wiring there (the handler module -// is still concatenated for the import-routine assertions below). -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-integration-sync.ts", +import { + WITHINGS_ACTIVITY_CRON, + WITHINGS_ACTIVITY_QUEUE, + WITHINGS_ECG_SYNC_CRON, + WITHINGS_SLEEP_CRON, + WITHINGS_SLEEP_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; +import { + handleWithingsActivitySync, + handleWithingsSleepSync, +} from "@/lib/jobs/reminder/withings-sync"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { WITHINGS_ECG_SYNC_QUEUE } from "@/lib/jobs/withings-ecg-queue"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "withings-sync.ts"), + "utf8", ); -const source = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync(join(__dirname, "..", "reminder", "withings-sync.ts"), "utf8"); - -describe("reminder-worker — Withings activity + sleep v2 queues", () => { +describe("Withings activity + sleep v2 queues", () => { it("declares the activity-sync queue at the documented cadence", () => { - expect(source).toMatch( - /WITHINGS_ACTIVITY_QUEUE\s*=\s*["']withings-activity-sync["']/, - ); - expect(source).toMatch( - /WITHINGS_ACTIVITY_CRON\s*=\s*["']0 \* \* \* \*["']/, - ); + expect(WITHINGS_ACTIVITY_QUEUE).toBe("withings-activity-sync"); + expect(WITHINGS_ACTIVITY_CRON).toBe("0 * * * *"); }); it("declares the sleep-sync queue at the offset cadence", () => { - expect(source).toMatch( - /WITHINGS_SLEEP_QUEUE\s*=\s*["']withings-sleep-sync["']/, - ); + expect(WITHINGS_SLEEP_QUEUE).toBe("withings-sleep-sync"); // :15 offset keeps it out of lockstep with the measure cron at :00. - expect(source).toMatch(/WITHINGS_SLEEP_CRON\s*=\s*["']15 \* \* \* \*["']/); + expect(WITHINGS_SLEEP_CRON).toBe("15 * * * *"); }); - it("registers the activity-sync handler against pg-boss", () => { - expect(source).toMatch(/handleWithingsActivitySync\b/); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}WITHINGS_ACTIVITY_QUEUE[\s\S]{0,200}handleWithingsActivitySync/, + it("binds the activity-sync handler", async () => { + expect(groupOf(WITHINGS_ACTIVITY_QUEUE).name).toBe("integration_sync"); + expect(await loadedHandler(WITHINGS_ACTIVITY_QUEUE)).toBe( + handleWithingsActivitySync, ); }); - it("registers the sleep-sync handler against pg-boss", () => { - expect(source).toMatch(/handleWithingsSleepSync\b/); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}WITHINGS_SLEEP_QUEUE[\s\S]{0,200}handleWithingsSleepSync/, + it("binds the sleep-sync handler", async () => { + expect(groupOf(WITHINGS_SLEEP_QUEUE).name).toBe("integration_sync"); + expect(await loadedHandler(WITHINGS_SLEEP_QUEUE)).toBe( + handleWithingsSleepSync, ); }); - it("imports the new activity / sleep sync routines", () => { - expect(source).toMatch( + it("imports the activity / sleep sync routines", () => { + expect(handlerSource).toMatch( /import\s*\{\s*syncUserActivity\s*\}\s*from\s*["']@\/lib\/withings\/sync-activity["']/, ); - expect(source).toMatch( + expect(handlerSource).toMatch( /import\s*\{\s*syncUserSleep\s*\}\s*from\s*["']@\/lib\/withings\/sync-sleep["']/, ); }); - it("schedules both crons via boss.schedule (allQueues + schedules tables)", () => { - expect(source).toMatch( - /\[WITHINGS_ACTIVITY_QUEUE,\s*WITHINGS_ACTIVITY_CRON\]/, - ); - expect(source).toMatch(/\[WITHINGS_SLEEP_QUEUE,\s*WITHINGS_SLEEP_CRON\]/); + it("schedules both crons", () => { + expect(jobOf(WITHINGS_ACTIVITY_QUEUE).cron).toBe(WITHINGS_ACTIVITY_CRON); + expect(jobOf(WITHINGS_SLEEP_QUEUE).cron).toBe(WITHINGS_SLEEP_CRON); }); it("schedules the ECG catch-net cron at :41 so watch-only accounts pull strips", () => { - // The queue is already in allQueues + createAndWork-bound; W-1 adds the missing - // schedule entry so a payload-less cron tick walks the connection cohort. - expect(source).toMatch( - /WITHINGS_ECG_SYNC_CRON\s*=\s*["']41 \* \* \* \*["']/, - ); - expect(source).toMatch( - /\[WITHINGS_ECG_SYNC_QUEUE,\s*WITHINGS_ECG_SYNC_CRON\]/, - ); + // A payload-less cron tick walks the connection cohort. + expect(WITHINGS_ECG_SYNC_CRON).toBe("41 * * * *"); + expect(jobOf(WITHINGS_ECG_SYNC_QUEUE).cron).toBe(WITHINGS_ECG_SYNC_CRON); }); }); diff --git a/src/lib/jobs/__tests__/workout-insight-queue.test.ts b/src/lib/jobs/__tests__/workout-insight-queue.test.ts index 7af32f112..ce17b76df 100644 --- a/src/lib/jobs/__tests__/workout-insight-queue.test.ts +++ b/src/lib/jobs/__tests__/workout-insight-queue.test.ts @@ -1,40 +1,41 @@ /** - * v1.4.37 dead-queue guard for the `workout-insight-generate` queue. + * Dead-queue guard for the `workout-insight-generate` queue. * * This queue fails open in the worst possible way. Nothing warms it, no cron * touches it, and no read path triggers it — the ONLY thing that ever puts work - * on it is the arrival spine's dispatch. So if the queue name is missing from - * `allQueues`, pg-boss never provisions it, every `boss.send` resolves + * on it is the arrival spine's dispatch. So if the registry does not declare + * the queue, pg-boss never provisions it, every `boss.send` resolves * successfully, every job vanishes, and the only symptom is that no workout * ever gets a paragraph. That is indistinguishable from the feature's own * honest empty state, which is exactly why it needs a structural guard rather * than a behavioural one. * - * Source-text assertions over the registrar, matching the shape of the sibling - * `*-queue.test.ts` guards. + * Reads the worker registry for the queue facts. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -const statusRegistrar = readFileSync( - join(__dirname, "..", "reminder", "register-status.ts"), - "utf8", -); +import { handleWorkoutInsightGenerate } from "@/lib/jobs/workout-insight-generate"; +import { + WORKOUT_INSIGHT_GENERATE_CONCURRENCY, + WORKOUT_INSIGHT_GENERATE_QUEUE, +} from "@/lib/jobs/workout-insight-generate-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; describe("workout-insight-generate queue wiring", () => { - it("provisions the queue in the allQueues list", () => { - const allQueues = statusRegistrar.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bWORKOUT_INSIGHT_GENERATE_QUEUE\b/); + it("provisions the queue in the status group", () => { + expect(groupOf(WORKOUT_INSIGHT_GENERATE_QUEUE).name).toBe("status"); }); - it("binds a createAndWork handler that drains it", () => { - expect(statusRegistrar).toMatch( - /createAndWork[\s\S]{0,300}WORKOUT_INSIGHT_GENERATE_QUEUE[\s\S]{0,300}handleWorkoutInsightGenerate/, + it("binds the handler that drains it", async () => { + expect(await loadedHandler(WORKOUT_INSIGHT_GENERATE_QUEUE)).toBe( + handleWorkoutInsightGenerate, ); }); @@ -42,19 +43,17 @@ describe("workout-insight-generate queue wiring", () => { // This IS the provider path the spine deliberately does not walk. Widening // the concurrency would let one sync's worth of workouts fan out into // parallel completions. - expect(statusRegistrar).toMatch( - /WORKOUT_INSIGHT_GENERATE_QUEUE[\s\S]{0,300}localConcurrency:\s*WORKOUT_INSIGHT_GENERATE_CONCURRENCY/, - ); + expect(jobOf(WORKOUT_INSIGHT_GENERATE_QUEUE).work).toEqual({ + localConcurrency: WORKOUT_INSIGHT_GENERATE_CONCURRENCY, + }); }); it("has no cron schedule — it is dispatched by arrival only", () => { // A cron here would be a standing invitation to regenerate, which is the // saturation surface the design refuses. - // Tolerates the optional third schedule element, so a cron sneaking in - // with send options still trips this. - expect(statusRegistrar).not.toMatch( - /\[\s*WORKOUT_INSIGHT_GENERATE_QUEUE,\s*\w+_CRON\s*[,\]]/, - ); + const job = jobOf(WORKOUT_INSIGHT_GENERATE_QUEUE); + expect(job.cron).toBeUndefined(); + expect(job.scheduleOptions).toBeUndefined(); }); it("is reached from the spine only through the generator-free module", () => { diff --git a/src/lib/jobs/achievement-unlock-sweep-queue.ts b/src/lib/jobs/achievement-unlock-sweep-queue.ts new file mode 100644 index 000000000..4b64470b9 --- /dev/null +++ b/src/lib/jobs/achievement-unlock-sweep-queue.ts @@ -0,0 +1,8 @@ +/** + * The queue facts of `achievement-unlock-sweep.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const ACHIEVEMENT_UNLOCK_SWEEP_QUEUE = "achievement-unlock-sweep"; + +export const ACHIEVEMENT_UNLOCK_SWEEP_CRON = "25 4 * * *"; diff --git a/src/lib/jobs/achievement-unlock-sweep.ts b/src/lib/jobs/achievement-unlock-sweep.ts index 0a4a12685..190a17189 100644 --- a/src/lib/jobs/achievement-unlock-sweep.ts +++ b/src/lib/jobs/achievement-unlock-sweep.ts @@ -31,10 +31,6 @@ import { resolveModuleMap } from "@/lib/modules/gate"; import { buildAchievementsResult } from "@/lib/gamification/achievements-result"; import { getWorkerPrisma } from "./reminder/shared"; -export const ACHIEVEMENT_UNLOCK_SWEEP_QUEUE = "achievement-unlock-sweep"; - -export const ACHIEVEMENT_UNLOCK_SWEEP_CRON = "25 4 * * *"; - export interface AchievementUnlockSweepPayload { triggeredAt: string; } diff --git a/src/lib/jobs/apple-health-import-queue.ts b/src/lib/jobs/apple-health-import-queue.ts new file mode 100644 index 000000000..e862c9b7d --- /dev/null +++ b/src/lib/jobs/apple-health-import-queue.ts @@ -0,0 +1,36 @@ +/** + * The queue facts of `apple-health-import-worker.ts`: its queue names and + * concurrency. They live apart from the worker so the registry and the upload + * routes can name the queue without loading the parser. + */ +/** + * Queue + cron for the periodic orphan-ImportJob sweep. v1.32.1 + * (issue #588) — `reconcileOrphanImportJobs()` used to run only once, + * at worker boot. `restart: unless-stopped` in `docker-compose.yml` + * DOES bring a crashed/OOM-killed worker back up, so the boot-time + * pass usually catches a dead run — but only when that restart lands + * AFTER the row's heartbeat has already gone stale (30 minutes) or + * pg-boss no longer reports the backing job as live. A fast restart + * lands neither condition true yet, and because the pass runs + * exactly once per boot, a worker that then stays up and healthy + * NEVER re-evaluates that row — it is stuck in `unpacking` / + * `parsing` / `upserting` with "0 rows imported" forever and no + * failure ever surfaced to the status poll. A 15-minute cron closes + * that gap: within two ticks of the heartbeat going stale, the same + * idempotent reconcile flips the row to `failed` with an honest + * reason, so the UI's "Unpacking the archive…" spinner eventually + * turns into a visible, actionable error instead of an indefinite + * silent stall. + */ +export const IMPORT_JOB_RECONCILE_QUEUE = "apple-health-import-reconcile"; + +export const IMPORT_JOB_RECONCILE_CRON = "*/15 * * * *"; + +/** Queue name isolated from revision-1 workers during rolling deployments. */ +export const APPLE_HEALTH_IMPORT_V2_QUEUE = "apple-health-import-v2"; + +/** Queue used by pre-revision-2 binaries; new workers only bridge its backlog. */ +export const APPLE_HEALTH_IMPORT_LEGACY_QUEUE = "apple-health-import"; + +/** Concurrency cap per worker host. */ +export const APPLE_HEALTH_IMPORT_CONCURRENCY = 1; diff --git a/src/lib/jobs/apple-health-import-worker.ts b/src/lib/jobs/apple-health-import-worker.ts index f2cbfe0d9..01a3e3649 100644 --- a/src/lib/jobs/apple-health-import-worker.ts +++ b/src/lib/jobs/apple-health-import-worker.ts @@ -55,10 +55,12 @@ import { type StagingInUse, } from "@/lib/import/apple-health-staging"; import { caughtAs, logCaught } from "@/lib/logging/signal"; +import { HEALTH_CONNECT_IMPORT_KIND } from "@/lib/jobs/health-connect-import-worker"; +import { HEALTH_CONNECT_IMPORT_QUEUE } from "@/lib/jobs/health-connect-import-queue"; import { - HEALTH_CONNECT_IMPORT_KIND, - HEALTH_CONNECT_IMPORT_QUEUE, -} from "@/lib/jobs/health-connect-import-worker"; + APPLE_HEALTH_IMPORT_V2_QUEUE, + APPLE_HEALTH_IMPORT_LEGACY_QUEUE, +} from "./apple-health-import-queue"; /** `ImportJob.kind` of the rows this worker owns. */ export const APPLE_HEALTH_IMPORT_KIND = "apple_health"; @@ -76,28 +78,6 @@ function queuesForKind(kind: string): readonly string[] { : [APPLE_HEALTH_IMPORT_V2_QUEUE, APPLE_HEALTH_IMPORT_LEGACY_QUEUE]; } -/** - * Queue + cron for the periodic orphan-ImportJob sweep. v1.32.1 - * (issue #588) — `reconcileOrphanImportJobs()` used to run only once, - * at worker boot. `restart: unless-stopped` in `docker-compose.yml` - * DOES bring a crashed/OOM-killed worker back up, so the boot-time - * pass usually catches a dead run — but only when that restart lands - * AFTER the row's heartbeat has already gone stale (30 minutes) or - * pg-boss no longer reports the backing job as live. A fast restart - * lands neither condition true yet, and because the pass runs - * exactly once per boot, a worker that then stays up and healthy - * NEVER re-evaluates that row — it is stuck in `unpacking` / - * `parsing` / `upserting` with "0 rows imported" forever and no - * failure ever surfaced to the status poll. A 15-minute cron closes - * that gap: within two ticks of the heartbeat going stale, the same - * idempotent reconcile flips the row to `failed` with an honest - * reason, so the UI's "Unpacking the archive…" spinner eventually - * turns into a visible, actionable error instead of an indefinite - * silent stall. - */ -export const IMPORT_JOB_RECONCILE_QUEUE = "apple-health-import-reconcile"; -export const IMPORT_JOB_RECONCILE_CRON = "*/15 * * * *"; - /** * Cron handler for the periodic orphan-ImportJob sweep. Delegates to * the same `reconcileOrphanImportJobs()` the boot path calls — the @@ -145,12 +125,6 @@ export async function handleImportJobReconcileTick( ); } -/** Queue name isolated from revision-1 workers during rolling deployments. */ -export const APPLE_HEALTH_IMPORT_V2_QUEUE = "apple-health-import-v2"; - -/** Queue used by pre-revision-2 binaries; new workers only bridge its backlog. */ -export const APPLE_HEALTH_IMPORT_LEGACY_QUEUE = "apple-health-import"; - /** Parser semantics carried by every newly-created ImportJob. */ export const APPLE_HEALTH_IMPORT_PARSER_REVISION = 3; @@ -163,9 +137,6 @@ const APPLE_HEALTH_ECG_ARCHIVE_LIMITS = { } as const; const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000; -/** Concurrency cap per worker host. */ -export const APPLE_HEALTH_IMPORT_CONCURRENCY = 1; - /** * (issue #775) Memory preflight for the parse. The streaming pipeline * keeps per-record work off the heap, so what remains resident scales diff --git a/src/lib/jobs/backup-restore-queue.ts b/src/lib/jobs/backup-restore-queue.ts new file mode 100644 index 000000000..7993df548 --- /dev/null +++ b/src/lib/jobs/backup-restore-queue.ts @@ -0,0 +1,6 @@ +/** + * The queue facts of `backup-restore.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const BACKUP_RESTORE_QUEUE = "backup-restore"; diff --git a/src/lib/jobs/backup-restore.ts b/src/lib/jobs/backup-restore.ts index e17aa8865..ab721743e 100644 --- a/src/lib/jobs/backup-restore.ts +++ b/src/lib/jobs/backup-restore.ts @@ -78,8 +78,7 @@ import { type RestoreFailureCause, } from "@/lib/export/restore-failure-cause"; import { logCaught } from "@/lib/logging/signal"; - -export const BACKUP_RESTORE_QUEUE = "backup-restore"; +import { BACKUP_RESTORE_QUEUE } from "./backup-restore-queue"; /** Stable code of the 409 a second restore of the same account gets. */ export const BACKUP_RESTORE_ACTIVE_CODE = "backup.restore.active"; diff --git a/src/lib/jobs/coach-nudge-queue.ts b/src/lib/jobs/coach-nudge-queue.ts new file mode 100644 index 000000000..bef4d8ac7 --- /dev/null +++ b/src/lib/jobs/coach-nudge-queue.ts @@ -0,0 +1,10 @@ +/** + * The queue facts of `coach-nudge.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +/** Pg-boss queue + cron — imported by the reminder worker's bootstrap. */ +export const COACH_NUDGE_QUEUE = "coach-nudge"; + +/** 05:15 Europe/Berlin — after the 04:45–04:55 score crons settled. */ +export const COACH_NUDGE_CRON = "15 5 * * *"; diff --git a/src/lib/jobs/coach-nudge.ts b/src/lib/jobs/coach-nudge.ts index f8fb77cc5..b092ce85e 100644 --- a/src/lib/jobs/coach-nudge.ts +++ b/src/lib/jobs/coach-nudge.ts @@ -121,11 +121,6 @@ const SELF_CONTEXT_AI_SECTIONS = [ "COACH_FOCUS", ] as const satisfies readonly HealthProfileAiSection[]; -/** Pg-boss queue + cron — imported by the reminder worker's bootstrap. */ -export const COACH_NUDGE_QUEUE = "coach-nudge"; -/** 05:15 Europe/Berlin — after the 04:45–04:55 score crons settled. */ -export const COACH_NUDGE_CRON = "15 5 * * *"; - /** * Frequency cap default: one nudge per user per rolling week. v1.16.5 — * the per-user `nudgeFrequency` pref ("weekly" | "biweekly") can widen diff --git a/src/lib/jobs/coach-reminder-sweep-queue.ts b/src/lib/jobs/coach-reminder-sweep-queue.ts new file mode 100644 index 000000000..de3a9240e --- /dev/null +++ b/src/lib/jobs/coach-reminder-sweep-queue.ts @@ -0,0 +1,10 @@ +/** + * The queue facts of `coach-reminder-sweep.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const COACH_REMINDER_SWEEP_QUEUE = "coach-reminder-sweep"; + +// Daily at 05:20 Europe/Berlin — just after the 05:15 nudge tick so both +// proactive passes run on settled overnight rows. +export const COACH_REMINDER_SWEEP_CRON = "20 5 * * *"; diff --git a/src/lib/jobs/coach-reminder-sweep.ts b/src/lib/jobs/coach-reminder-sweep.ts index dab2112eb..fb47ce4b5 100644 --- a/src/lib/jobs/coach-reminder-sweep.ts +++ b/src/lib/jobs/coach-reminder-sweep.ts @@ -71,11 +71,6 @@ import { surfaceCoachReminders, } from "@/lib/ai/coach/reminder-surface"; -export const COACH_REMINDER_SWEEP_QUEUE = "coach-reminder-sweep"; -// Daily at 05:20 Europe/Berlin — just after the 05:15 nudge tick so both -// proactive passes run on settled overnight rows. -export const COACH_REMINDER_SWEEP_CRON = "20 5 * * *"; - /** Bound the plan-review fan-out per tick so one sweep can't run unbounded. */ const PLAN_REVIEW_BATCH = 200; /** Bound the surfacing fan-out per tick for the same reason. */ diff --git a/src/lib/jobs/compaction-tombstone-purge-queue.ts b/src/lib/jobs/compaction-tombstone-purge-queue.ts new file mode 100644 index 000000000..82b7b775e --- /dev/null +++ b/src/lib/jobs/compaction-tombstone-purge-queue.ts @@ -0,0 +1,6 @@ +/** + * The queue facts of `compaction-tombstone-purge.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const COMPACTION_TOMBSTONE_PURGE_QUEUE = "compaction-tombstone-purge"; diff --git a/src/lib/jobs/compaction-tombstone-purge.ts b/src/lib/jobs/compaction-tombstone-purge.ts index 415eee0dc..a43c080da 100644 --- a/src/lib/jobs/compaction-tombstone-purge.ts +++ b/src/lib/jobs/compaction-tombstone-purge.ts @@ -67,8 +67,7 @@ import { findCompactionTombstones, } from "@/lib/measurements/folded-window"; import { prisma as defaultPrisma } from "@/lib/db"; - -export const COMPACTION_TOMBSTONE_PURGE_QUEUE = "compaction-tombstone-purge"; +import { COMPACTION_TOMBSTONE_PURGE_QUEUE } from "./compaction-tombstone-purge-queue"; /** One key for every send, so a queued run absorbs a second boot's send. */ export const COMPACTION_TOMBSTONE_PURGE_SINGLETON_KEY = diff --git a/src/lib/jobs/daily-briefing-queue.ts b/src/lib/jobs/daily-briefing-queue.ts new file mode 100644 index 000000000..95948809e --- /dev/null +++ b/src/lib/jobs/daily-briefing-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `daily-briefing.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DAILY_BRIEFING_QUEUE = "daily-briefing"; + +/** Every 15 minutes; the per-user local-hour gate selects the fallback slot. */ +export const DAILY_BRIEFING_CRON = "*/15 * * * *"; diff --git a/src/lib/jobs/daily-briefing.ts b/src/lib/jobs/daily-briefing.ts index 522cb29c0..f8a50b9d1 100644 --- a/src/lib/jobs/daily-briefing.ts +++ b/src/lib/jobs/daily-briefing.ts @@ -30,10 +30,6 @@ import { } from "@/lib/daily/daily-briefing-push"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; -export const DAILY_BRIEFING_QUEUE = "daily-briefing"; -/** Every 15 minutes; the per-user local-hour gate selects the fallback slot. */ -export const DAILY_BRIEFING_CRON = "*/15 * * * *"; - export interface DailyBriefingTickSummary { candidatesScanned: number; inSlot: number; diff --git a/src/lib/jobs/dawarich-sync-queue.ts b/src/lib/jobs/dawarich-sync-queue.ts new file mode 100644 index 000000000..c597fab3b --- /dev/null +++ b/src/lib/jobs/dawarich-sync-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `dawarich-sync.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DAWARICH_SYNC_QUEUE = "dawarich-sync"; + +/** Daily 03:20 Europe/Berlin, after the night's environment fetch. */ +export const DAWARICH_SYNC_CRON = "20 3 * * *"; diff --git a/src/lib/jobs/dawarich-sync.ts b/src/lib/jobs/dawarich-sync.ts index e856a388a..ebd4bac30 100644 --- a/src/lib/jobs/dawarich-sync.ts +++ b/src/lib/jobs/dawarich-sync.ts @@ -30,11 +30,7 @@ import { import { daysBetweenDateKeys, shiftDateKey } from "@/lib/tz/format"; import { resolveUserTimezone } from "@/lib/tz/resolver"; import { DAWARICH_SYNC_MAX_DAYS } from "@/lib/validations/dawarich"; - -export const DAWARICH_SYNC_QUEUE = "dawarich-sync"; - -/** Daily 03:20 Europe/Berlin, after the night's environment fetch. */ -export const DAWARICH_SYNC_CRON = "20 3 * * *"; +import { DAWARICH_SYNC_QUEUE } from "./dawarich-sync-queue"; /** The nightly window: yesterday and the six days before it. */ const NIGHTLY_WINDOW_DAYS = 7; diff --git a/src/lib/jobs/dense-intraday-hourly-rebuild-queue.ts b/src/lib/jobs/dense-intraday-hourly-rebuild-queue.ts new file mode 100644 index 000000000..0d80fb3db --- /dev/null +++ b/src/lib/jobs/dense-intraday-hourly-rebuild-queue.ts @@ -0,0 +1,14 @@ +/** + * The queue facts of `dense-intraday-hourly-rebuild.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const DENSE_INTRADAY_HOURLY_REBUILD_QUEUE = + "dense-intraday-hourly-rebuild"; + +/** + * Serial concurrency — a rebuild walks every folded day for one account + * with a transaction per day; concurrency-1 keeps it from crowding the + * request pool, matching the retention drain's convention. + */ +export const DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY = 1; diff --git a/src/lib/jobs/dense-intraday-hourly-rebuild.ts b/src/lib/jobs/dense-intraday-hourly-rebuild.ts index a90de1ea7..14ead6307 100644 --- a/src/lib/jobs/dense-intraday-hourly-rebuild.ts +++ b/src/lib/jobs/dense-intraday-hourly-rebuild.ts @@ -34,16 +34,7 @@ import { DENSE_INTRADAY_RETENTION_TYPES, } from "@/lib/measurements/dense-intraday-retention"; import { runDenseIntradayHourlyRebuild } from "@/lib/measurements/dense-intraday-hourly-rebuild"; - -export const DENSE_INTRADAY_HOURLY_REBUILD_QUEUE = - "dense-intraday-hourly-rebuild"; - -/** - * Serial concurrency — a rebuild walks every folded day for one account - * with a transaction per day; concurrency-1 keeps it from crowding the - * request pool, matching the retention drain's convention. - */ -export const DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY = 1; +import { DENSE_INTRADAY_HOURLY_REBUILD_QUEUE } from "./dense-intraday-hourly-rebuild-queue"; /** * Boot-discovery jobs start only after this delay (seconds). The rebuild diff --git a/src/lib/jobs/dense-intraday-retention-queue.ts b/src/lib/jobs/dense-intraday-retention-queue.ts new file mode 100644 index 000000000..e2a6e80da --- /dev/null +++ b/src/lib/jobs/dense-intraday-retention-queue.ts @@ -0,0 +1,14 @@ +/** + * The queue facts of `dense-intraday-retention.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const DENSE_INTRADAY_RETENTION_QUEUE = "dense-intraday-retention"; + +/** + * Serial concurrency — the drain walks every out-of-window per-sample + * dense-tier row for a user and writes a transaction per day; + * concurrency-1 keeps it from crowding the request pool, matching the + * mean-consolidation + step-consolidation convention. + */ +export const DENSE_INTRADAY_RETENTION_CONCURRENCY = 1; diff --git a/src/lib/jobs/dense-intraday-retention.ts b/src/lib/jobs/dense-intraday-retention.ts index 7cbcdfad1..c807020a9 100644 --- a/src/lib/jobs/dense-intraday-retention.ts +++ b/src/lib/jobs/dense-intraday-retention.ts @@ -27,16 +27,7 @@ import { DENSE_INTRADAY_RETENTION_DAYS, DENSE_INTRADAY_FOLD_SOURCES, } from "@/lib/measurements/dense-intraday-retention"; - -export const DENSE_INTRADAY_RETENTION_QUEUE = "dense-intraday-retention"; - -/** - * Serial concurrency — the drain walks every out-of-window per-sample - * dense-tier row for a user and writes a transaction per day; - * concurrency-1 keeps it from crowding the request pool, matching the - * mean-consolidation + step-consolidation convention. - */ -export const DENSE_INTRADAY_RETENTION_CONCURRENCY = 1; +import { DENSE_INTRADAY_RETENTION_QUEUE } from "./dense-intraday-retention-queue"; /** * Daily at 03:50 Europe/Berlin — inside the existing 03:xx maintenance diff --git a/src/lib/jobs/document-ai-run-queue.ts b/src/lib/jobs/document-ai-run-queue.ts new file mode 100644 index 000000000..53583d010 --- /dev/null +++ b/src/lib/jobs/document-ai-run-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-ai-run.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DOCUMENT_AI_RUN_QUEUE = "document-ai-run"; + +/** Two reads at a time: provider calls, kept off the request pool. */ +export const DOCUMENT_AI_RUN_CONCURRENCY = 2; diff --git a/src/lib/jobs/document-ai-run-reaper-queue.ts b/src/lib/jobs/document-ai-run-reaper-queue.ts new file mode 100644 index 000000000..143869c04 --- /dev/null +++ b/src/lib/jobs/document-ai-run-reaper-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-ai-run-reaper.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the job so the worker + * registry and every sender can read them without loading it. + */ +export const DOCUMENT_AI_RUN_REAPER_QUEUE = "document-ai-run-reaper"; + +/** Every five minutes: a queued run fails within twenty minutes at worst. */ +export const DOCUMENT_AI_RUN_REAPER_CRON = "*/5 * * * *"; diff --git a/src/lib/jobs/document-ai-run-reaper.ts b/src/lib/jobs/document-ai-run-reaper.ts index f2e79eb40..ba6f8bd65 100644 --- a/src/lib/jobs/document-ai-run-reaper.ts +++ b/src/lib/jobs/document-ai-run-reaper.ts @@ -15,10 +15,6 @@ import { reapAiRuns } from "@/lib/documents/ai-runs/store"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; -export const DOCUMENT_AI_RUN_REAPER_QUEUE = "document-ai-run-reaper"; -/** Every five minutes: a queued run fails within twenty minutes at worst. */ -export const DOCUMENT_AI_RUN_REAPER_CRON = "*/5 * * * *"; - export interface DocumentAiRunReaperPayload { triggeredAt?: string; } diff --git a/src/lib/jobs/document-ai-run.ts b/src/lib/jobs/document-ai-run.ts index 2000054b4..6d9f25a52 100644 --- a/src/lib/jobs/document-ai-run.ts +++ b/src/lib/jobs/document-ai-run.ts @@ -97,11 +97,7 @@ import { executeSupplementLabelExtraction, requireSupplementLabelProvider, } from "@/lib/medications/ingredients/label-extract"; - -export const DOCUMENT_AI_RUN_QUEUE = "document-ai-run"; - -/** Two reads at a time: provider calls, kept off the request pool. */ -export const DOCUMENT_AI_RUN_CONCURRENCY = 2; +import { DOCUMENT_AI_RUN_QUEUE } from "./document-ai-run-queue"; /** * The job's expiry: one hour. Three model calls at the longest response diff --git a/src/lib/jobs/document-content-index-backfill-queue.ts b/src/lib/jobs/document-content-index-backfill-queue.ts new file mode 100644 index 000000000..513072f8d --- /dev/null +++ b/src/lib/jobs/document-content-index-backfill-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-content-index-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const CONTENT_INDEX_BACKFILL_QUEUE = "document-content-index-backfill"; + +/** Serial concurrency — provider calls, kept off the request pool. */ +export const CONTENT_INDEX_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-content-index-backfill.ts b/src/lib/jobs/document-content-index-backfill.ts index f355cb7d3..2402e7149 100644 --- a/src/lib/jobs/document-content-index-backfill.ts +++ b/src/lib/jobs/document-content-index-backfill.ts @@ -55,11 +55,7 @@ import { transcribeDocument } from "@/lib/documents/describe"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; import { annotate } from "@/lib/logging/context"; - -export const CONTENT_INDEX_BACKFILL_QUEUE = "document-content-index-backfill"; - -/** Serial concurrency — provider calls, kept off the request pool. */ -export const CONTENT_INDEX_BACKFILL_CONCURRENCY = 1; +import { CONTENT_INDEX_BACKFILL_QUEUE } from "./document-content-index-backfill-queue"; /** Discovery page size (id-only; blobs are loaded one at a time). */ const PAGE_SIZE = 50; diff --git a/src/lib/jobs/document-index-queue.ts b/src/lib/jobs/document-index-queue.ts new file mode 100644 index 000000000..caf1d936d --- /dev/null +++ b/src/lib/jobs/document-index-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-index.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DOCUMENT_INDEX_QUEUE = "document-index"; + +/** Serial concurrency — provider calls + PDF parse, kept off the request pool. */ +export const DOCUMENT_INDEX_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-index.ts b/src/lib/jobs/document-index.ts index abca1d45a..4cfacec37 100644 --- a/src/lib/jobs/document-index.ts +++ b/src/lib/jobs/document-index.ts @@ -17,11 +17,7 @@ import { indexDocumentContent } from "@/lib/documents/index-document"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { annotate } from "@/lib/logging/context"; import { caughtAs, logCaught } from "@/lib/logging/signal"; - -export const DOCUMENT_INDEX_QUEUE = "document-index"; - -/** Serial concurrency — provider calls + PDF parse, kept off the request pool. */ -export const DOCUMENT_INDEX_CONCURRENCY = 1; +import { DOCUMENT_INDEX_QUEUE } from "./document-index-queue"; export interface DocumentIndexPayload { userId: string; diff --git a/src/lib/jobs/document-summary-catchup-queue.ts b/src/lib/jobs/document-summary-catchup-queue.ts new file mode 100644 index 000000000..3fa82c31f --- /dev/null +++ b/src/lib/jobs/document-summary-catchup-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-summary-catchup.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DOCUMENT_SUMMARY_CATCHUP_QUEUE = "document-summary-catchup"; + +/** Serial concurrency — cheap discovery + fan-out enqueues. */ +export const DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-summary-catchup.ts b/src/lib/jobs/document-summary-catchup.ts index 3cb5c191a..9d8ce97d3 100644 --- a/src/lib/jobs/document-summary-catchup.ts +++ b/src/lib/jobs/document-summary-catchup.ts @@ -40,11 +40,7 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; import { annotate } from "@/lib/logging/context"; import { logCaught } from "@/lib/logging/signal"; - -export const DOCUMENT_SUMMARY_CATCHUP_QUEUE = "document-summary-catchup"; - -/** Serial concurrency — cheap discovery + fan-out enqueues. */ -export const DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY = 1; +import { DOCUMENT_SUMMARY_CATCHUP_QUEUE } from "./document-summary-catchup-queue"; /** Discovery page size (id-only). */ const PAGE_SIZE = 100; diff --git a/src/lib/jobs/document-summary-queue.ts b/src/lib/jobs/document-summary-queue.ts new file mode 100644 index 000000000..c509924d7 --- /dev/null +++ b/src/lib/jobs/document-summary-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-summary.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DOCUMENT_SUMMARY_QUEUE = "document-summary"; + +/** Serial concurrency — provider calls, kept off the request pool. */ +export const DOCUMENT_SUMMARY_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-summary-reaper-queue.ts b/src/lib/jobs/document-summary-reaper-queue.ts new file mode 100644 index 000000000..0eb2012db --- /dev/null +++ b/src/lib/jobs/document-summary-reaper-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-summary-reaper.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the job so the worker + * registry and every sender can read them without loading it. + */ +export const DOCUMENT_SUMMARY_REAPER_QUEUE = "document-summary-reaper"; + +/** Hourly at :50 — off the maintenance herd's minute marks. */ +export const DOCUMENT_SUMMARY_REAPER_CRON = "50 * * * *"; diff --git a/src/lib/jobs/document-summary-reaper.ts b/src/lib/jobs/document-summary-reaper.ts index f3e7b9362..775a90926 100644 --- a/src/lib/jobs/document-summary-reaper.ts +++ b/src/lib/jobs/document-summary-reaper.ts @@ -30,10 +30,6 @@ import { getWorkerPrisma } from "@/lib/jobs/reminder/shared"; import { SUMMARY_PENDING_TTL_MS } from "@/lib/documents/store"; import type { PrismaClient } from "@/generated/prisma/client"; -export const DOCUMENT_SUMMARY_REAPER_QUEUE = "document-summary-reaper"; -/** Hourly at :50 — off the maintenance herd's minute marks. */ -export const DOCUMENT_SUMMARY_REAPER_CRON = "50 * * * *"; - export interface DocumentSummaryReaperPayload { triggeredAt?: string; } diff --git a/src/lib/jobs/document-summary.ts b/src/lib/jobs/document-summary.ts index de889748f..dfebd34e6 100644 --- a/src/lib/jobs/document-summary.ts +++ b/src/lib/jobs/document-summary.ts @@ -49,8 +49,7 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { annotate } from "@/lib/logging/context"; import type { DocumentSummaryState } from "@/generated/prisma/client"; import { logCaught } from "@/lib/logging/signal"; - -export const DOCUMENT_SUMMARY_QUEUE = "document-summary"; +import { DOCUMENT_SUMMARY_QUEUE } from "./document-summary-queue"; /** * Record what became of this document's summary. Never touches a READY row — @@ -73,9 +72,6 @@ async function markSummaryState( }); } -/** Serial concurrency — provider calls, kept off the request pool. */ -export const DOCUMENT_SUMMARY_CONCURRENCY = 1; - export interface DocumentSummaryPayload { userId: string; documentId: string; diff --git a/src/lib/jobs/document-thumbnail-backfill-queue.ts b/src/lib/jobs/document-thumbnail-backfill-queue.ts new file mode 100644 index 000000000..d8c086432 --- /dev/null +++ b/src/lib/jobs/document-thumbnail-backfill-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-thumbnail-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the job so the worker + * registry and every sender can read them without loading it. + */ +export const DOCUMENT_THUMBNAIL_BACKFILL_QUEUE = "document-thumbnail-backfill"; + +/** Serial concurrency — cheap discovery + fan-out enqueues. */ +export const DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-thumbnail-backfill.ts b/src/lib/jobs/document-thumbnail-backfill.ts index e09568c6b..bdd373642 100644 --- a/src/lib/jobs/document-thumbnail-backfill.ts +++ b/src/lib/jobs/document-thumbnail-backfill.ts @@ -30,11 +30,7 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; import { annotate } from "@/lib/logging/context"; import { logCaught } from "@/lib/logging/signal"; - -export const DOCUMENT_THUMBNAIL_BACKFILL_QUEUE = "document-thumbnail-backfill"; - -/** Serial concurrency — cheap discovery + fan-out enqueues. */ -export const DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY = 1; +import { DOCUMENT_THUMBNAIL_BACKFILL_QUEUE } from "./document-thumbnail-backfill-queue"; /** Discovery page size (id-only). */ const PAGE_SIZE = 100; diff --git a/src/lib/jobs/document-thumbnail-queue.ts b/src/lib/jobs/document-thumbnail-queue.ts new file mode 100644 index 000000000..312b7269f --- /dev/null +++ b/src/lib/jobs/document-thumbnail-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-thumbnail.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the job so the worker + * registry and every sender can read them without loading it. + */ +export const DOCUMENT_THUMBNAIL_QUEUE = "document-thumbnail"; + +/** Serial concurrency — canvas/pdfjs decode, kept off the request pool. */ +export const DOCUMENT_THUMBNAIL_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-thumbnail.ts b/src/lib/jobs/document-thumbnail.ts index f593ab760..4195820ee 100644 --- a/src/lib/jobs/document-thumbnail.ts +++ b/src/lib/jobs/document-thumbnail.ts @@ -34,11 +34,7 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { withBackgroundEvent } from "@/lib/logging/background"; import { annotate } from "@/lib/logging/context"; import { logCaught } from "@/lib/logging/signal"; - -export const DOCUMENT_THUMBNAIL_QUEUE = "document-thumbnail"; - -/** Serial concurrency — canvas/pdfjs decode, kept off the request pool. */ -export const DOCUMENT_THUMBNAIL_CONCURRENCY = 1; +import { DOCUMENT_THUMBNAIL_QUEUE } from "./document-thumbnail-queue"; export interface DocumentThumbnailPayload { userId: string; diff --git a/src/lib/jobs/drain-cumulative-queue.ts b/src/lib/jobs/drain-cumulative-queue.ts new file mode 100644 index 000000000..352049c6c --- /dev/null +++ b/src/lib/jobs/drain-cumulative-queue.ts @@ -0,0 +1,20 @@ +/** + * The nightly drain of per-sample cumulative rows: its queue, cron and + * payload. It collapses each user × cumulative type × day into one `stats:` + * row, and the same tick folds the daily-mean and dense intra-day retention + * passes (`reminder/rollup-handlers.ts`). 03:45 sits between the 03:15 + * audit-log cleanup and the 04:00 feedback aggregator; the 36-hour grace + * window keeps today and the trailing watch-sync window raw. + */ +export const DRAIN_CUMULATIVE_QUEUE = "drain-per-sample-cumulative"; + +export const DRAIN_CUMULATIVE_CRON = "45 3 * * *"; + +export interface DrainCumulativePayload { + triggeredAt?: string; + /** + * How many times this tick has already handed itself on. Absent on the + * cron's own job. See `nextDrainContinuation`. + */ + continuation?: number; +} diff --git a/src/lib/jobs/encryption-key-rotate-queue.ts b/src/lib/jobs/encryption-key-rotate-queue.ts new file mode 100644 index 000000000..40e686ca1 --- /dev/null +++ b/src/lib/jobs/encryption-key-rotate-queue.ts @@ -0,0 +1,8 @@ +/** + * The queue facts of `encryption-key-rotate.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the job so the worker + * registry and every sender can read them without loading it. + */ +export const ENCRYPTION_KEY_ROTATE_QUEUE = "encryption-key-rotate"; + +export const ENCRYPTION_KEY_ROTATE_CONCURRENCY = 1; diff --git a/src/lib/jobs/encryption-key-rotate.ts b/src/lib/jobs/encryption-key-rotate.ts index cf1c52250..ccde5044a 100644 --- a/src/lib/jobs/encryption-key-rotate.ts +++ b/src/lib/jobs/encryption-key-rotate.ts @@ -36,9 +36,6 @@ import { retireCanariesWithoutData, type CanaryClient, } from "@/lib/crypto/canary"; - -export const ENCRYPTION_KEY_ROTATE_QUEUE = "encryption-key-rotate"; -export const ENCRYPTION_KEY_ROTATE_CONCURRENCY = 1; /** Fixed key so duplicate admin triggers coalesce into one queued run. */ export const ENCRYPTION_KEY_ROTATE_SINGLETON = "encryption-key-rotate"; diff --git a/src/lib/jobs/fitbit-backfill.ts b/src/lib/jobs/fitbit-backfill.ts index ad457bfae..02f84fa21 100644 --- a/src/lib/jobs/fitbit-backfill.ts +++ b/src/lib/jobs/fitbit-backfill.ts @@ -21,15 +21,7 @@ import { } from "@/lib/integrations/page-walk"; import { syncUserFitbit } from "@/lib/fitbit/sync"; import { isFitbitRetired } from "@/lib/fitbit/retirement"; - -export const FITBIT_BACKFILL_QUEUE = "fitbit-backfill"; - -/** - * Serial concurrency — a backfill walks years of history for one account and is - * rate-bounded by Google's per-app quota; concurrency-1 keeps it from crowding - * the request pool, matching `WHOOP_BACKFILL_CONCURRENCY`. - */ -export const FITBIT_BACKFILL_CONCURRENCY = 1; +import { FITBIT_BACKFILL_QUEUE } from "./../fitbit/jobs/queues"; export interface FitbitBackfillPayload { userId: string; diff --git a/src/lib/jobs/fitbit-sleep-repair.ts b/src/lib/jobs/fitbit-sleep-repair.ts index 0b681ff1b..f79ebb8f2 100644 --- a/src/lib/jobs/fitbit-sleep-repair.ts +++ b/src/lib/jobs/fitbit-sleep-repair.ts @@ -46,15 +46,7 @@ import { } from "@/lib/fitbit/sync-core"; import { syncUserSleep } from "@/lib/fitbit/sync-sleep"; import { isFitbitRetired } from "@/lib/fitbit/retirement"; - -export const FITBIT_SLEEP_REPAIR_QUEUE = "fitbit-sleep-repair"; - -/** - * Serial concurrency — a repair walks a year of sleep history for one account - * in 30-day requests against the tight per-user budget; concurrency-1 keeps it - * from crowding the request pool, matching `FITBIT_BACKFILL_CONCURRENCY`. - */ -export const FITBIT_SLEEP_REPAIR_CONCURRENCY = 1; +import { FITBIT_SLEEP_REPAIR_QUEUE } from "./../fitbit/jobs/queues"; export interface FitbitSleepRepairPayload { userId: string; diff --git a/src/lib/jobs/google-health-backfill-queue.ts b/src/lib/jobs/google-health-backfill-queue.ts new file mode 100644 index 000000000..26856d461 --- /dev/null +++ b/src/lib/jobs/google-health-backfill-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `google-health-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const GOOGLE_HEALTH_BACKFILL_QUEUE = "google-health-backfill"; + +/** + * Serial concurrency — a backfill walks years of history for one account and is + * rate-bounded by Google's per-app quota; concurrency-1 keeps it from crowding + * the request pool, matching `FITBIT_BACKFILL_CONCURRENCY`. + */ +export const GOOGLE_HEALTH_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/google-health-backfill.ts b/src/lib/jobs/google-health-backfill.ts index d070872de..94d44bb9e 100644 --- a/src/lib/jobs/google-health-backfill.ts +++ b/src/lib/jobs/google-health-backfill.ts @@ -44,15 +44,7 @@ import { import { reportJobProgress } from "@/lib/jobs/job-observer"; import { LANE_YIELD_DELAY_SECONDS, laneWanted } from "@/lib/jobs/heavy-lane"; import { GOOGLE_HEALTH_COLLECTION_KEYS } from "@/lib/google-health/collection-keys"; - -export const GOOGLE_HEALTH_BACKFILL_QUEUE = "google-health-backfill"; - -/** - * Serial concurrency — a backfill walks years of history for one account and is - * rate-bounded by Google's per-app quota; concurrency-1 keeps it from crowding - * the request pool, matching `FITBIT_BACKFILL_CONCURRENCY`. - */ -export const GOOGLE_HEALTH_BACKFILL_CONCURRENCY = 1; +import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "./google-health-backfill-queue"; /** * One slice of a backfill: how long it walks and how many pages it reads diff --git a/src/lib/jobs/google-health-sleep-repair-queue.ts b/src/lib/jobs/google-health-sleep-repair-queue.ts new file mode 100644 index 000000000..4f83dcf38 --- /dev/null +++ b/src/lib/jobs/google-health-sleep-repair-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `google-health-sleep-repair.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE = "google-health-sleep-repair"; + +/** + * Serial concurrency — a repair walks the full sleep history for one account + * and is rate-bounded by Google's per-app quota; concurrency-1 keeps it from + * crowding the request pool, matching `GOOGLE_HEALTH_BACKFILL_CONCURRENCY`. + */ +export const GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY = 1; diff --git a/src/lib/jobs/google-health-sleep-repair.ts b/src/lib/jobs/google-health-sleep-repair.ts index 8c01adc9b..e4fe7dedf 100644 --- a/src/lib/jobs/google-health-sleep-repair.ts +++ b/src/lib/jobs/google-health-sleep-repair.ts @@ -37,15 +37,7 @@ import { GOOGLE_HEALTH_SYNC_LOCK_WAIT_MS, withGoogleHealthSyncLock, } from "@/lib/google-health/sync-lock"; - -export const GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE = "google-health-sleep-repair"; - -/** - * Serial concurrency — a repair walks the full sleep history for one account - * and is rate-bounded by Google's per-app quota; concurrency-1 keeps it from - * crowding the request pool, matching `GOOGLE_HEALTH_BACKFILL_CONCURRENCY`. - */ -export const GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY = 1; +import { GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE } from "./google-health-sleep-repair-queue"; export interface GoogleHealthSleepRepairPayload { userId: string; diff --git a/src/lib/jobs/health-connect-import-queue.ts b/src/lib/jobs/health-connect-import-queue.ts new file mode 100644 index 000000000..55b3f3c1b --- /dev/null +++ b/src/lib/jobs/health-connect-import-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `health-connect-import-worker.ts`: its queue names and + * concurrency. They live apart from the worker so the registry and the upload + * routes can name the queue without loading the parser. + */ +export const HEALTH_CONNECT_IMPORT_QUEUE = "health-connect-import"; + +/** One import at a time per worker: the SQLite read and the writes are heavy. */ +export const HEALTH_CONNECT_IMPORT_CONCURRENCY = 1; diff --git a/src/lib/jobs/health-connect-import-worker.ts b/src/lib/jobs/health-connect-import-worker.ts index 08850c72a..a989b56c5 100644 --- a/src/lib/jobs/health-connect-import-worker.ts +++ b/src/lib/jobs/health-connect-import-worker.ts @@ -33,8 +33,6 @@ import { annotate } from "@/lib/logging/context"; import { withBackgroundEvent } from "@/lib/logging/background"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; -export const HEALTH_CONNECT_IMPORT_QUEUE = "health-connect-import"; - /** `ImportJob.kind` of the rows this worker owns. */ export const HEALTH_CONNECT_IMPORT_KIND = "health_connect"; @@ -45,9 +43,6 @@ export const HEALTH_CONNECT_IMPORT_KIND = "health_connect"; */ export const HEALTH_CONNECT_IMPORT_PARSER_REVISION = 1; -/** One import at a time per worker: the SQLite read and the writes are heavy. */ -export const HEALTH_CONNECT_IMPORT_CONCURRENCY = 1; - /** Same shape and reasoning as `APPLE_HEALTH_IMPORT_SEND_OPTIONS`. */ export const HEALTH_CONNECT_IMPORT_SEND_OPTIONS = { retryLimit: 0, diff --git a/src/lib/jobs/insight-pregenerate-shared.ts b/src/lib/jobs/insight-pregenerate-shared.ts index 136d9353a..f15a9e605 100644 --- a/src/lib/jobs/insight-pregenerate-shared.ts +++ b/src/lib/jobs/insight-pregenerate-shared.ts @@ -249,3 +249,10 @@ export async function enqueuePregenerateFailureRetry(payload: { // catch-net, exactly the pre-retry behaviour. } } + +/** + * 04:30 Europe/Berlin. Slots between the cumulative drain (03:30) and + * the feedback aggregator (04:00) maintenance window's tail so the + * LLM-bound pass doesn't contend with the SQL-bound nightly folds. + */ +export const INSIGHT_PREGENERATE_CRON = "30 4 * * *"; diff --git a/src/lib/jobs/insight-pregenerate.ts b/src/lib/jobs/insight-pregenerate.ts index 509551b51..b8a6ce6b8 100644 --- a/src/lib/jobs/insight-pregenerate.ts +++ b/src/lib/jobs/insight-pregenerate.ts @@ -91,13 +91,6 @@ import { export { INSIGHT_PREGENERATE_QUEUE, enqueueForceWarm }; export type { InsightPregeneratePayload }; -/** - * 04:30 Europe/Berlin. Slots between the cumulative drain (03:30) and - * the feedback aggregator (04:00) maintenance window's tail so the - * LLM-bound pass doesn't contend with the SQL-bound nightly folds. - */ -export const INSIGHT_PREGENERATE_CRON = "30 4 * * *"; - /** Stale-cache threshold for the nightly discovery pass. * * The cron runs every 24 h and the advisor cache lives 24 h — so ANY diff --git a/src/lib/jobs/insight-status-generate-queue.ts b/src/lib/jobs/insight-status-generate-queue.ts new file mode 100644 index 000000000..bee53259d --- /dev/null +++ b/src/lib/jobs/insight-status-generate-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `insight-status-generate-shared.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const INSIGHT_STATUS_GENERATE_QUEUE = "insight-status-generate"; + +/** + * Low concurrency keeps the on-demand warm pass from saturating the + * Prisma pool or fanning out an unbounded number of concurrent provider + * calls when several cards request generation on the same first visit. + */ +export const INSIGHT_STATUS_GENERATE_CONCURRENCY = 2; diff --git a/src/lib/jobs/insight-status-generate-shared.ts b/src/lib/jobs/insight-status-generate-shared.ts index 5296732d1..e448872d9 100644 --- a/src/lib/jobs/insight-status-generate-shared.ts +++ b/src/lib/jobs/insight-status-generate-shared.ts @@ -21,15 +21,7 @@ import { type ProviderWorkAuthority, } from "@/lib/sharing/provider-work-authority"; import { logCaught } from "@/lib/logging/signal"; - -export const INSIGHT_STATUS_GENERATE_QUEUE = "insight-status-generate"; - -/** - * Low concurrency keeps the on-demand warm pass from saturating the - * Prisma pool or fanning out an unbounded number of concurrent provider - * calls when several cards request generation on the same first visit. - */ -export const INSIGHT_STATUS_GENERATE_CONCURRENCY = 2; +import { INSIGHT_STATUS_GENERATE_QUEUE } from "./insight-status-generate-queue"; /** * The seven per-metric status scopes the queue can generate. Mirrors the diff --git a/src/lib/jobs/insight-status-generate.ts b/src/lib/jobs/insight-status-generate.ts index 80118c473..33b525017 100644 --- a/src/lib/jobs/insight-status-generate.ts +++ b/src/lib/jobs/insight-status-generate.ts @@ -49,13 +49,15 @@ import { } from "@/lib/insights/derived/derived-assessment-ai"; import { isDerivedMetricId } from "@/lib/insights/derived/registry"; import { - INSIGHT_STATUS_GENERATE_QUEUE, - INSIGHT_STATUS_GENERATE_CONCURRENCY, INSIGHT_STATUS_METRICS, enqueueStatusGeneration, type InsightStatusMetric, type InsightStatusGeneratePayload, } from "@/lib/jobs/insight-status-generate-shared"; +import { + INSIGHT_STATUS_GENERATE_QUEUE, + INSIGHT_STATUS_GENERATE_CONCURRENCY, +} from "@/lib/jobs/insight-status-generate-queue"; import { aiCapabilityForJob } from "@/lib/ai/capabilities/gate"; import { mayDispatchProviderWork, diff --git a/src/lib/jobs/intake-auto-skip-queue.ts b/src/lib/jobs/intake-auto-skip-queue.ts new file mode 100644 index 000000000..6488f849d --- /dev/null +++ b/src/lib/jobs/intake-auto-skip-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `intake-auto-skip.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const INTAKE_AUTO_SKIP_QUEUE = "intake-auto-skip"; + +/** + * Cron schedule. Hourly at :05 so the tick doesn't collide with the + * top-of-the-hour reminder-check (:00) or the rollup fold (:30) + * cadences. Europe/Berlin matches the rest of the worker's schedules. + */ +export const INTAKE_AUTO_SKIP_CRON = "5 * * * *"; diff --git a/src/lib/jobs/intake-auto-skip.ts b/src/lib/jobs/intake-auto-skip.ts index 0220bc51e..db63b19c0 100644 --- a/src/lib/jobs/intake-auto-skip.ts +++ b/src/lib/jobs/intake-auto-skip.ts @@ -73,15 +73,6 @@ import { } from "@/lib/medications/scheduling/live-era"; import { INTAKE_AUTO_TAKE_LOOKBACK_MS } from "@/lib/jobs/intake-auto-take"; -export const INTAKE_AUTO_SKIP_QUEUE = "intake-auto-skip"; - -/** - * Cron schedule. Hourly at :05 so the tick doesn't collide with the - * top-of-the-hour reminder-check (:00) or the rollup fold (:30) - * cadences. Europe/Berlin matches the rest of the worker's schedules. - */ -export const INTAKE_AUTO_SKIP_CRON = "5 * * * *"; - /** * Grace floor before auto-skip kicks in. 24 h matches the spec and * the user's real-world cadence (late entries up to a full day after diff --git a/src/lib/jobs/intake-auto-take-queue.ts b/src/lib/jobs/intake-auto-take-queue.ts new file mode 100644 index 000000000..8572b837b --- /dev/null +++ b/src/lib/jobs/intake-auto-take-queue.ts @@ -0,0 +1,12 @@ +/** + * The queue facts of `intake-auto-take.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const INTAKE_AUTO_TAKE_QUEUE = "intake-auto-take"; + +/** + * Every 15 minutes, off the reminder tick (:00/:15/:30/:45) and the hourly + * auto-miss pass (:05). A 19:00 dose reads taken by 19:10 at the latest. + */ +export const INTAKE_AUTO_TAKE_CRON = "10,25,40,55 * * * *"; diff --git a/src/lib/jobs/intake-auto-take.ts b/src/lib/jobs/intake-auto-take.ts index e26a42dca..16265f1d2 100644 --- a/src/lib/jobs/intake-auto-take.ts +++ b/src/lib/jobs/intake-auto-take.ts @@ -98,14 +98,6 @@ export { type SlotDecision, } from "@/lib/medications/auto-take"; -export const INTAKE_AUTO_TAKE_QUEUE = "intake-auto-take"; - -/** - * Every 15 minutes, off the reminder tick (:00/:15/:30/:45) and the hourly - * auto-miss pass (:05). A 19:00 dose reads taken by 19:10 at the latest. - */ -export const INTAKE_AUTO_TAKE_CRON = "10,25,40,55 * * * *"; - const MINUTE_MS = 60 * 1000; const HOUR_MS = 60 * MINUTE_MS; const DAY_MS = 24 * HOUR_MS; diff --git a/src/lib/jobs/integration-backfill-forward.ts b/src/lib/jobs/integration-backfill-forward.ts new file mode 100644 index 000000000..054fc111b --- /dev/null +++ b/src/lib/jobs/integration-backfill-forward.ts @@ -0,0 +1,27 @@ +/** + * The handler of every full-history source queue (`whoop-backfill`, + * `strava-backfill`, the sleep repairs, ...): it forwards each job to the + * shared admission queue, where the import itself runs in the one + * cross-provider lane. The source queue keeps its per-connection singleton + * key, which is what boot discovery re-offers against. + */ +import type { Job, PgBoss } from "pg-boss"; + +import { + enqueueIntegrationBackfillAdmission, + type IntegrationBackfillData, + type IntegrationBackfillKind, +} from "@/lib/jobs/integration-backfill-admission"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; + +export function forwardToBackfillAdmission( + boss: PgBoss, + kind: IntegrationBackfillKind, +): (jobs: Job[]) => Promise { + return async (jobs) => { + for (const job of jobs) { + await enqueueIntegrationBackfillAdmission(boss, { kind, data: job.data }); + } + return jobDone({ jobs: jobs.length }); + }; +} diff --git a/src/lib/jobs/integration-poll-queues.ts b/src/lib/jobs/integration-poll-queues.ts index addd5e94d..415921c0e 100644 --- a/src/lib/jobs/integration-poll-queues.ts +++ b/src/lib/jobs/integration-poll-queues.ts @@ -1,27 +1,85 @@ /** - * Queue names for the poll-only integration syncs. + * Names and crons of the integration sync queues. * - * Split out of `reminder/register-integration-sync.ts` so a request-path caller - * (an OAuth callback enqueueing the first sync at connect time) can name the - * queue without importing the registrar — which would drag every worker handler - * and its transitive graph into the request bundle. Same shape as - * `STRAVA_BACKFILL_QUEUE`, which the Strava callback already imports from its - * own module. + * A light module on purpose: the worker registry declares these queues from + * here (`registry/integration-sync.ts`), and a request-path caller (an OAuth + * callback enqueueing the first sync, a webhook enqueueing one resource) names + * a queue without loading the sync it feeds. * - * The registrar remains the single place that creates, schedules, and binds a - * handler to each of these; the values here are its source. A name declared - * here but missing from `allQueues` there is the v1.4.37 dead-queue class, and - * the queue-wiring guards read both files for exactly that reason. + * The hourly minutes are staggered so the polls do not land on one boss poll: + * used are :00 :05 :08 :11 :13 :15 :17 :18 :20 :35 :41 :47 :50. */ -/** Hourly Nightscout CGM poll (:11). */ +/** Withings measure catch-net (the webhook is primary). */ +export const WITHINGS_SYNC_QUEUE = "withings-fallback-sync"; +export const WITHINGS_SYNC_CRON = "0 * * * *"; + +/** + * Withings activity and sleep: webhook-primary (appli 16 / 44), with an hourly + * catch-net for the deliveries Withings drops. + */ +export const WITHINGS_ACTIVITY_QUEUE = "withings-activity-sync"; +export const WITHINGS_ACTIVITY_CRON = "0 * * * *"; +export const WITHINGS_SLEEP_QUEUE = "withings-sleep-sync"; +export const WITHINGS_SLEEP_CRON = "15 * * * *"; + +/** + * Withings ECG catch-net. The queue is webhook-primary (appli 54), but appli 54 + * is not in the subscription set and the manual sync measures only, so a + * watch-only account would never have its strips pulled without this walk. + */ +export const WITHINGS_ECG_SYNC_CRON = "41 * * * *"; + +/** Daily sweep of the Withings OAuth state ledger. */ +export const WITHINGS_OAUTH_STATE_CLEANUP_QUEUE = + "withings-oauth-state-cleanup"; +export const WITHINGS_OAUTH_STATE_CLEANUP_CRON = "20 3 * * *"; + +/** + * WHOOP: recovery, sleep and workout are webhook-primary with an hourly + * catch-net; cycle has no webhook, so its cron is the only driver. + */ +export const WHOOP_RECOVERY_SYNC_QUEUE = "whoop-recovery-sync"; +export const WHOOP_RECOVERY_SYNC_CRON = "5 * * * *"; +export const WHOOP_SLEEP_SYNC_QUEUE = "whoop-sleep-sync"; +export const WHOOP_SLEEP_SYNC_CRON = "20 * * * *"; +export const WHOOP_WORKOUT_SYNC_QUEUE = "whoop-workout-sync"; +export const WHOOP_WORKOUT_SYNC_CRON = "35 * * * *"; +export const WHOOP_CYCLE_SYNC_QUEUE = "whoop-cycle-sync"; +export const WHOOP_CYCLE_SYNC_CRON = "50 * * * *"; + +/** Daily sweep of the WHOOP OAuth state ledger. */ +export const WHOOP_OAUTH_STATE_CLEANUP_QUEUE = "whoop-oauth-state-cleanup"; +export const WHOOP_OAUTH_STATE_CLEANUP_CRON = "22 3 * * *"; + +/** Google Health poll (no webhook). */ +export const GOOGLE_HEALTH_SYNC_QUEUE = "google-health-sync"; +export const GOOGLE_HEALTH_SYNC_CRON = "18 * * * *"; + +/** Daily sweep of the Google Health OAuth state ledger. */ +export const GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE = + "google-health-oauth-state-cleanup"; +export const GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON = "26 3 * * *"; + +/** Nightscout CGM poll. */ export const NIGHTSCOUT_SYNC_QUEUE = "nightscout-sync"; +export const NIGHTSCOUT_SYNC_CRON = "11 * * * *"; -/** Hourly Polar AccessLink poll (:13). */ +/** Polar AccessLink poll. */ export const POLAR_SYNC_QUEUE = "polar-sync"; +export const POLAR_SYNC_CRON = "13 * * * *"; -/** Hourly Oura Cloud poll (:15). */ +/** Oura Cloud poll. */ export const OURA_SYNC_QUEUE = "oura-sync"; +export const OURA_SYNC_CRON = "15 * * * *"; -/** Hourly Strava poll (:17). */ +/** Strava poll (the webhook is not wired). */ export const STRAVA_SYNC_QUEUE = "strava-sync"; +export const STRAVA_SYNC_CRON = "17 * * * *"; + +/** + * Daily sweep of the native OIDC handoff ledger. Handoff codes are single-use + * and expire at read, so this only reaps rows the app never came back for. + */ +export const OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE = "oidc-native-handoff-cleanup"; +export const OIDC_NATIVE_HANDOFF_CLEANUP_CRON = "28 3 * * *"; diff --git a/src/lib/jobs/integration-watchdog-queue.ts b/src/lib/jobs/integration-watchdog-queue.ts new file mode 100644 index 000000000..6324b1e1f --- /dev/null +++ b/src/lib/jobs/integration-watchdog-queue.ts @@ -0,0 +1,18 @@ +/** + * The queue facts of `integration-watchdog.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const INTEGRATION_WATCHDOG_HOURLY_QUEUE = "integration-watchdog-hourly"; + +export const INTEGRATION_WATCHDOG_DAILY_QUEUE = "integration-watchdog-daily"; + +/** + * :47 is a free minute off every hourly sync tick (used: 0, 5, 8, 11, 13, 15, + * 17, 18, 20, 30, 35, 41, 50), late enough in the hour that the syncs before it + * have written what they fetched. + */ +export const INTEGRATION_WATCHDOG_HOURLY_CRON = "47 * * * *"; + +/** Once a day, inside the maintenance window, off the hourly :47. */ +export const INTEGRATION_WATCHDOG_DAILY_CRON = "52 4 * * *"; diff --git a/src/lib/jobs/integration-watchdog.ts b/src/lib/jobs/integration-watchdog.ts index f4ce883e8..155ee95a5 100644 --- a/src/lib/jobs/integration-watchdog.ts +++ b/src/lib/jobs/integration-watchdog.ts @@ -16,18 +16,6 @@ import { type WatchdogMode, } from "@/lib/integrations/watchdog"; -export const INTEGRATION_WATCHDOG_HOURLY_QUEUE = "integration-watchdog-hourly"; -export const INTEGRATION_WATCHDOG_DAILY_QUEUE = "integration-watchdog-daily"; - -/** - * :47 is a free minute off every hourly sync tick (used: 0, 5, 8, 11, 13, 15, - * 17, 18, 20, 30, 35, 41, 50), late enough in the hour that the syncs before it - * have written what they fetched. - */ -export const INTEGRATION_WATCHDOG_HOURLY_CRON = "47 * * * *"; -/** Once a day, inside the maintenance window, off the hourly :47. */ -export const INTEGRATION_WATCHDOG_DAILY_CRON = "52 4 * * *"; - const LOCK_KEY = "job:integration-watchdog:pass"; export interface IntegrationWatchdogPayload { diff --git a/src/lib/jobs/measurement-fold-repair-queue.ts b/src/lib/jobs/measurement-fold-repair-queue.ts new file mode 100644 index 000000000..ff3060202 --- /dev/null +++ b/src/lib/jobs/measurement-fold-repair-queue.ts @@ -0,0 +1,6 @@ +/** + * The queue facts of `measurement-fold-repair.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const MEASUREMENT_FOLD_REPAIR_QUEUE = "measurement-fold-repair"; diff --git a/src/lib/jobs/measurement-fold-repair.ts b/src/lib/jobs/measurement-fold-repair.ts index 1efc75a9c..ffab94285 100644 --- a/src/lib/jobs/measurement-fold-repair.ts +++ b/src/lib/jobs/measurement-fold-repair.ts @@ -90,10 +90,8 @@ import { holdAccountFoldLock, } from "@/lib/export/restore-lock"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; -import { - COMPACTION_TOMBSTONE_PURGE_QUEUE, - COMPACTION_TOMBSTONE_PURGE_SINGLETON_KEY, -} from "@/lib/jobs/compaction-tombstone-purge"; +import { COMPACTION_TOMBSTONE_PURGE_SINGLETON_KEY } from "@/lib/jobs/compaction-tombstone-purge"; +import { COMPACTION_TOMBSTONE_PURGE_QUEUE } from "@/lib/jobs/compaction-tombstone-purge-queue"; import { jobBudget } from "@/lib/jobs/job-budget"; import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { reportJobRun, type JobRunCandidate } from "@/lib/jobs/job-run-report"; @@ -136,8 +134,7 @@ import { } from "@/lib/measurements/fold-constituents"; import { recomputeBucketsForMeasurement } from "@/lib/rollups/measurement-rollups"; import { localDayWindow } from "@/lib/tz/local-day"; - -export const MEASUREMENT_FOLD_REPAIR_QUEUE = "measurement-fold-repair"; +import { MEASUREMENT_FOLD_REPAIR_QUEUE } from "./measurement-fold-repair-queue"; /** Delay before the follow-up of a run that stopped early, in seconds. */ export const FOLD_REPAIR_CONTINUATION_DELAY_SECONDS = 30; diff --git a/src/lib/jobs/medication-intake-import-queue.ts b/src/lib/jobs/medication-intake-import-queue.ts new file mode 100644 index 000000000..da52962dc --- /dev/null +++ b/src/lib/jobs/medication-intake-import-queue.ts @@ -0,0 +1,8 @@ +/** + * The queue facts of `medication-intake-import.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const MEDICATION_INTAKE_IMPORT_QUEUE = "medication-intake-import"; + +export const MEDICATION_INTAKE_IMPORT_CONCURRENCY = 2; diff --git a/src/lib/jobs/medication-intake-import.ts b/src/lib/jobs/medication-intake-import.ts index 3fda3d1b2..bea85b8c1 100644 --- a/src/lib/jobs/medication-intake-import.ts +++ b/src/lib/jobs/medication-intake-import.ts @@ -9,9 +9,6 @@ import { jobBudget } from "@/lib/jobs/job-budget"; import { enqueueSupplementNutrientRecompute } from "@/lib/medications/ingredients/recompute-job"; import { consumeImportedIntakesBatch } from "@/lib/medications/inventory/consumption"; import { queueMedicationIntakeSync } from "@/lib/notifications/medication-intake-sync"; - -export const MEDICATION_INTAKE_IMPORT_QUEUE = "medication-intake-import"; -export const MEDICATION_INTAKE_IMPORT_CONCURRENCY = 2; export const MEDICATION_INTAKE_IMPORT_CHUNK_SIZE = 100; export const MEDICATION_INTAKE_IMPORT_STALE_AFTER_MS = 15 * 60_000; diff --git a/src/lib/jobs/medication-low-stock-queue.ts b/src/lib/jobs/medication-low-stock-queue.ts new file mode 100644 index 000000000..aeeca6dc3 --- /dev/null +++ b/src/lib/jobs/medication-low-stock-queue.ts @@ -0,0 +1,14 @@ +/** + * The queue facts of `medication-low-stock.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +/** Pg-boss queue + cron — imported by the reminder worker's bootstrap. */ +export const MEDICATION_LOW_STOCK_QUEUE = "medication-low-stock"; + +/** + * 09:00 Europe/Berlin — a supply alert is an errand prompt ("order a + * refill"), not a night signal; it fires once daily at a time the user + * can act on it. + */ +export const MEDICATION_LOW_STOCK_CRON = "0 9 * * *"; diff --git a/src/lib/jobs/medication-low-stock.ts b/src/lib/jobs/medication-low-stock.ts index 5c2e8af5c..42b2db001 100644 --- a/src/lib/jobs/medication-low-stock.ts +++ b/src/lib/jobs/medication-low-stock.ts @@ -86,15 +86,6 @@ import { resolveJobLocale } from "@/lib/i18n/job-locale"; import { annotate, getEvent } from "@/lib/logging/context"; import { TRACKED_INTAKE_WHERE } from "@/lib/medications/intake-tracking"; -/** Pg-boss queue + cron — imported by the reminder worker's bootstrap. */ -export const MEDICATION_LOW_STOCK_QUEUE = "medication-low-stock"; -/** - * 09:00 Europe/Berlin — a supply alert is an errand prompt ("order a - * refill"), not a night signal; it fires once daily at a time the user - * can act on it. - */ -export const MEDICATION_LOW_STOCK_CRON = "0 9 * * *"; - /** * An as-needed medication warns when this many doses or fewer are left: no * cadence says how many days that is, so the floor is a dose count. diff --git a/src/lib/jobs/mood-prognosis-refresh-queue.ts b/src/lib/jobs/mood-prognosis-refresh-queue.ts new file mode 100644 index 000000000..41f75c78c --- /dev/null +++ b/src/lib/jobs/mood-prognosis-refresh-queue.ts @@ -0,0 +1,8 @@ +/** + * The queue facts of `mood-prognosis-refresh.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const MOOD_PROGNOSIS_REFRESH_QUEUE = "mood-prognosis-refresh"; + +export const MOOD_PROGNOSIS_REFRESH_CRON = "35 4 * * *"; diff --git a/src/lib/jobs/mood-prognosis-refresh.ts b/src/lib/jobs/mood-prognosis-refresh.ts index 30ab95fba..e63ff411a 100644 --- a/src/lib/jobs/mood-prognosis-refresh.ts +++ b/src/lib/jobs/mood-prognosis-refresh.ts @@ -50,10 +50,6 @@ import { isModuleEnabled } from "@/lib/modules/gate"; import { getWorkerPrisma } from "./reminder/shared"; -export const MOOD_PROGNOSIS_REFRESH_QUEUE = "mood-prognosis-refresh"; - -export const MOOD_PROGNOSIS_REFRESH_CRON = "35 4 * * *"; - export interface MoodPrognosisRefreshPayload { triggeredAt: string; } diff --git a/src/lib/jobs/offhost-backup-queue.ts b/src/lib/jobs/offhost-backup-queue.ts new file mode 100644 index 000000000..13e6a0353 --- /dev/null +++ b/src/lib/jobs/offhost-backup-queue.ts @@ -0,0 +1,27 @@ +/** + * The queue facts of `offhost-backup.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy"; + +/** + * The off-host pass's expiry: four hours. It uploads every account and + * already stopped between accounts at three quarters of the default fifteen + * minutes, which on an instance of any size meant a retry per night; one + * large account past the default was retried beside itself. Four hours with + * the same budget and a lock (`lockedPass`) leaves room for the night's pass. + */ +export const OFFHOST_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60; + +export const OFFHOST_BACKUP_QUEUE = "data-backup-offhost"; + +/** + * What the nightly schedule sends with: the four hours above, and the + * heartbeat that notices a process that died under the pass within minutes + * rather than at the end of them (`BACKUP_HEARTBEAT_SECONDS`). + */ +export const OFFHOST_BACKUP_SEND_OPTIONS = { + expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS, + heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS, +} as const; diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 1016ce032..cf6fbf1ba 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -41,35 +41,14 @@ import { orderInterruptedLast, readAccountsToTakeLast, } from "@/lib/jobs/backup-pass-attempts"; -import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy"; import { annotate, getEvent } from "@/lib/logging/context"; import { envValue } from "@/lib/env"; +import { OFFHOST_BACKUP_QUEUE } from "./offhost-backup-queue"; const ALGORITHM = "aes-256-gcm"; const IV_LENGTH = 12; const TAG_LENGTH = 16; -/** - * The off-host pass's expiry: four hours. It uploads every account and - * already stopped between accounts at three quarters of the default fifteen - * minutes, which on an instance of any size meant a retry per night; one - * large account past the default was retried beside itself. Four hours with - * the same budget and a lock (`lockedPass`) leaves room for the night's pass. - */ -export const OFFHOST_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60; - -export const OFFHOST_BACKUP_QUEUE = "data-backup-offhost"; - -/** - * What the nightly schedule sends with: the four hours above, and the - * heartbeat that notices a process that died under the pass within minutes - * rather than at the end of them (`BACKUP_HEARTBEAT_SECONDS`). - */ -export const OFFHOST_BACKUP_SEND_OPTIONS = { - expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS, - heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS, -} as const; - export interface OffhostBackupConfig { endpoint: string; bucket: string; diff --git a/src/lib/jobs/offhost-purge-queue.ts b/src/lib/jobs/offhost-purge-queue.ts new file mode 100644 index 000000000..4630131a5 --- /dev/null +++ b/src/lib/jobs/offhost-purge-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `offhost-purge.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge"; + +/** 03:40, after the nightly upload (02:30) has had its hour. */ +export const OFFHOST_PURGE_CRON = "40 3 * * *"; diff --git a/src/lib/jobs/offhost-purge.ts b/src/lib/jobs/offhost-purge.ts index cc08a3acc..2cdf06cd8 100644 --- a/src/lib/jobs/offhost-purge.ts +++ b/src/lib/jobs/offhost-purge.ts @@ -46,10 +46,7 @@ import { reportWorkerError } from "@/lib/jobs/report-worker-error"; import { getWorkerPrisma } from "@/lib/jobs/reminder/shared"; import { withBackgroundEvent } from "@/lib/logging/background"; import { logCaught } from "@/lib/logging/signal"; - -export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge"; -/** 03:40, after the nightly upload (02:30) has had its hour. */ -export const OFFHOST_PURGE_CRON = "40 3 * * *"; +import { OFFHOST_PURGE_QUEUE } from "./offhost-purge-queue"; export type OffhostPurgeReason = "account_deleted" | "data_wiped" | "managed_profile_deleted"; diff --git a/src/lib/jobs/oura-backfill-queue.ts b/src/lib/jobs/oura-backfill-queue.ts new file mode 100644 index 000000000..47fdb4c66 --- /dev/null +++ b/src/lib/jobs/oura-backfill-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `oura-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const OURA_BACKFILL_QUEUE = "oura-backfill"; + +/** One account at a time; the shared lane bounds the instance. */ +export const OURA_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/oura-backfill.ts b/src/lib/jobs/oura-backfill.ts index 6fad28bef..413267412 100644 --- a/src/lib/jobs/oura-backfill.ts +++ b/src/lib/jobs/oura-backfill.ts @@ -68,11 +68,7 @@ import { withoutDenied, type OuraBackfillProgress, } from "@/lib/oura/backfill-progress"; - -export const OURA_BACKFILL_QUEUE = "oura-backfill"; - -/** One account at a time; the shared lane bounds the instance. */ -export const OURA_BACKFILL_CONCURRENCY = 1; +import { OURA_BACKFILL_QUEUE } from "./oura-backfill-queue"; /** * One slice: five minutes of walking, then the account is let go and the diff --git a/src/lib/jobs/period-narrative-queue.ts b/src/lib/jobs/period-narrative-queue.ts new file mode 100644 index 000000000..5628280d9 --- /dev/null +++ b/src/lib/jobs/period-narrative-queue.ts @@ -0,0 +1,24 @@ +/** + * The queue facts of `period-narrative-shared.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const PERIOD_NARRATIVE_QUEUE = "period-narrative-warm"; + +/** + * The job's expiry: three hours. On a boundary night the scheduled pass + * writes up to two narratives for each of 200 accounts, which can outlast + * pg-boss's fifteen-minute default. The pass stops itself at three quarters + * of this (`jobBudget`) and holds a lock (`lockedPass`); every send and the + * schedule carry it. + */ +export const PERIOD_NARRATIVE_EXPIRE_SECONDS = 3 * 60 * 60; + +/** + * Nightly at 05:05 Europe/Berlin — inside the existing maintenance window, + * after the comprehensive insight pre-generation (04:30) and the computed + * scores (04:45–04:55) so the rollup signals it reads are already folded. + * The handler only does real work near a week/month boundary; on every other + * night it short-circuits cheaply. + */ +export const PERIOD_NARRATIVE_CRON = "5 5 * * *"; diff --git a/src/lib/jobs/period-narrative-shared.ts b/src/lib/jobs/period-narrative-shared.ts index 788ab0258..ac43bb66d 100644 --- a/src/lib/jobs/period-narrative-shared.ts +++ b/src/lib/jobs/period-narrative-shared.ts @@ -21,26 +21,10 @@ import { import type { NarrativePeriod } from "@/lib/insights/narrative/period-narrative"; import type { Locale } from "@/lib/i18n/config"; import { logCaught } from "@/lib/logging/signal"; - -export const PERIOD_NARRATIVE_QUEUE = "period-narrative-warm"; - -/** - * The job's expiry: three hours. On a boundary night the scheduled pass - * writes up to two narratives for each of 200 accounts, which can outlast - * pg-boss's fifteen-minute default. The pass stops itself at three quarters - * of this (`jobBudget`) and holds a lock (`lockedPass`); every send and the - * schedule carry it. - */ -export const PERIOD_NARRATIVE_EXPIRE_SECONDS = 3 * 60 * 60; - -/** - * Nightly at 05:05 Europe/Berlin — inside the existing maintenance window, - * after the comprehensive insight pre-generation (04:30) and the computed - * scores (04:45–04:55) so the rollup signals it reads are already folded. - * The handler only does real work near a week/month boundary; on every other - * night it short-circuits cheaply. - */ -export const PERIOD_NARRATIVE_CRON = "5 5 * * *"; +import { + PERIOD_NARRATIVE_QUEUE, + PERIOD_NARRATIVE_EXPIRE_SECONDS, +} from "./period-narrative-queue"; export interface PeriodNarrativePayload { /** Set on the nightly scheduled tick (informational). */ diff --git a/src/lib/jobs/period-narrative-warm.ts b/src/lib/jobs/period-narrative-warm.ts index d162d8506..51aa05dd9 100644 --- a/src/lib/jobs/period-narrative-warm.ts +++ b/src/lib/jobs/period-narrative-warm.ts @@ -37,11 +37,13 @@ import type { NarrativePeriod } from "@/lib/insights/narrative/period-narrative" import { defaultLocale } from "@/lib/i18n/config"; import { resolveJobLocale } from "@/lib/i18n/job-locale"; import { - PERIOD_NARRATIVE_QUEUE, - PERIOD_NARRATIVE_CRON, enqueueNarrativeWarm, type PeriodNarrativePayload, } from "@/lib/jobs/period-narrative-shared"; +import { + PERIOD_NARRATIVE_QUEUE, + PERIOD_NARRATIVE_CRON, +} from "@/lib/jobs/period-narrative-queue"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; export { PERIOD_NARRATIVE_QUEUE, PERIOD_NARRATIVE_CRON, enqueueNarrativeWarm }; diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 80b1feeb1..071481bfe 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -6,7 +6,7 @@ * it, and it does not stop the handler, so a job that legitimately needs longer * ends up running twice beside itself. No queue policy prevents that: the * policies constrain job rows, and an expired row is no longer active. Every - * queue bound with `createAndWork` is therefore declared here as one of: + * queue the worker registry declares is therefore declared here as one of: * * - `short`: a job cannot come near fifteen minutes (one account's bounded * window, one document, a forwarder, a bounded delete). It relies on the @@ -28,15 +28,15 @@ import { INTEGRATION_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/integration-backfill-admission"; import { NIGHTLY_INSIGHT_PASS_EXPIRE_SECONDS } from "@/lib/jobs/reminder/registrar-shared"; import { INSIGHT_PREGENERATE_EXPIRE_SECONDS } from "@/lib/jobs/insight-pregenerate-shared"; -import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "@/lib/jobs/period-narrative-shared"; +import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "@/lib/jobs/period-narrative-queue"; import { PR_DETECTION_EXPIRE_SECONDS } from "@/lib/jobs/pr-detection"; import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key-rotate"; import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill"; import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run"; import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy"; -import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup"; +import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue"; import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore"; -import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill"; +import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue"; import { APPLE_HEALTH_IMPORT_SEND_OPTIONS } from "@/lib/jobs/apple-health-import-worker"; import { MEDICATION_INTAKE_IMPORT_SEND_OPTIONS } from "@/lib/jobs/medication-intake-import"; import { HEALTH_CONNECT_IMPORT_SEND_OPTIONS } from "@/lib/jobs/health-connect-import-worker"; @@ -47,8 +47,8 @@ import { ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS } from "@/lib/rollups/measurement-r /** pg-boss's default job expiry, in seconds. */ export const DEFAULT_JOB_EXPIRE_SECONDS = 15 * 60; -/** Where a check lives: `binding` is the `createAndWork` call itself. */ -export type SourceSite = "binding" | { file: string; fn: string }; +/** Where a check lives: a function (or const) in a file under `src/`. */ +export type SourceSite = { file: string; fn: string }; export type QueueRuntime = | { runtime: "short"; why: string } @@ -64,8 +64,9 @@ export type QueueRuntime = /** Where the handler stops on the job's budget, or `oneShot`. */ stop: SourceSite | { oneShot: string }; /** - * What keeps a second run from working beside this one: `lockedPass` - * at the binding, or a marker in the named function. + * What keeps a second run from working beside this one: the `lock` + * its registry definition declares (`lockedPass`), or a marker in the + * named function. */ exclusive: "lockedPass" | { marker: string; at: SourceSite }; why: string; @@ -177,7 +178,10 @@ export const QUEUE_RUNTIME: Readonly> = { runtime: "long", expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS, expiryVia: ["PERIOD_NARRATIVE_EXPIRE_SECONDS"], - stop: "binding", + stop: { + file: "lib/jobs/reminder/status-handlers.ts", + fn: "handlePeriodNarrative", + }, exclusive: "lockedPass", why: "On a boundary night up to 200 accounts with a provider call per period each; a single-account warm is short.", }, @@ -198,7 +202,10 @@ export const QUEUE_RUNTIME: Readonly> = { runtime: "long", expireInSeconds: ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS, expiryVia: ["ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS"], - stop: "binding", + stop: { + file: "lib/jobs/reminder/rollup-handlers.ts", + fn: "handleRollupFullBackfill", + }, exclusive: "lockedPass", why: "One account's five-year fold: four aggregates under the statement timeout and four persist transactions of up to 120 s each, about thirteen minutes at worst under the defaults and unbounded when the operator lifts the statement timeout.", }, @@ -220,22 +227,34 @@ export const QUEUE_RUNTIME: Readonly> = { ), "mean-consolidation": { runtime: "budgeted", - budget: "binding", + budget: { + file: "lib/jobs/reminder/rollup-handlers.ts", + fn: "handleMeanConsolidation", + }, why: "One account's whole history, a day at a time; stops at three quarters of the default expiry and resumes.", }, "dense-intraday-retention": { runtime: "budgeted", - budget: "binding", + budget: { + file: "lib/jobs/reminder/rollup-handlers.ts", + fn: "handleDenseIntradayRetention", + }, why: "A dense account's history, a day at a time; stops at three quarters of the default expiry and resumes.", }, "dense-intraday-hourly-rebuild": { runtime: "budgeted", - budget: "binding", + budget: { + file: "lib/jobs/reminder/rollup-handlers.ts", + fn: "handleDenseIntradayHourlyRebuild", + }, why: "A dense account's hourly rows, a day at a time; stops at three quarters of the default expiry and resumes.", }, "drain-per-sample-cumulative": { runtime: "budgeted", - budget: "binding", + budget: { + file: "lib/jobs/reminder/rollup-handlers.ts", + fn: "drainCumulativeHandler", + }, why: "Every account's per-sample drains; stops at three quarters of the default expiry and hands the rest to a continuation job.", }, @@ -471,7 +490,10 @@ export const QUEUE_RUNTIME: Readonly> = { runtime: "long", expireInSeconds: CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS, expiryVia: ["CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS"], - stop: "binding", + stop: { + file: "lib/jobs/reminder/document-handlers.ts", + fn: "handleContentIndexBackfill", + }, exclusive: "lockedPass", why: "Up to 200 documents of one account, a provider transcription each.", }, diff --git a/src/lib/jobs/recovery-score-queue.ts b/src/lib/jobs/recovery-score-queue.ts new file mode 100644 index 000000000..181e66c5f --- /dev/null +++ b/src/lib/jobs/recovery-score-queue.ts @@ -0,0 +1,14 @@ +/** + * The queue facts of `recovery-score.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const RECOVERY_SCORE_QUEUE = "recovery-score-compute"; + +/** + * Daily at 04:45 Europe/Berlin — inside the existing 03:xx–04:xx maintenance + * window, after the cumulative drain (03:45) + the daily-mean consolidation + * and the insight pre-generation (04:30) so the nightly signals it reads are + * already folded into the rollup tier by the time it runs. + */ +export const RECOVERY_SCORE_CRON = "45 4 * * *"; diff --git a/src/lib/jobs/recovery-score.ts b/src/lib/jobs/recovery-score.ts index cc2a2fb2f..c0a4a796f 100644 --- a/src/lib/jobs/recovery-score.ts +++ b/src/lib/jobs/recovery-score.ts @@ -24,16 +24,6 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { persistRecoveryScore } from "@/lib/insights/recovery-score"; import { runScoreBatch } from "@/lib/insights/score-row"; -export const RECOVERY_SCORE_QUEUE = "recovery-score-compute"; - -/** - * Daily at 04:45 Europe/Berlin — inside the existing 03:xx–04:xx maintenance - * window, after the cumulative drain (03:45) + the daily-mean consolidation - * and the insight pre-generation (04:30) so the nightly signals it reads are - * already folded into the rollup tier by the time it runs. - */ -export const RECOVERY_SCORE_CRON = "45 4 * * *"; - /** * Per-run user cap. A nightly fan-out across every user could starve the * pool; the discovery query is ordered so users with the most recent vital diff --git a/src/lib/jobs/registry/__tests__/registry-contract.test.ts b/src/lib/jobs/registry/__tests__/registry-contract.test.ts new file mode 100644 index 000000000..afcbdc995 --- /dev/null +++ b/src/lib/jobs/registry/__tests__/registry-contract.test.ts @@ -0,0 +1,129 @@ +/** + * The contract of the worker registry, checked over every declaration. + * + * The dead-queue class (a queue named but never created, scheduled or + * drained) cannot be written any more: a definition is the queue, its cron, + * its policy and its handler in one object, and `registerJobGroup` turns each + * into all four. What is left to check is what the type system cannot see: + * + * - every queue is declared once, under a well-formed name, and no retired + * queue is declared again; + * - every cron has five fields, every policy names one this codebase uses + * and says why; + * - the groups register in the order the worker always used; + * - loading the registry loads none of the work: its static import graph + * stays inside a small budget and reaches no handler module. A handler is + * named only through its definition's dynamic `import("…")`. + * + * What pg-boss is left holding after registration is pinned byte for byte by + * `tests/integration/queue-registration-parity.test.ts`, which also loads + * every handler once. + */ +import { describe, expect, it } from "vitest"; + +import { staticImportClosure } from "@/__tests__/helpers/source-metrics"; +import { JOB_GROUP_NAMES } from "../define-job"; +import { allJobDefinitions, JOB_GROUPS } from "../index"; + +const definitions = allJobDefinitions(); + +/** Modules the registry may not load: the work the definitions point at. */ +const HANDLER_GRAPH = + /^lib\/(?:jobs\/reminder\/(?!registrar-shared\.ts|shared\.ts)|ai\/|insights\/|withings\/|whoop\/|oura\/|polar\/|strava\/|google-health\/|nightscout\/|dawarich\/|documents\/|export\/|import\/|measurements\/import)/; + +/** + * Light modules inside those folders the registry may load, with the reason. + */ +const ALLOWED: Readonly> = { + "lib/ai/coach/coach-memory-shared.ts": + "the Coach memory queue's name and payload, with its byte codec; no model code", + "lib/ai/coach/bytes-codec.ts": "pulled by coach-memory-shared; a codec", + "lib/export/restore-lock.ts": + "the per-account restore lock the consolidation passes take; a lock primitive", + "lib/export/restore-failure-cause.ts": + "pulled by restore-lock; an error type", +}; + +/** + * The registry's static graph: the declarations, the light queue-fact + * modules and the boot infrastructure they share (the database client, the + * logger, the lane and the lock). Measured at 288 on 2026-10-23, against the + * 1,164 modules the five hand-written registrars loaded; it may only fall. + */ +const REGISTRY_GRAPH_BUDGET = 300; + +describe("worker registry", () => { + it("declares every queue once, under a well-formed name", () => { + expect(definitions.length).toBeGreaterThan(100); + const names = definitions.map((d) => d.queue); + const twice = names.filter((n, i) => names.indexOf(n) !== i); + expect(twice).toEqual([]); + expect(names.filter((n) => !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(n))).toEqual( + [], + ); + }); + + it("declares no retired queue again", () => { + const declared = new Set(definitions.map((d) => d.queue)); + const retired = JOB_GROUPS.flatMap((g) => g.retired ?? []); + expect(retired.length).toBeGreaterThan(0); + expect(retired.filter((q) => declared.has(q))).toEqual([]); + }); + + it("gives every cron five fields and every policy a known kind and a reason", () => { + const crons = definitions.filter((d) => d.cron); + expect(crons.length).toBeGreaterThan(60); + expect( + crons + .filter((d) => d.cron!.trim().split(/\s+/).length !== 5) + .map((d) => d.queue), + ).toEqual([]); + const policies = definitions.filter((d) => d.policy); + expect(policies.length).toBeGreaterThan(30); + expect( + policies + .filter( + (d) => + !["short", "exclusive"].includes(d.policy!.policy) || + d.policy!.reason.length < 30, + ) + .map((d) => d.queue), + ).toEqual([]); + }); + + it("binds every queue with an explicit concurrency", () => { + expect( + definitions + .filter( + (d) => + !Number.isInteger(d.work.localConcurrency) || + (d.work.localConcurrency ?? 0) < 1, + ) + .map((d) => d.queue), + ).toEqual([]); + }); + + it("registers the groups in the worker's order", () => { + expect(JOB_GROUPS.map((g) => g.name)).toEqual([...JOB_GROUP_NAMES]); + }); + + it("loads none of the work when the registry loads", () => { + const graph = staticImportClosure(["lib/jobs/registry/index.ts"]); + // A floor, so a walk that resolves nothing cannot pass. + expect(graph.size).toBeGreaterThan(20); + expect(graph.has("lib/jobs/registry/maintenance.ts")).toBe(true); + expect( + [...graph].filter((m) => HANDLER_GRAPH.test(m) && !(m in ALLOWED)).sort(), + ).toEqual([]); + expect(graph.size).toBeLessThanOrEqual(REGISTRY_GRAPH_BUDGET); + }); + + it("catches a handler module in the registry graph (planted)", () => { + // The same check over the graph a registrar that imported its handlers + // would have. + const graph = staticImportClosure(["lib/jobs/reminder/status-handlers.ts"]); + expect( + [...graph].filter((m) => HANDLER_GRAPH.test(m)).length, + ).toBeGreaterThan(0); + }); +}); diff --git a/src/lib/jobs/registry/__tests__/registry-helpers.ts b/src/lib/jobs/registry/__tests__/registry-helpers.ts new file mode 100644 index 000000000..eba79f863 --- /dev/null +++ b/src/lib/jobs/registry/__tests__/registry-helpers.ts @@ -0,0 +1,61 @@ +/** + * Reading the worker registry from a test: the declaration of one queue, the + * group that holds it, the handler its loader resolves to, and a group's boot + * discovery passes. Every helper fails loudly on a queue the registry does + * not declare, so a test cannot pass by asserting on nothing. + */ +import type { PgBoss } from "pg-boss"; + +import { JOB_GROUPS } from "../index"; +import type { JobDefinition, JobGroup, JobGroupName } from "../define-job"; + +export function jobOf(queue: string): JobDefinition { + for (const group of JOB_GROUPS) { + const job = group.jobs.find((j) => j.queue === queue); + if (job) return job; + } + throw new Error(`the registry declares no queue "${queue}"`); +} + +export function groupOf(queue: string): JobGroup { + const group = JOB_GROUPS.find((g) => g.jobs.some((j) => j.queue === queue)); + if (!group) throw new Error(`the registry declares no queue "${queue}"`); + return group; +} + +export function groupNamed(name: JobGroupName): JobGroup { + const group = JOB_GROUPS.find((g) => g.name === name); + if (!group) throw new Error(`no registry group "${name}"`); + return group; +} + +/** The names of a group's boot discovery passes, in run order. */ +export function discoveryOf(name: JobGroupName): string[] { + return (groupNamed(name).discovery ?? []).map((pass) => pass.name); +} + +/** + * A boss that records sends and refuses everything else, for loaders that + * close over it (the forwarders, the drain's follow-up). + */ +export function recordingBoss(): { + boss: PgBoss; + sends: Array<{ queue: string; data: unknown; options: unknown }>; +} { + const sends: Array<{ queue: string; data: unknown; options: unknown }> = []; + const boss = { + send: async (queue: string, data: unknown, options: unknown) => { + sends.push({ queue, data, options }); + return `job-${sends.length}`; + }, + } as unknown as PgBoss; + return { boss, sends }; +} + +/** The handler the queue's loader resolves to (its module is loaded). */ +export async function loadedHandler( + queue: string, + boss: PgBoss = recordingBoss().boss, +): Promise { + return jobOf(queue).load({ boss }); +} diff --git a/src/lib/jobs/registry/define-job.ts b/src/lib/jobs/registry/define-job.ts new file mode 100644 index 000000000..acaa67254 --- /dev/null +++ b/src/lib/jobs/registry/define-job.ts @@ -0,0 +1,109 @@ +/** + * The declaration of one pg-boss queue: everything the worker needs to + * provision, schedule and drain it, in one object. + * + * Every queue the worker runs is a `defineJob(...)` entry in one of the + * registry groups (`./integration-sync.ts`, `./status.ts`, `./rollup.ts`, + * `./reminders.ts`, `./maintenance.ts`). `registerJobGroup` turns a group into + * the four facts the dead-queue class keeps losing: the queue exists, its cron + * is scheduled with the send options it carries, its de-duplication policy is + * set, and a handler drains it. A queue cannot be declared without all four. + * + * The handler is not imported here. `load` names the module it lives in with + * a dynamic import, and the worker evaluates it when the queue's first job + * arrives, so booting the worker loads the registry and nothing of the work. + * The facts a declaration needs at boot (names, crons, concurrency, send + * options) therefore come from light modules (`*-queue.ts`, `*-shared.ts`), + * never from a handler module; `job-registry-graph.test.ts` holds that. + */ +import type { Job, PgBoss, WorkOptions } from "pg-boss"; + +import type { JobOutcome } from "@/lib/jobs/job-outcome"; +import type { QueuePolicyDecision } from "@/lib/jobs/reminder/registrar-shared"; + +/** The groups, in the order the worker registers them. */ +export const JOB_GROUP_NAMES = [ + "integration_sync", + "status", + "rollup", + "reminders", + "maintenance", +] as const; + +export type JobGroupName = (typeof JOB_GROUP_NAMES)[number]; + +/** What a handler loader is handed: the worker's own boss, for re-sends. */ +export interface JobContext { + readonly boss: PgBoss; +} + +/** A handler as pg-boss calls it: a batch of jobs, one outcome. */ +export type BatchHandler> = ( + jobs: J[], +) => Promise; + +export interface JobSpec> { + readonly queue: string; + /** The cron that feeds the queue, in the default zone. */ + readonly cron?: string; + /** + * Send options every job the cron mints carries (retries, expiry). They ride + * on the schedule because `createQueue` cannot change an existing queue. + */ + readonly scheduleOptions?: Readonly>; + /** De-duplication policy; absent keeps pg-boss's `standard`. */ + readonly policy?: QueuePolicyDecision; + /** Work options of the binding (concurrency, metadata, poll). */ + readonly work: WorkOptions; + /** + * One run per identity at a time (`lockedPass`), held outside the lane so a + * retry pg-boss starts after an expiry finds the run still going. + */ + readonly lock?: (job: J) => string; + /** + * The job runs in the heavy lane (`heavyLane`): `true` for every job, or a + * filter of the jobs that are heavy on a queue that also carries light ones. + */ + readonly lane?: true | ((job: J) => boolean); + /** Loads the handler that drains the queue. */ + readonly load: (ctx: JobContext) => Promise>; +} + +/** A declaration with its payload type erased, as the registry holds it. */ +export type JobDefinition = JobSpec>; + +export function defineJob>( + spec: JobSpec, +): JobDefinition { + // The payload type is the declaration's own business; the registry only + // passes jobs through to the handler that declared it. + return spec as unknown as JobDefinition; +} + +/** What a boot discovery pass reports, as the backfill helpers return it. */ +export interface DiscoveryResult { + /** How many jobs it enqueued, or whether it enqueued its one job. */ + readonly enqueued: number | boolean; + readonly skipped?: number; + readonly error?: string | null; +} + +/** + * A self-converging pass the worker runs once at boot (after every queue is + * bound): it finds the accounts that still need a backfill or repair and + * enqueues one job each. `name` is the log prefix. + */ +export interface BootDiscovery { + readonly name: string; + readonly run: () => Promise; +} + +export interface JobGroup { + readonly name: JobGroupName; + readonly jobs: readonly JobDefinition[]; + /** Queues an older release created and this one removes, with whatever they hold. */ + readonly retired?: readonly string[]; + /** Work that needs the group's handlers bound, before the boot discovery. */ + readonly afterBind?: (ctx: JobContext) => Promise; + readonly discovery?: readonly BootDiscovery[]; +} diff --git a/src/lib/jobs/registry/index.ts b/src/lib/jobs/registry/index.ts new file mode 100644 index 000000000..a6d02a764 --- /dev/null +++ b/src/lib/jobs/registry/index.ts @@ -0,0 +1,55 @@ +/** + * The worker registry: every queue the worker runs, by group, in the order + * the worker registers them. `reminder-worker.ts` walks it at boot; + * `tests/integration/queue-registration-parity.test.ts` holds what it leaves + * in pg-boss to the golden. + */ +import type { PgBoss } from "pg-boss"; + +import type { JobDefinition, JobGroup } from "./define-job"; +import { INTEGRATION_SYNC_GROUP } from "./integration-sync"; +import { MAINTENANCE_GROUP } from "./maintenance"; +import { REMINDERS_GROUP } from "./reminders"; +import { registerJobGroup, runBootDiscovery } from "./register"; +import { ROLLUP_GROUP } from "./rollup"; +import { STATUS_GROUP } from "./status"; + +export const JOB_GROUPS: readonly JobGroup[] = [ + INTEGRATION_SYNC_GROUP, + STATUS_GROUP, + ROLLUP_GROUP, + REMINDERS_GROUP, + MAINTENANCE_GROUP, +]; + +/** Every definition, across the groups. */ +export function allJobDefinitions(): readonly JobDefinition[] { + return JOB_GROUPS.flatMap((group) => group.jobs); +} + +/** + * Register every group in order. `onGroup` runs before each one, so the boot + * can time the groups as steps. Returns every queue name created. + */ +export async function registerAllJobGroups( + boss: PgBoss, + onGroup: (group: JobGroup) => void = () => {}, +): Promise { + const queues: string[] = []; + for (const group of JOB_GROUPS) { + onGroup(group); + queues.push(...(await registerJobGroup(boss, group))); + } + return queues; +} + +/** + * Run the boot discovery of every group: rollup first, then the integration + * backfills, then maintenance, the order the worker always used. + */ +export async function runAllBootDiscovery(): Promise { + for (const name of ["rollup", "integration_sync", "maintenance"] as const) { + const group = JOB_GROUPS.find((g) => g.name === name); + if (group?.discovery) await runBootDiscovery(group.discovery); + } +} diff --git a/src/lib/jobs/registry/integration-sync.ts b/src/lib/jobs/registry/integration-sync.ts new file mode 100644 index 000000000..d5ec529a3 --- /dev/null +++ b/src/lib/jobs/registry/integration-sync.ts @@ -0,0 +1,391 @@ +/** + * The third-party provider sync queues: Withings, WHOOP, Fitbit, Google + * Health, Nightscout, Polar, Oura, Strava, the full-history backfills and + * repairs that run through the shared admission lane, the integration + * watchdog, and the OAuth-state and handoff prunes. + * + * The recurring polls are keyless cron ticks, so they carry no policy. The + * one-shot backfills and repairs share one shape: one job per connection, + * keyed per user, enqueued by a boot discovery that only stops offering the + * job once it has completed. `exclusive` covers queued, active and + * retry-backoff, so a worker restarting during a long pass cannot append a + * second identical job, and since discovery re-offers outstanding work a + * suppressed send never loses any. + * + * The prunes carry `cronIsTheRetry`: a DELETE that could not run will not run + * better a second later, and the daily tick is the retry. The sync crons keep + * the default retries; a provider outage is transient. + */ +import { + FITBIT_BOOT_DISCOVERY, + FITBIT_JOBS, +} from "@/lib/fitbit/jobs/definitions"; +import { + INTEGRATION_BACKFILL_ADMISSION_QUEUE, + INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, + bootStaggerSecondsFor, + type IntegrationBackfillKind, +} from "@/lib/jobs/integration-backfill-admission"; +import { + GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON, + GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, + GOOGLE_HEALTH_SYNC_CRON, + GOOGLE_HEALTH_SYNC_QUEUE, + NIGHTSCOUT_SYNC_CRON, + NIGHTSCOUT_SYNC_QUEUE, + OIDC_NATIVE_HANDOFF_CLEANUP_CRON, + OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, + OURA_SYNC_CRON, + OURA_SYNC_QUEUE, + POLAR_SYNC_CRON, + POLAR_SYNC_QUEUE, + STRAVA_SYNC_CRON, + STRAVA_SYNC_QUEUE, + WHOOP_CYCLE_SYNC_CRON, + WHOOP_CYCLE_SYNC_QUEUE, + WHOOP_OAUTH_STATE_CLEANUP_CRON, + WHOOP_OAUTH_STATE_CLEANUP_QUEUE, + WHOOP_RECOVERY_SYNC_CRON, + WHOOP_RECOVERY_SYNC_QUEUE, + WHOOP_SLEEP_SYNC_CRON, + WHOOP_SLEEP_SYNC_QUEUE, + WHOOP_WORKOUT_SYNC_CRON, + WHOOP_WORKOUT_SYNC_QUEUE, + WITHINGS_ACTIVITY_CRON, + WITHINGS_ACTIVITY_QUEUE, + WITHINGS_ECG_SYNC_CRON, + WITHINGS_OAUTH_STATE_CLEANUP_CRON, + WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, + WITHINGS_SLEEP_CRON, + WITHINGS_SLEEP_QUEUE, + WITHINGS_SYNC_CRON, + WITHINGS_SYNC_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; +import { + INTEGRATION_WATCHDOG_DAILY_CRON, + INTEGRATION_WATCHDOG_DAILY_QUEUE, + INTEGRATION_WATCHDOG_HOURLY_CRON, + INTEGRATION_WATCHDOG_HOURLY_QUEUE, +} from "@/lib/jobs/integration-watchdog-queue"; +import { + GOOGLE_HEALTH_BACKFILL_CONCURRENCY, + GOOGLE_HEALTH_BACKFILL_QUEUE, +} from "@/lib/jobs/google-health-backfill-queue"; +import { + GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY, + GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, +} from "@/lib/jobs/google-health-sleep-repair-queue"; +import { + LAB_BIOMARKER_BACKFILL_CONCURRENCY, + LAB_BIOMARKER_BACKFILL_QUEUE, +} from "@/lib/jobs/lab-biomarker-backfill"; +import { + OURA_BACKFILL_CONCURRENCY, + OURA_BACKFILL_QUEUE, +} from "@/lib/jobs/oura-backfill-queue"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + SLEEP_TIMELINE_BACKFILL_CONCURRENCY, + SLEEP_TIMELINE_BACKFILL_QUEUE, +} from "@/lib/jobs/sleep-timeline-backfill-queue"; +import { + STRAVA_BACKFILL_CONCURRENCY, + STRAVA_BACKFILL_QUEUE, +} from "@/lib/jobs/strava-backfill-queue"; +import { + WHOOP_BACKFILL_CONCURRENCY, + WHOOP_BACKFILL_QUEUE, +} from "@/lib/jobs/whoop-backfill-queue"; +import { WITHINGS_ECG_SYNC_QUEUE } from "@/lib/jobs/withings-ecg-queue"; + +import { + defineJob, + type BootDiscovery, + type DiscoveryResult, + type JobDefinition, + type JobGroup, +} from "./define-job"; + +const SELF_CONVERGING_BACKFILL = + "Per-connection full-history backfill; boot-discovery driven and self-converging."; + +/** A source queue that forwards each job to the shared admission lane. */ +function backfillSource( + queue: string, + concurrency: number, + kind: IntegrationBackfillKind, + reason: string = SELF_CONVERGING_BACKFILL, +): JobDefinition { + return defineJob({ + queue, + policy: { policy: "exclusive", reason }, + work: { localConcurrency: concurrency }, + load: async ({ boss }) => + ( + await import("@/lib/jobs/integration-backfill-forward") + ).forwardToBackfillAdmission(boss, kind), + }); +} + +const withingsSync = () => import("@/lib/jobs/reminder/withings-sync"); +const whoopSync = () => import("@/lib/jobs/reminder/whoop-sync"); +const googleHealthSync = () => import("@/lib/jobs/reminder/google-health-sync"); +const cleanups = () => import("@/lib/jobs/reminder/cleanup-handlers"); + +const jobs: readonly JobDefinition[] = [ + // pg-boss scopes groupConcurrency to one queue name, so every provider + // reaches this shared queue before doing full-history work. The group is + // enforced in Postgres across workers; the heavy lane keeps an import off + // the heap while a backup, a restore or an upload is on it. + defineJob({ + queue: INTEGRATION_BACKFILL_ADMISSION_QUEUE, + policy: { + policy: "exclusive", + reason: + "Shared full-history drain; keyed per source job so boot rediscovery cannot duplicate queued, active, or retrying work.", + }, + work: { + localConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, + groupConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, + }, + lane: true, + load: async () => + (await import("@/lib/jobs/integration-backfill-drain")) + .drainIntegrationBackfillAdmission, + }), + defineJob({ + queue: WITHINGS_SYNC_QUEUE, + cron: WITHINGS_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await withingsSync()).handleWithingsFallbackSync, + }), + defineJob({ + queue: WITHINGS_ACTIVITY_QUEUE, + cron: WITHINGS_ACTIVITY_CRON, + work: { localConcurrency: 1 }, + load: async () => (await withingsSync()).handleWithingsActivitySync, + }), + defineJob({ + queue: WITHINGS_SLEEP_QUEUE, + cron: WITHINGS_SLEEP_CRON, + work: { localConcurrency: 1 }, + load: async () => (await withingsSync()).handleWithingsSleepSync, + }), + defineJob({ + queue: WITHINGS_ECG_SYNC_QUEUE, + cron: WITHINGS_ECG_SYNC_CRON, + policy: { + policy: "short", + reason: + "Webhook retries share one identity while queued; once work starts, a later replay remains admissible as a rescue if the original exhausts retries.", + }, + work: { localConcurrency: 1 }, + load: async () => (await withingsSync()).handleWithingsEcgSync, + }), + defineJob({ + queue: WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, + cron: WITHINGS_OAUTH_STATE_CLEANUP_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => (await cleanups()).handleWithingsOAuthStateCleanup, + }), + // WHOOP per-resource syncs: a webhook job is one user, the cron walks + // everyone. Serial, inside WHOOP's 100 requests a minute per app. + defineJob({ + queue: WHOOP_RECOVERY_SYNC_QUEUE, + cron: WHOOP_RECOVERY_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await whoopSync()).handleWhoopRecoverySync, + }), + defineJob({ + queue: WHOOP_SLEEP_SYNC_QUEUE, + cron: WHOOP_SLEEP_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await whoopSync()).handleWhoopSleepSync, + }), + defineJob({ + queue: WHOOP_WORKOUT_SYNC_QUEUE, + cron: WHOOP_WORKOUT_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await whoopSync()).handleWhoopWorkoutSync, + }), + defineJob({ + queue: WHOOP_CYCLE_SYNC_QUEUE, + cron: WHOOP_CYCLE_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await whoopSync()).handleWhoopCycleSync, + }), + backfillSource( + WHOOP_BACKFILL_QUEUE, + WHOOP_BACKFILL_CONCURRENCY, + "whoop-backfill", + "Per-connection full-history backfill; long-running, and boot discovery re-enqueues until the connection is marked backfilled.", + ), + defineJob({ + queue: WHOOP_OAUTH_STATE_CLEANUP_QUEUE, + cron: WHOOP_OAUTH_STATE_CLEANUP_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => (await cleanups()).handleWhoopOAuthStateCleanup, + }), + ...FITBIT_JOBS, + defineJob({ + queue: GOOGLE_HEALTH_SYNC_QUEUE, + cron: GOOGLE_HEALTH_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await googleHealthSync()).handleGoogleHealthSync, + }), + backfillSource( + GOOGLE_HEALTH_BACKFILL_QUEUE, + GOOGLE_HEALTH_BACKFILL_CONCURRENCY, + "google-health-backfill", + ), + backfillSource( + GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, + GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY, + "google-health-sleep-repair", + "Per-connection one-shot sleep re-read; discovery drops the connection once it is marked repaired.", + ), + defineJob({ + queue: GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, + cron: GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await googleHealthSync()).handleGoogleHealthOAuthStateCleanup, + }), + backfillSource( + SLEEP_TIMELINE_BACKFILL_QUEUE, + SLEEP_TIMELINE_BACKFILL_CONCURRENCY, + "sleep-timeline-backfill", + "Per-connection one-shot timeline backfill, keyed per provider+user; discovery drops the connection once its rows carry the new shape.", + ), + backfillSource( + LAB_BIOMARKER_BACKFILL_QUEUE, + LAB_BIOMARKER_BACKFILL_CONCURRENCY, + "lab-biomarker-backfill", + "Per-user one-shot catalog link-up; discovery drops the user once no unlinked lab readings remain.", + ), + // The cohort polls: the hourly tick (no `userId`) walks every connection; + // one connection's failure is booked on its ledger, not the tick's. + defineJob({ + queue: NIGHTSCOUT_SYNC_QUEUE, + cron: NIGHTSCOUT_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/nightscout-sync")) + .handleNightscoutSync, + }), + defineJob({ + queue: POLAR_SYNC_QUEUE, + cron: POLAR_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/polar-sync")).handlePolarSync, + }), + defineJob({ + queue: OURA_SYNC_QUEUE, + cron: OURA_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/oura-sync")).handleOuraSync, + }), + backfillSource( + OURA_BACKFILL_QUEUE, + OURA_BACKFILL_CONCURRENCY, + "oura-backfill", + ), + defineJob({ + queue: STRAVA_SYNC_QUEUE, + cron: STRAVA_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/strava-sync")).handleStravaSync, + }), + backfillSource( + STRAVA_BACKFILL_QUEUE, + STRAVA_BACKFILL_CONCURRENCY, + "strava-backfill", + ), + defineJob({ + queue: OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, + cron: OIDC_NATIVE_HANDOFF_CLEANUP_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => (await cleanups()).handleOidcNativeHandoffCleanup, + }), + // The watchdog: an hourly cheap pass and a daily one that re-reads the + // history behind standing silence flags. Both share one advisory lock, so + // they never run side by side on any replica; the next tick is the retry. + defineJob({ + queue: INTEGRATION_WATCHDOG_HOURLY_QUEUE, + cron: INTEGRATION_WATCHDOG_HOURLY_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/integration-watchdog")) + .handleIntegrationWatchdogHourly, + }), + defineJob({ + queue: INTEGRATION_WATCHDOG_DAILY_QUEUE, + cron: INTEGRATION_WATCHDOG_DAILY_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/integration-watchdog")) + .handleIntegrationWatchdogDaily, + }), +]; + +/** A backfill discovery staggered by its kind's place in the boot order. */ +function backfillDiscovery( + kind: IntegrationBackfillKind, + enqueue: (staggerSeconds: number) => Promise, +): BootDiscovery { + return { name: kind, run: () => enqueue(bootStaggerSecondsFor(kind)) }; +} + +const discovery: readonly BootDiscovery[] = [ + backfillDiscovery("whoop-backfill", async (stagger) => + (await import("@/lib/jobs/whoop-backfill")).enqueueBootTimeWhoopBackfill( + stagger, + ), + ), + backfillDiscovery("google-health-backfill", async (stagger) => + ( + await import("@/lib/jobs/google-health-backfill") + ).enqueueBootTimeGoogleHealthBackfill(stagger), + ), + backfillDiscovery("google-health-sleep-repair", async (stagger) => + ( + await import("@/lib/jobs/google-health-sleep-repair") + ).enqueueBootTimeGoogleHealthSleepRepair(stagger), + ), + backfillDiscovery("sleep-timeline-backfill", async (stagger) => + ( + await import("@/lib/jobs/sleep-timeline-backfill") + ).enqueueBootTimeSleepTimelineBackfill(stagger), + ), + backfillDiscovery("strava-backfill", async (stagger) => + (await import("@/lib/jobs/strava-backfill")).enqueueBootTimeStravaBackfill( + stagger, + ), + ), + backfillDiscovery("oura-backfill", async (stagger) => + (await import("@/lib/jobs/oura-backfill")).enqueueBootTimeOuraBackfill( + stagger, + ), + ), + backfillDiscovery("lab-biomarker-backfill", async (stagger) => + ( + await import("@/lib/jobs/lab-biomarker-backfill") + ).enqueueBootTimeLabBiomarkerBackfill(stagger), + ), + ...FITBIT_BOOT_DISCOVERY, +]; + +export const INTEGRATION_SYNC_GROUP: JobGroup = { + name: "integration_sync", + jobs, + discovery, +}; diff --git a/src/lib/jobs/registry/maintenance-discovery.ts b/src/lib/jobs/registry/maintenance-discovery.ts new file mode 100644 index 000000000..aa137ea13 --- /dev/null +++ b/src/lib/jobs/registry/maintenance-discovery.ts @@ -0,0 +1,100 @@ +/** + * The boot discovery of the maintenance group: the duplicate dose-slot + * cleanup, the encryption backfills, the thumbnail backfill, the air-quality + * history, the fold repair and the purge that waits for it, the GeoLite2 + * fetch, and the detached climb backfill. Each pass is idempotent across + * boots; the heavy ones start past the boot storm. + */ +import { BOOT_BACKFILL_STAGGER_SECONDS } from "@/lib/jobs/reminder/shared"; + +import type { BootDiscovery } from "./define-job"; + +export const MAINTENANCE_DISCOVERY: readonly BootDiscovery[] = [ + { + name: "intake-slot-dedup", + run: async () => + ( + await import("@/lib/medications/intake-slot-dedup") + ).enqueueBootTimeIntakeSlotDedup(), + }, + // The heavy loads start after the boot storm; the daily ticks pass no + // offset and stay immediate. + { + name: "note-encryption-backfill", + run: async () => + ( + await import("@/lib/jobs/note-encryption-backfill") + ).enqueueBootTimeNoteEncryptionBackfill( + BOOT_BACKFILL_STAGGER_SECONDS * 5, + ), + }, + { + name: "med-notes-encryption-backfill", + run: async () => + ( + await import("@/lib/jobs/med-notes-encryption-backfill") + ).enqueueBootTimeMedNotesEncryptionBackfill(), + }, + { + name: "free-text-encryption-backfill", + run: async () => + ( + await import("@/lib/jobs/free-text-encryption-backfill") + ).enqueueBootTimeFreeTextEncryptionBackfill( + BOOT_BACKFILL_STAGGER_SECONDS * 6, + ), + }, + { + name: "document-thumbnail-backfill", + run: async () => { + const { enqueued } = await ( + await import("@/lib/jobs/document-thumbnail-backfill") + ).enqueueBootTimeThumbnailBackfill(); + return { enqueued }; + }, + }, + { + name: "environment-aq-history", + run: async () => + ( + await import("@/lib/jobs/environment-air-quality-history") + ).enqueueBootTimeAirQualityHistory(BOOT_BACKFILL_STAGGER_SECONDS * 7), + }, + // Once per account; the purge below waits for it. + { + name: "measurement-fold-repair", + run: async () => + ( + await import("@/lib/jobs/measurement-fold-repair") + ).enqueueBootTimeMeasurementFoldRepair(), + }, + // Self-limiting and idempotent; a boot with no backlog costs one query. + { + name: "compaction-tombstone-purge", + run: async () => { + const { enqueued } = await ( + await import("@/lib/jobs/compaction-tombstone-purge") + ).enqueueBootTimeCompactionTombstonePurge(); + return { enqueued }; + }, + }, + // Only with a MaxMind key, egress allowed and no offline database present. + { + name: "geolite2-fetch", + run: async () => { + const { enqueued } = await ( + await import("@/lib/jobs/geolite2-fetch") + ).enqueueGeolite2FetchBootDiscovery(); + return { enqueued }; + }, + }, + // Detached: reads one track at a time with a pause between batches and + // resumes from its cursor on the next boot; a finished pass returns at once. + { + name: "workout-climb-backfill", + run: async () => + ( + await import("@/lib/jobs/reminder/climb-backfill-at-boot") + ).startClimbBackfill(), + }, +]; diff --git a/src/lib/jobs/registry/maintenance.ts b/src/lib/jobs/registry/maintenance.ts new file mode 100644 index 000000000..d0f2ec23e --- /dev/null +++ b/src/lib/jobs/registry/maintenance.ts @@ -0,0 +1,744 @@ +/** + * The housekeeping queues: backups (weekly in-database, nightly off-host), + * the restore drill and restores, every ledger and retention cleanup, the + * medication maintenance passes, the imports (Apple Health, Health Connect, + * medication intake), the converging encryption backfills, the document vault + * work, the environment and Dawarich context, the geo and TLS probes, PR + * detection and the measurement table maintenance. + * + * `cronIsTheRetry` (no retries) marks the passes whose failure repeats: a bulk + * DELETE or UPDATE that timed out will time out again on an immediate retry, + * and the next tick arrives with the whole gap as backoff. The transient ones + * (the TLS probe, the off-host upload, a provider fetch) keep the default + * retries. The restore drill sends `retryLimit: 0` in its own options, because + * it pages each failed account itself. + * + * Policies. The retention ticks are keyless, so they carry none. Deliberately + * left at `standard`: the environment fetch (an explicit-range backfill sends + * with no key so it always runs; a policy would merge two requested ranges), + * and the imports, the backup and PR detection, whose runs are each distinct + * work. The per-user converging backfills are `exclusive`; the per-document + * jobs and the runs that queue their own follow-up while active are `short`. + */ +import type { Job } from "pg-boss"; + +import { + ACHIEVEMENT_UNLOCK_SWEEP_CRON, + ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, +} from "@/lib/jobs/achievement-unlock-sweep-queue"; +import { + APPLE_HEALTH_IMPORT_CONCURRENCY, + APPLE_HEALTH_IMPORT_LEGACY_QUEUE, + APPLE_HEALTH_IMPORT_V2_QUEUE, + IMPORT_JOB_RECONCILE_CRON, + IMPORT_JOB_RECONCILE_QUEUE, +} from "@/lib/jobs/apple-health-import-queue"; +import { + AUTH_LEDGER_CLEANUP_CRON, + AUTH_LEDGER_CLEANUP_QUEUE, +} from "@/lib/jobs/auth-ledger-cleanup"; +import { BACKUP_RESTORE_QUEUE } from "@/lib/jobs/backup-restore-queue"; +import { COMPACTION_TOMBSTONE_PURGE_QUEUE } from "@/lib/jobs/compaction-tombstone-purge-queue"; +import { + CYCLE_PREDICTION_REFRESH_CRON, + CYCLE_PREDICTION_REFRESH_QUEUE, +} from "@/lib/jobs/cycle-prediction-refresh"; +import { + DATA_BACKUP_QUEUE, + DATA_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/data-backup-policy"; +import { + DAWARICH_SYNC_CRON, + DAWARICH_SYNC_QUEUE, +} from "@/lib/jobs/dawarich-sync-queue"; +import { + DOCUMENT_AI_RUN_CONCURRENCY, + DOCUMENT_AI_RUN_QUEUE, +} from "@/lib/jobs/document-ai-run-queue"; +import { + DOCUMENT_AI_RUN_REAPER_CRON, + DOCUMENT_AI_RUN_REAPER_QUEUE, +} from "@/lib/jobs/document-ai-run-reaper-queue"; +import { + CONTENT_INDEX_BACKFILL_CONCURRENCY, + CONTENT_INDEX_BACKFILL_QUEUE, +} from "@/lib/jobs/document-content-index-backfill-queue"; +import { + DOCUMENT_INDEX_CONCURRENCY, + DOCUMENT_INDEX_QUEUE, +} from "@/lib/jobs/document-index-queue"; +import { + DOCUMENT_PURGE_CRON, + DOCUMENT_PURGE_QUEUE, +} from "@/lib/jobs/document-purge"; +import { + DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY, + DOCUMENT_SUMMARY_CATCHUP_QUEUE, +} from "@/lib/jobs/document-summary-catchup-queue"; +import { + DOCUMENT_SUMMARY_CONCURRENCY, + DOCUMENT_SUMMARY_QUEUE, +} from "@/lib/jobs/document-summary-queue"; +import { + DOCUMENT_SUMMARY_REAPER_CRON, + DOCUMENT_SUMMARY_REAPER_QUEUE, +} from "@/lib/jobs/document-summary-reaper-queue"; +import { + DOCUMENT_THUMBNAIL_CONCURRENCY, + DOCUMENT_THUMBNAIL_QUEUE, +} from "@/lib/jobs/document-thumbnail-queue"; +import { + DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY, + DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, +} from "@/lib/jobs/document-thumbnail-backfill-queue"; +import { + ENCRYPTION_KEY_ROTATE_CONCURRENCY, + ENCRYPTION_KEY_ROTATE_QUEUE, +} from "@/lib/jobs/encryption-key-rotate-queue"; +import { + ENVIRONMENT_AQ_HISTORY_CRON, + ENVIRONMENT_AQ_HISTORY_QUEUE, +} from "@/lib/jobs/environment-air-quality-history"; +import { + ENVIRONMENT_FETCH_CONCURRENCY, + ENVIRONMENT_FETCH_CRON, + ENVIRONMENT_FETCH_QUEUE, +} from "@/lib/jobs/environment-fetch"; +import { + FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY, + FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, +} from "@/lib/jobs/free-text-encryption-backfill"; +import { GEO_BACKFILL_CRON, GEO_BACKFILL_QUEUE } from "@/lib/jobs/geo-backfill"; +import { + GEOLITE2_FETCH_CRON, + GEOLITE2_FETCH_QUEUE, +} from "@/lib/jobs/geolite2-fetch"; +import { + HEALTH_CONNECT_IMPORT_CONCURRENCY, + HEALTH_CONNECT_IMPORT_QUEUE, +} from "@/lib/jobs/health-connect-import-queue"; +import { + INTAKE_AUTO_SKIP_CRON, + INTAKE_AUTO_SKIP_QUEUE, +} from "@/lib/jobs/intake-auto-skip-queue"; +import { + INTAKE_AUTO_TAKE_CRON, + INTAKE_AUTO_TAKE_QUEUE, +} from "@/lib/jobs/intake-auto-take-queue"; +import { WHOLE_PASS } from "@/lib/jobs/long-pass"; +import { + MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY, + MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, +} from "@/lib/jobs/med-notes-encryption-backfill"; +import { MEASUREMENT_FOLD_REPAIR_QUEUE } from "@/lib/jobs/measurement-fold-repair-queue"; +import { + MEASUREMENT_MAINTENANCE_CRON, + MEASUREMENT_MAINTENANCE_QUEUE, + MEASUREMENT_MAINTENANCE_SEND_OPTIONS, +} from "@/lib/jobs/measurement-maintenance"; +import { + MEDICATION_INTAKE_IMPORT_CONCURRENCY, + MEDICATION_INTAKE_IMPORT_QUEUE, +} from "@/lib/jobs/medication-intake-import-queue"; +import { + MEDICATION_INVENTORY_EXPIRE_CRON, + MEDICATION_INVENTORY_EXPIRE_QUEUE, +} from "@/lib/jobs/medication-inventory-expire"; +import { + MOOD_PROGNOSIS_REFRESH_CRON, + MOOD_PROGNOSIS_REFRESH_QUEUE, +} from "@/lib/jobs/mood-prognosis-refresh-queue"; +import { + NOTE_ENCRYPTION_BACKFILL_CONCURRENCY, + NOTE_ENCRYPTION_BACKFILL_QUEUE, +} from "@/lib/jobs/note-encryption-backfill"; +import { + OFFHOST_BACKUP_QUEUE, + OFFHOST_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/offhost-backup-queue"; +import { + OFFHOST_PURGE_CRON, + OFFHOST_PURGE_QUEUE, +} from "@/lib/jobs/offhost-purge-queue"; +import { + PR_DETECTION_CONCURRENCY, + PR_DETECTION_EXPIRE_SECONDS, + PR_DETECTION_FALLBACK_CRON, + PR_DETECTION_QUEUE, + type PrDetectionPayload, +} from "@/lib/jobs/pr-detection"; +import { PULSE_DAY_MEAN_QUEUE } from "@/lib/jobs/pulse-day-mean"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + RESTORE_DRILL_CRON, + RESTORE_DRILL_QUEUE, + RESTORE_DRILL_SEND_OPTIONS, +} from "@/lib/jobs/restore-drill-queue"; +import { + SOFT_DELETE_PURGE_CRON, + SOFT_DELETE_PURGE_QUEUE, +} from "@/lib/jobs/soft-delete-purge"; +import { + TLS_PIN_MONITOR_CRON, + TLS_PIN_MONITOR_QUEUE, +} from "@/lib/jobs/tls-pin-monitor-queue"; +import { + INTAKE_SLOT_DEDUP_CONCURRENCY, + INTAKE_SLOT_DEDUP_QUEUE, +} from "@/lib/medications/intake-slot-dedup"; + +import { MAINTENANCE_DISCOVERY } from "./maintenance-discovery"; +import { + defineJob, + type JobDefinition, + type JobGroup, + type JobSpec, +} from "./define-job"; + +/** Weekly, Sunday 03:00. The send options carry the backup's long expiry. */ +const DATA_BACKUP_CRON = "0 3 * * 0"; +/** 02:30: after the audit and idempotency cleanups, before the weekly backup. */ +const OFFHOST_BACKUP_CRON = "30 2 * * *"; +const RATE_LIMIT_CLEANUP_QUEUE = "rate-limit-cleanup"; +const RATE_LIMIT_CLEANUP_CRON = "*/5 * * * *"; +const IDEMPOTENCY_CLEANUP_QUEUE = "idempotency-cleanup"; +const IDEMPOTENCY_CLEANUP_CRON = "0 3 * * *"; +/** An elevation lives five minutes; the sweep only keeps the table small. */ +const STEP_UP_ELEVATION_CLEANUP_QUEUE = "step-up-elevation-cleanup"; +const STEP_UP_ELEVATION_CLEANUP_CRON = "*/15 * * * *"; +const AUDIT_LOG_CLEANUP_QUEUE = "audit-log-cleanup"; +const AUDIT_LOG_CLEANUP_CRON = "15 3 * * *"; +/** Daily duplicate dose-slot discovery; the empty payload is the fan-out. */ +const INTAKE_SLOT_DEDUP_CRON = "28 3 * * *"; +/** Every five minutes; the admin chart polls at the same interval. */ +const HOST_METRIC_QUEUE = "host-metric-sample"; +const HOST_METRIC_CRON = "*/5 * * * *"; +/** After the cleanups, so the previous day's noise is gone. */ +const FEEDBACK_AGGREGATOR_QUEUE = "feedback-aggregator"; +const FEEDBACK_AGGREGATOR_CRON = "0 4 * * *"; +const MOOD_REMINDER_CLEANUP_QUEUE = "mood-reminder-cleanup"; +const MOOD_REMINDER_CLEANUP_CRON = "25 3 * * *"; +const PUSH_ATTEMPT_CLEANUP_QUEUE = "push-attempt-cleanup"; +const PUSH_ATTEMPT_CLEANUP_CRON = "35 3 * * *"; +const ARRIVAL_REACTION_CLEANUP_QUEUE = "arrival-reaction-cleanup"; +const ARRIVAL_REACTION_CLEANUP_CRON = "10 4 * * *"; +const MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE = "measurement-tombstone-cleanup"; +const MEASUREMENT_TOMBSTONE_CLEANUP_CRON = "40 3 * * *"; +const COACH_MESSAGE_CLEANUP_QUEUE = "coach-message-cleanup"; +const COACH_MESSAGE_CLEANUP_CRON = "50 3 * * *"; +const WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE = "workout-insight-claim-cleanup"; +const WORKOUT_INSIGHT_CLAIM_CLEANUP_CRON = "52 3 * * *"; +const NOTE_ENCRYPTION_BACKFILL_CRON = "55 3 * * *"; +const MCP_TOKEN_CLEANUP_QUEUE = "mcp-token-cleanup"; +const MCP_TOKEN_CLEANUP_CRON = "0 4 * * *"; +const MED_NOTES_ENCRYPTION_BACKFILL_CRON = "5 4 * * *"; +const FREE_TEXT_ENCRYPTION_BACKFILL_CRON = "7 4 * * *"; + +const backups = () => import("@/lib/jobs/reminder/backup-handlers"); +const cleanups = () => import("@/lib/jobs/reminder/cleanup-handlers"); +const ops = () => import("@/lib/jobs/reminder/ops-handlers"); +const medication = () => import("@/lib/jobs/reminder/medication-maintenance"); +const maintenance = () => import("@/lib/jobs/reminder/maintenance-handlers"); +const encryption = () => + import("@/lib/jobs/reminder/encryption-backfill-handlers"); +const documents = () => import("@/lib/jobs/reminder/document-handlers"); + +/** A daily keyless prune: one bounded, idempotent delete, single-flight. */ +function prune>( + queue: string, + cron: string, + load: JobSpec["load"], + options: { retry: "cron" | "default" } = { retry: "cron" }, +): JobDefinition { + return defineJob({ + queue, + cron, + ...(options.retry === "cron" ? { scheduleOptions: cronIsTheRetry } : {}), + work: { localConcurrency: 1 }, + load, + }); +} + +const perUserBackfill = (reason: string) => + ({ policy: "exclusive", reason }) as const; + +const jobs: readonly JobDefinition[] = [ + defineJob({ + queue: ENVIRONMENT_FETCH_QUEUE, + cron: ENVIRONMENT_FETCH_CRON, + work: { localConcurrency: ENVIRONMENT_FETCH_CONCURRENCY }, + load: async ({ boss }) => + (await maintenance()).environmentFetchHandler(boss), + }), + // A run with work left sends its own follow-up while still active, which + // `exclusive` would swallow. + defineJob({ + queue: ENVIRONMENT_AQ_HISTORY_QUEUE, + cron: ENVIRONMENT_AQ_HISTORY_CRON, + policy: { + policy: "short", + reason: + "Per-account singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", + }, + work: { localConcurrency: 1 }, + load: async ({ boss }) => + (await maintenance()).environmentAqHistoryHandler(boss), + }), + // A failed run is an outage at the instance; the default retries apply. + defineJob({ + queue: DAWARICH_SYNC_QUEUE, + cron: DAWARICH_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async ({ boss }) => (await maintenance()).dawarichSyncHandler(boss), + }), + defineJob({ + queue: DATA_BACKUP_QUEUE, + cron: DATA_BACKUP_CRON, + scheduleOptions: DATA_BACKUP_SEND_OPTIONS, + work: { localConcurrency: 1 }, + lock: () => WHOLE_PASS, + lane: true, + load: async () => (await backups()).handleDataBackup, + }), + prune( + RATE_LIMIT_CLEANUP_QUEUE, + RATE_LIMIT_CLEANUP_CRON, + async () => (await cleanups()).handleRateLimitCleanup, + ), + prune( + IDEMPOTENCY_CLEANUP_QUEUE, + IDEMPOTENCY_CLEANUP_CRON, + async () => (await cleanups()).handleIdempotencyCleanup, + ), + prune( + AUDIT_LOG_CLEANUP_QUEUE, + AUDIT_LOG_CLEANUP_CRON, + async () => (await cleanups()).handleAuditLogCleanup, + { retry: "default" }, + ), + prune( + STEP_UP_ELEVATION_CLEANUP_QUEUE, + STEP_UP_ELEVATION_CLEANUP_CRON, + async () => (await cleanups()).handleStepUpElevationCleanup, + ), + // Bound with metadata: a retry resumes from the job's creation time. + defineJob({ + queue: OFFHOST_BACKUP_QUEUE, + cron: OFFHOST_BACKUP_CRON, + scheduleOptions: OFFHOST_BACKUP_SEND_OPTIONS, + work: { localConcurrency: 1, includeMetadata: true }, + lock: () => WHOLE_PASS, + lane: true, + load: async () => (await backups()).handleOffhostBackup, + }), + // Locked: a drill that outlives a restart never reads the bucket beside + // another one. + defineJob({ + queue: RESTORE_DRILL_QUEUE, + cron: RESTORE_DRILL_CRON, + scheduleOptions: RESTORE_DRILL_SEND_OPTIONS, + work: { localConcurrency: 1 }, + lock: () => WHOLE_PASS, + lane: true, + load: async () => + (await import("@/lib/jobs/restore-drill")).handleRestoreDrill, + }), + // The nightly backstop for the purge an account deletion kicks at once. + prune( + OFFHOST_PURGE_QUEUE, + OFFHOST_PURGE_CRON, + async () => (await import("@/lib/jobs/offhost-purge")).handleOffhostPurge, + ), + // One restore at a time: each holds a transaction over a whole account. + defineJob({ + queue: BACKUP_RESTORE_QUEUE, + work: { localConcurrency: 1 }, + lane: true, + load: async () => + (await import("@/lib/jobs/backup-restore")).handleBackupRestore, + }), + prune( + HOST_METRIC_QUEUE, + HOST_METRIC_CRON, + async () => (await ops()).handleHostMetricSample, + ), + prune( + FEEDBACK_AGGREGATOR_QUEUE, + FEEDBACK_AGGREGATOR_CRON, + async () => (await ops()).handleFeedbackAggregator, + ), + // Hourly at :40, capped per pass, for audit rows that landed without a geo. + prune( + GEO_BACKFILL_QUEUE, + GEO_BACKFILL_CRON, + async () => (await ops()).handleGeoBackfill, + ), + // Monthly; a month is far too long to be the retry, so default retries. + prune( + GEOLITE2_FETCH_QUEUE, + GEOLITE2_FETCH_CRON, + async () => (await ops()).handleGeolite2Fetch, + { retry: "default" }, + ), + // Every six hours, inside the re-pin window the app's leaf pin needs. + prune( + TLS_PIN_MONITOR_QUEUE, + TLS_PIN_MONITOR_CRON, + async () => (await ops()).handleTlsPinMonitor, + { retry: "default" }, + ), + // An ingest job is one account; the fallback rescan every 30 minutes walks + // every account, one run at a time. + defineJob({ + queue: PR_DETECTION_QUEUE, + cron: PR_DETECTION_FALLBACK_CRON, + scheduleOptions: { expireInSeconds: PR_DETECTION_EXPIRE_SECONDS }, + work: { localConcurrency: PR_DETECTION_CONCURRENCY }, + lock: (job: Job) => { + const userId = (job.data as PrDetectionPayload | undefined)?.userId; + return userId ? `user:${userId}` : WHOLE_PASS; + }, + load: async () => (await ops()).handlePrDetection, + }), + prune( + MEDICATION_INVENTORY_EXPIRE_QUEUE, + MEDICATION_INVENTORY_EXPIRE_CRON, + async () => (await medication()).handleMedicationInventoryExpire, + ), + prune( + INTAKE_AUTO_SKIP_QUEUE, + INTAKE_AUTO_SKIP_CRON, + async () => (await medication()).handleIntakeAutoSkip, + ), + prune( + INTAKE_AUTO_TAKE_QUEUE, + INTAKE_AUTO_TAKE_CRON, + async () => (await medication()).handleIntakeAutoTake, + ), + // CPU-bound parse of up to a gigabyte; a second at once would race the + // first for memory. + defineJob({ + queue: APPLE_HEALTH_IMPORT_V2_QUEUE, + work: { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, + lane: true, + load: async () => (await maintenance()).handleAppleHealthImportJobs, + }), + defineJob({ + queue: APPLE_HEALTH_IMPORT_LEGACY_QUEUE, + work: { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, + load: async () => (await maintenance()).handleLegacyAppleHealthImportJobs, + }), + // Every 15 minutes: an import a crashed worker left mid-run converges to a + // visible `failed` within two ticks. + prune( + IMPORT_JOB_RECONCILE_QUEUE, + IMPORT_JOB_RECONCILE_CRON, + async () => + (await import("@/lib/jobs/apple-health-import-worker")) + .handleImportJobReconcileTick, + ), + defineJob({ + queue: MEDICATION_INTAKE_IMPORT_QUEUE, + work: { + localConcurrency: MEDICATION_INTAKE_IMPORT_CONCURRENCY, + includeMetadata: true, + }, + load: async () => (await maintenance()).handleMedicationIntakeImportJobs, + }), + defineJob({ + queue: HEALTH_CONNECT_IMPORT_QUEUE, + work: { localConcurrency: HEALTH_CONNECT_IMPORT_CONCURRENCY }, + lane: true, + load: async () => + (await import("@/lib/jobs/health-connect-import-worker")) + .handleHealthConnectImport, + }), + // A run that stops early sends its own follow-up while still active, which + // `exclusive` would swallow. Serial: each batch holds an account against a + // concurrent restore. + defineJob({ + queue: COMPACTION_TOMBSTONE_PURGE_QUEUE, + policy: { + policy: "short", + reason: + "Fixed singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/compaction-tombstone-purge")) + .handleCompactionTombstonePurge, + }), + defineJob({ + queue: MEASUREMENT_FOLD_REPAIR_QUEUE, + policy: { + policy: "short", + reason: + "Per-account singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/measurement-fold-repair")) + .handleMeasurementFoldRepair, + }), + // The nightly check vacuums when dead rows pile up; the same singleton key + // as the operator's run, so the two collapse. + defineJob({ + queue: MEASUREMENT_MAINTENANCE_QUEUE, + cron: MEASUREMENT_MAINTENANCE_CRON, + scheduleOptions: MEASUREMENT_MAINTENANCE_SEND_OPTIONS, + policy: { + policy: "exclusive", + reason: + "Fixed singleton key on the admin send and the nightly tick alike. Two maintenance passes would vacuum or rebuild side by side; the route reports a suppressed send back as enqueued: false.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/measurement-maintenance")) + .handleMeasurementMaintenance, + }), + defineJob({ + queue: PULSE_DAY_MEAN_QUEUE, + policy: { + policy: "exclusive", + reason: + "Fixed singleton key, admin-triggered. A dry run beside a rebuild would compare against rollups being rewritten; the route answers 409 busy instead.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/pulse-day-mean")).handlePulseDayMean, + }), + defineJob({ + queue: INTAKE_SLOT_DEDUP_QUEUE, + cron: INTAKE_SLOT_DEDUP_CRON, + policy: perUserBackfill( + "Per-user duplicate-slot cleanup; discovery drops the user once no colliding intake pair remains.", + ), + work: { localConcurrency: INTAKE_SLOT_DEDUP_CONCURRENCY }, + load: async () => (await maintenance()).handleIntakeSlotDedup, + }), + prune( + MOOD_REMINDER_CLEANUP_QUEUE, + MOOD_REMINDER_CLEANUP_CRON, + async () => (await cleanups()).handleMoodReminderCleanup, + ), + prune( + PUSH_ATTEMPT_CLEANUP_QUEUE, + PUSH_ATTEMPT_CLEANUP_CRON, + async () => (await cleanups()).handlePushAttemptCleanup, + ), + prune( + ARRIVAL_REACTION_CLEANUP_QUEUE, + ARRIVAL_REACTION_CLEANUP_CRON, + async () => (await cleanups()).handleArrivalReactionCleanup, + ), + // The nightly refreshes: tomorrow's tick is the retry. + prune( + CYCLE_PREDICTION_REFRESH_QUEUE, + CYCLE_PREDICTION_REFRESH_CRON, + async () => + (await import("@/lib/jobs/cycle-prediction-refresh")) + .handleCyclePredictionRefresh, + ), + prune( + MOOD_PROGNOSIS_REFRESH_QUEUE, + MOOD_PROGNOSIS_REFRESH_CRON, + async () => + (await import("@/lib/jobs/mood-prognosis-refresh")) + .handleMoodPrognosisRefresh, + ), + prune( + ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, + ACHIEVEMENT_UNLOCK_SWEEP_CRON, + async () => + (await import("@/lib/jobs/achievement-unlock-sweep")) + .handleAchievementUnlockSweep, + ), + prune( + MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, + MEASUREMENT_TOMBSTONE_CLEANUP_CRON, + async () => (await cleanups()).handleMeasurementTombstoneCleanup, + { retry: "default" }, + ), + prune( + COACH_MESSAGE_CLEANUP_QUEUE, + COACH_MESSAGE_CLEANUP_CRON, + async () => (await cleanups()).handleCoachMessageCleanup, + ), + prune( + WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE, + WORKOUT_INSIGHT_CLAIM_CLEANUP_CRON, + async () => (await cleanups()).handleWorkoutInsightClaimCleanup, + ), + // The converging encryption backfills: boot discovery plus a daily tick + // whose empty payload fans out one job per account still holding a + // readable value. + defineJob({ + queue: NOTE_ENCRYPTION_BACKFILL_QUEUE, + cron: NOTE_ENCRYPTION_BACKFILL_CRON, + policy: perUserBackfill( + "Per-user note-encryption backfill; discovery drops the user once no plaintext note remains.", + ), + work: { localConcurrency: NOTE_ENCRYPTION_BACKFILL_CONCURRENCY }, + load: async () => (await encryption()).handleNoteEncryptionBackfill, + }), + // Admin-triggered, one corpus rotation at a time. + defineJob({ + queue: ENCRYPTION_KEY_ROTATE_QUEUE, + policy: { + policy: "exclusive", + reason: + "Fixed singleton key, admin-triggered. Two concurrent corpus rotations must never overlap; the route already reports a suppressed send back as alreadyQueued.", + }, + work: { localConcurrency: ENCRYPTION_KEY_ROTATE_CONCURRENCY }, + lock: () => WHOLE_PASS, + load: async () => + (await import("@/lib/jobs/encryption-key-rotate")) + .handleEncryptionKeyRotate, + }), + prune( + MCP_TOKEN_CLEANUP_QUEUE, + MCP_TOKEN_CLEANUP_CRON, + async () => (await cleanups()).handleMcpTokenCleanup, + ), + defineJob({ + queue: MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, + cron: MED_NOTES_ENCRYPTION_BACKFILL_CRON, + policy: perUserBackfill( + "Per-user medication-note encryption backfill; discovery drops the user once no plaintext note remains.", + ), + work: { localConcurrency: MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY }, + load: async () => (await encryption()).handleMedNotesEncryptionBackfill, + }), + defineJob({ + queue: FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, + cron: FREE_TEXT_ENCRYPTION_BACKFILL_CRON, + policy: perUserBackfill( + "Per-user encryption backfill (Coach title, custom-metric note, practitioner phone + address, workout GPS track); discovery drops the user once no readable value remains.", + ), + work: { localConcurrency: FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY }, + load: async () => (await encryption()).handleFreeTextEncryptionBackfill, + }), + // Physical purge of documents past the 30-day undo grace. + prune( + DOCUMENT_PURGE_QUEUE, + DOCUMENT_PURGE_CRON, + async () => (await import("@/lib/jobs/document-purge")).handleDocumentPurge, + ), + prune( + SOFT_DELETE_PURGE_QUEUE, + SOFT_DELETE_PURGE_CRON, + async () => + (await import("@/lib/jobs/soft-delete-purge")).handleSoftDeletePurge, + ), + prune( + AUTH_LEDGER_CLEANUP_QUEUE, + AUTH_LEDGER_CLEANUP_CRON, + async () => + (await import("@/lib/jobs/auth-ledger-cleanup")).handleAuthLedgerCleanup, + ), + // Hourly heal of summaries stuck on PENDING. + prune( + DOCUMENT_SUMMARY_REAPER_QUEUE, + DOCUMENT_SUMMARY_REAPER_CRON, + async () => + (await import("@/lib/jobs/document-summary-reaper")) + .handleDocumentSummaryReaper, + ), + // Index-all for one account, idempotent: a second press while one runs is + // correctly a no-op. + defineJob({ + queue: CONTENT_INDEX_BACKFILL_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user index-all run, triggered from the documents surface. Idempotent (it indexes only not-yet-indexed documents), so a second press while one runs is correctly a no-op.", + }, + work: { localConcurrency: CONTENT_INDEX_BACKFILL_CONCURRENCY }, + lock: (job: Job<{ userId?: string }>) => `user:${job.data.userId ?? ""}`, + load: async () => (await documents()).handleContentIndexBackfill, + }), + // Per-document jobs re-read the document when they start, so only queued + // duplicates may collapse; nothing re-offers a dropped re-process. + defineJob({ + queue: DOCUMENT_INDEX_QUEUE, + policy: { + policy: "short", + reason: + "Per-document, no re-converging discovery pass. Collapse queued duplicates only, so a re-index after a content change is never dropped.", + }, + work: { localConcurrency: DOCUMENT_INDEX_CONCURRENCY }, + load: async () => (await documents()).handleDocumentIndex, + }), + // One run per job under its run id's lock, so a redelivery never reads + // beside the first delivery. + defineJob({ + queue: DOCUMENT_AI_RUN_QUEUE, + work: { localConcurrency: DOCUMENT_AI_RUN_CONCURRENCY }, + lock: (job: Job<{ runId?: string }>) => `run:${job.data.runId ?? ""}`, + load: async () => + (await import("@/lib/jobs/document-ai-run")).handleDocumentAiRunJobs, + }), + prune( + DOCUMENT_AI_RUN_REAPER_QUEUE, + DOCUMENT_AI_RUN_REAPER_CRON, + async () => + (await import("@/lib/jobs/document-ai-run-reaper")) + .handleDocumentAiRunReaper, + ), + defineJob({ + queue: DOCUMENT_THUMBNAIL_QUEUE, + policy: { + policy: "short", + reason: + "Per-document, no re-converging discovery pass. Collapse queued duplicates only.", + }, + work: { localConcurrency: DOCUMENT_THUMBNAIL_CONCURRENCY }, + load: async () => (await documents()).handleDocumentThumbnail, + }), + defineJob({ + queue: DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, + policy: perUserBackfill( + "Per-user thumbnail backfill; discovery drops the user once every thumbnailable document has a preview.", + ), + work: { localConcurrency: DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY }, + load: async () => (await documents()).handleDocumentThumbnailBackfill, + }), + defineJob({ + queue: DOCUMENT_SUMMARY_QUEUE, + policy: { + policy: "short", + reason: + "Per-document, no re-converging discovery pass. Collapse queued duplicates only.", + }, + work: { localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY }, + load: async () => (await documents()).handleDocumentSummary, + }), + defineJob({ + queue: DOCUMENT_SUMMARY_CATCHUP_QUEUE, + policy: { + policy: "short", + reason: + "Per-user, keyed on the opt-in flip. Collapse a double toggle into one pass; the pass re-reads its candidate set when it starts.", + }, + work: { localConcurrency: DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY }, + load: async () => (await documents()).handleDocumentSummaryCatchUp, + }), +]; + +/** A restore the previous process was running is picked up once its heartbeat is stale. */ +async function sweepInterruptedRestores({ + boss, +}: { + boss: Parameters[0]["boss"]; +}): Promise { + await ( + await import("@/lib/jobs/backup-restore") + ).enqueueBackupRestoreSweepAtBoot(boss); +} + +export const MAINTENANCE_GROUP: JobGroup = { + name: "maintenance", + jobs, + afterBind: sweepInterruptedRestores, + discovery: MAINTENANCE_DISCOVERY, +}; diff --git a/src/lib/jobs/registry/register.ts b/src/lib/jobs/registry/register.ts new file mode 100644 index 000000000..3a2a36bef --- /dev/null +++ b/src/lib/jobs/registry/register.ts @@ -0,0 +1,146 @@ +/** + * Turn the registry into running queues. + * + * For one group, in this order: create every queue (policy and retention + * included), reconcile the policies of queues that already exist, schedule + * every cron, remove the retired queues, then bind a handler to every queue. + * The order is the one the hand-written registrars kept, and + * `tests/integration/queue-registration-parity.test.ts` holds what it leaves + * in pg-boss to the golden they produced. + * + * A handler is loaded on its queue's first job, not here. The wrappers that + * shape how a job runs (the lock, the heavy lane) are applied at binding, so a + * queue's protections do not depend on whether its module has loaded yet. A + * module that fails to load fails the job that asked for it, with the queue's + * name on the error, and the next job tries the import again. + */ +import type { Job, PgBoss } from "pg-boss"; + +import { heavyLane } from "@/lib/jobs/heavy-lane"; +import { lockedPass } from "@/lib/jobs/long-pass"; +import { + createAndSchedule, + createAndWork, + retireQueues, + type QueuePolicyTable, + type ScheduleEntry, +} from "@/lib/jobs/reminder/registrar-shared"; +import { workerLog } from "@/lib/jobs/reminder/shared"; + +import type { + BatchHandler, + BootDiscovery, + JobContext, + JobDefinition, + JobGroup, +} from "./define-job"; + +/** + * The handler a definition binds: its loaded module behind the lock and the + * lane it declares. + */ +export function bindableHandler( + definition: JobDefinition, + ctx: JobContext, +): BatchHandler> { + let loading: Promise>> | null = null; + const load = (): Promise>> => { + loading ??= definition.load(ctx).catch((err: unknown) => { + loading = null; + throw err; + }); + return loading; + }; + let handler: BatchHandler> = async (jobs) => + (await load())(jobs); + if (definition.lane) { + handler = heavyLane( + definition.queue, + handler, + definition.lane === true ? undefined : definition.lane, + ); + } + if (definition.lock) { + handler = lockedPass(definition.queue, definition.lock, handler); + } + return handler; +} + +/** The schedule tuples of a group, in declaration order. */ +export function schedulesOf(jobs: readonly JobDefinition[]): ScheduleEntry[] { + return jobs.flatMap((job): ScheduleEntry[] => + job.cron + ? [ + job.scheduleOptions + ? [job.queue, job.cron, { ...job.scheduleOptions }] + : [job.queue, job.cron], + ] + : [], + ); +} + +/** The policy table of a group. */ +export function policiesOf(jobs: readonly JobDefinition[]): QueuePolicyTable { + return Object.fromEntries( + jobs.flatMap((job) => (job.policy ? [[job.queue, job.policy]] : [])), + ); +} + +/** + * Register one group: create, schedule, retire, bind. Returns the queue names + * it created. + */ +export async function registerJobGroup( + boss: PgBoss, + group: JobGroup, +): Promise { + const queues = group.jobs.map((job) => job.queue); + await createAndSchedule( + boss, + queues, + schedulesOf(group.jobs), + policiesOf(group.jobs), + ); + if (group.retired) await retireQueues(boss, group.retired); + const ctx: JobContext = { boss }; + for (const job of group.jobs) { + await createAndWork(boss, job.queue, job.work, bindableHandler(job, ctx)); + } + if (group.afterBind) await group.afterBind(ctx); + return queues; +} + +/** + * Run one group's boot discovery. A pass that fails is logged and the next + * one runs: a discovery miss never fails the boot, and the next boot or cron + * tick offers the work again. + */ +export async function runBootDiscovery( + passes: readonly BootDiscovery[], +): Promise { + for (const pass of passes) { + try { + const result = await pass.run(); + if (!result) continue; + if (result.error) { + workerLog( + "error", + `[${pass.name}] boot discovery failed: ${result.error}`, + ); + } else { + const skipped = + result.skipped === undefined ? "" : ` skipped=${result.skipped}`; + workerLog( + "info", + `[${pass.name}] boot discovery: enqueued=${result.enqueued}${skipped}`, + ); + } + } catch (err) { + workerLog( + "error", + `[${pass.name}] boot discovery threw an unexpected error`, + err, + ); + } + } +} diff --git a/src/lib/jobs/registry/reminders.ts b/src/lib/jobs/registry/reminders.ts new file mode 100644 index 000000000..e66f90434 --- /dev/null +++ b/src/lib/jobs/registry/reminders.ts @@ -0,0 +1,70 @@ +/** + * The user-facing reminder ticks: the medication overdue check, the daily + * mood reminder, the cycle reminder, the checkup (measurement) reminder, and + * the eventful reminder-satisfy queue. + * + * The three daily reminders tick every 15 minutes and short-circuit unless + * the user's local time is the reminder's hour, so one UTC cron serves every + * zone; each keeps its own once-a-day anchor (a dispatch ledger, the + * `push_attempts` row, the reminder's `nextDueAt`). Every tick is + * single-flight so two ticks never race that anchor. + * + * Every queue here keeps pg-boss's default retries: a job-level failure means + * the pass itself could not run, a transient. The satisfy queue sets its own + * `retryLimit` at send time (`reminder-satisfy.ts`). + */ +import { + REMINDER_SATISFY_CONCURRENCY, + REMINDER_SATISFY_QUEUE, +} from "@/lib/jobs/reminder-satisfy"; + +import { defineJob, type JobDefinition, type JobGroup } from "./define-job"; + +const MEDICATION_REMINDER_CHECK_QUEUE = "medication-reminder-check"; +const MEDICATION_REMINDER_CHECK_CRON = "*/15 * * * *"; +const MOOD_REMINDER_QUEUE = "mood-reminder-check"; +const MOOD_REMINDER_CRON = "*/15 * * * *"; +const CYCLE_REMINDER_QUEUE = "cycle-reminder-check"; +const CYCLE_REMINDER_CRON = "*/15 * * * *"; +const MEASUREMENT_REMINDER_QUEUE = "measurement-reminder-check"; +const MEASUREMENT_REMINDER_CRON = "*/15 * * * *"; + +const checks = () => import("@/lib/jobs/reminder/mood-cycle-checks"); + +const jobs: readonly JobDefinition[] = [ + defineJob({ + queue: MEDICATION_REMINDER_CHECK_QUEUE, + cron: MEDICATION_REMINDER_CHECK_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/medication-reminder-check")) + .handleReminderCheck, + }), + defineJob({ + queue: MOOD_REMINDER_QUEUE, + cron: MOOD_REMINDER_CRON, + work: { localConcurrency: 1 }, + load: async () => (await checks()).handleMoodReminderCheck, + }), + defineJob({ + queue: CYCLE_REMINDER_QUEUE, + cron: CYCLE_REMINDER_CRON, + work: { localConcurrency: 1 }, + load: async () => (await checks()).handleCycleReminderCheck, + }), + defineJob({ + queue: MEASUREMENT_REMINDER_QUEUE, + cron: MEASUREMENT_REMINDER_CRON, + work: { localConcurrency: 1 }, + load: async () => (await checks()).handleMeasurementReminderCheck, + }), + // No cron: enqueued by the ingest paths when a measurement or lab lands; + // the 15-minute checkup tick is the safety net. + defineJob({ + queue: REMINDER_SATISFY_QUEUE, + work: { localConcurrency: REMINDER_SATISFY_CONCURRENCY }, + load: async () => (await checks()).handleReminderSatisfy, + }), +]; + +export const REMINDERS_GROUP: JobGroup = { name: "reminders", jobs }; diff --git a/src/lib/jobs/registry/rollup.ts b/src/lib/jobs/registry/rollup.ts new file mode 100644 index 000000000..cb885c0dc --- /dev/null +++ b/src/lib/jobs/registry/rollup.ts @@ -0,0 +1,281 @@ +/** + * The pre-aggregation and consolidation queues: the measurement and mood + * rollup folds with their boot backfills, the legacy step consolidation and + * its repair, the cumulative-record re-derivation, the daily-mean + * consolidation, the dense intra-day retention and hourly rebuild, the + * supplement nutrient recompute, and the nightly drain. + * + * The two recompute queues and the supplement recompute are `short`: the + * handler reads the bucket's live rows when it starts, so sends that pile up + * while an identical job is still queued may collapse, while a send issued + * after the job started reading must still queue the next one. The per-user + * backfills are `exclusive`: a second run is duplicated work and discovery + * re-offers whatever is outstanding. The drain is a keyless cron tick. + * + * Boot discovery is staggered: each backfill starts a multiple of + * `BOOT_BACKFILL_STAGGER_SECONDS` later, so a heavy account's full-history + * loads do not all land on the first poll. The dense passes defer themselves + * further, past the boot window. + */ +import type { Job } from "pg-boss"; + +import { + DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY, + DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, +} from "@/lib/jobs/dense-intraday-hourly-rebuild-queue"; +import { + DENSE_INTRADAY_RETENTION_CONCURRENCY, + DENSE_INTRADAY_RETENTION_QUEUE, +} from "@/lib/jobs/dense-intraday-retention-queue"; +import { + DRAIN_CUMULATIVE_CRON, + DRAIN_CUMULATIVE_QUEUE, +} from "@/lib/jobs/drain-cumulative-queue"; +import { + MEAN_CONSOLIDATION_CONCURRENCY, + MEAN_CONSOLIDATION_QUEUE, +} from "@/lib/jobs/mean-consolidation"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { BOOT_BACKFILL_STAGGER_SECONDS } from "@/lib/jobs/reminder/shared"; +import { + STEP_CONSOLIDATION_CONCURRENCY, + STEP_CONSOLIDATION_QUEUE, +} from "@/lib/jobs/step-consolidation"; +import { + STEP_CONSOLIDATION_REPAIR_CONCURRENCY, + STEP_CONSOLIDATION_REPAIR_QUEUE, +} from "@/lib/jobs/step-consolidation-repair"; +import { + SUPPLEMENT_NUTRIENT_RECOMPUTE_CONCURRENCY, + SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE, +} from "@/lib/medications/ingredients/recompute-job"; +import { + CUMULATIVE_PR_REDERIVE_CONCURRENCY, + CUMULATIVE_PR_REDERIVE_QUEUE, +} from "@/lib/personal-records/cumulative-pr-rederivation"; +import { + ROLLUP_FULL_BACKFILL_CONCURRENCY, + ROLLUP_FULL_BACKFILL_QUEUE, + ROLLUP_RECOMPUTE_CONCURRENCY, + ROLLUP_RECOMPUTE_QUEUE, + type RollupFullBackfillPayload, +} from "@/lib/rollups/measurement-rollups"; +import { + MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY, + MOOD_ROLLUP_FULL_BACKFILL_QUEUE, + MOOD_ROLLUP_RECOMPUTE_CONCURRENCY, + MOOD_ROLLUP_RECOMPUTE_QUEUE, +} from "@/lib/rollups/mood-rollups"; + +import { + defineJob, + type BootDiscovery, + type JobDefinition, + type JobGroup, +} from "./define-job"; + +/** + * The boot backfill of the per-day medication-compliance rollup, whose table + * had no reader left and was dropped. Its queue is removed where it exists. + */ +const RETIRED_ROLLUP_QUEUES = ["medication-compliance-full-backfill"] as const; + +const handlers = () => import("@/lib/jobs/reminder/rollup-handlers"); + +const jobs: readonly JobDefinition[] = [ + defineJob({ + queue: ROLLUP_RECOMPUTE_QUEUE, + policy: { + policy: "short", + reason: + "Burst coalescing for the write-path fan-out. The handler re-reads the bucket at run time, so collapsing queued duplicates cannot strand a write; exclusive could.", + }, + work: { localConcurrency: ROLLUP_RECOMPUTE_CONCURRENCY }, + load: async () => (await handlers()).handleRollupRecompute, + }), + // The lock sits outside the lane: a retry pg-boss starts after the expiry + // while the first fold still runs finds the account's lock and settles, + // instead of waiting in the lane and folding the account again. + defineJob({ + queue: ROLLUP_FULL_BACKFILL_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user full-history backfill; long-running. Boot discovery re-enqueues while coverage is still missing, so suppressing a duplicate of an ACTIVE pass is safe.", + }, + work: { localConcurrency: ROLLUP_FULL_BACKFILL_CONCURRENCY }, + lock: (job: Job) => job.data.userId, + lane: true, + load: async () => (await handlers()).handleRollupFullBackfill, + }), + defineJob({ + queue: MOOD_ROLLUP_RECOMPUTE_QUEUE, + policy: { + policy: "short", + reason: + "Same shape as the measurement recompute: re-reads the bucket at run time, so only queued duplicates may be collapsed.", + }, + work: { localConcurrency: MOOD_ROLLUP_RECOMPUTE_CONCURRENCY }, + load: async () => (await handlers()).handleMoodRollupRecompute, + }), + defineJob({ + queue: MOOD_ROLLUP_FULL_BACKFILL_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user mood backfill; boot-discovery driven and self-converging on rollup coverage.", + }, + work: { localConcurrency: MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY }, + load: async () => (await handlers()).handleMoodRollupFullBackfill, + }), + defineJob({ + queue: STEP_CONSOLIDATION_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user legacy step consolidation; discovery drops the user once no live granular rows remain.", + }, + work: { localConcurrency: STEP_CONSOLIDATION_CONCURRENCY }, + load: async () => (await handlers()).handleStepConsolidation, + }), + defineJob({ + queue: STEP_CONSOLIDATION_REPAIR_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user one-shot repair; discovery drops the user once the affected provider rows are repaired.", + }, + work: { localConcurrency: STEP_CONSOLIDATION_REPAIR_CONCURRENCY }, + load: async () => (await handlers()).handleStepConsolidationRepair, + }), + defineJob({ + queue: CUMULATIVE_PR_REDERIVE_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user one-shot re-derivation; discovery drops the user once the inflated records are rewritten.", + }, + work: { localConcurrency: CUMULATIVE_PR_REDERIVE_CONCURRENCY }, + load: async () => (await handlers()).handleCumulativePrRederive, + }), + defineJob({ + queue: MEAN_CONSOLIDATION_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user daily-mean consolidation; discovery drops the user once the spot rows are folded.", + }, + work: { localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY }, + load: async () => (await handlers()).handleMeanConsolidation, + }), + defineJob({ + queue: DENSE_INTRADAY_RETENTION_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user dense-tier drain; the multi-hour pass on a heavy account is exactly the case where a restart used to append a duplicate full-history job.", + }, + work: { localConcurrency: DENSE_INTRADAY_RETENTION_CONCURRENCY }, + load: async () => (await handlers()).handleDenseIntradayRetention, + }), + defineJob({ + queue: DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user one-shot hourly rebuild; discovery drops the day once its retired daily row is gone.", + }, + work: { localConcurrency: DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY }, + load: async () => (await handlers()).handleDenseIntradayHourlyRebuild, + }), + // One sequential walk, so it never crowds the dashboard request pool. + defineJob({ + queue: DRAIN_CUMULATIVE_QUEUE, + cron: DRAIN_CUMULATIVE_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async ({ boss }) => (await handlers()).drainCumulativeHandler(boss), + }), + defineJob({ + queue: SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE, + policy: { + policy: "short", + reason: + "Per-account recompute of supplement nutrient rows; the pass reads the live doses and composition when it starts, so a queued duplicate may be collapsed, while a send during a running pass must still queue the next one.", + }, + work: { localConcurrency: SUPPLEMENT_NUTRIENT_RECOMPUTE_CONCURRENCY }, + load: async () => (await handlers()).handleSupplementNutrientRecompute, + }), +]; + +const discovery: readonly BootDiscovery[] = [ + { + name: "rollup-full-backfill", + run: async () => + ( + await import("@/lib/rollups/measurement-rollups") + ).enqueueBootTimeRollupBackfill(BOOT_BACKFILL_STAGGER_SECONDS * 0), + }, + { + name: "step-consolidation", + run: async () => + ( + await import("@/lib/jobs/step-consolidation") + ).enqueueBootTimeStepConsolidation(BOOT_BACKFILL_STAGGER_SECONDS * 1), + }, + { + name: "mean-consolidation", + run: async () => + ( + await import("@/lib/jobs/mean-consolidation") + ).enqueueBootTimeMeanConsolidation(BOOT_BACKFILL_STAGGER_SECONDS * 2), + }, + // Defers itself past the boot window (the pool-exhaustion fix). + { + name: "dense-intraday-retention", + run: async () => + ( + await import("@/lib/jobs/dense-intraday-retention") + ).enqueueBootTimeDenseIntradayRetention(), + }, + // Defers past the boot window and the retention drain's own stage. + { + name: "dense-intraday-hourly-rebuild", + run: async () => + ( + await import("@/lib/jobs/dense-intraday-hourly-rebuild") + ).enqueueBootTimeDenseIntradayHourlyRebuild(), + }, + { + name: "mood-rollup-full-backfill", + run: async () => + ( + await import("@/lib/rollups/mood-rollups") + ).enqueueBootTimeMoodRollupBackfill(BOOT_BACKFILL_STAGGER_SECONDS * 3), + }, + { + name: "step-consolidation-repair", + run: async () => + ( + await import("@/lib/jobs/step-consolidation-repair") + ).enqueueBootTimeStepConsolidationRepair( + BOOT_BACKFILL_STAGGER_SECONDS * 5, + ), + }, + { + name: "cumulative-pr-rederive", + run: async () => + ( + await import("@/lib/personal-records/cumulative-pr-rederivation") + ).enqueueBootTimeCumulativePrRederivation( + BOOT_BACKFILL_STAGGER_SECONDS * 6, + ), + }, +]; + +export const ROLLUP_GROUP: JobGroup = { + name: "rollup", + jobs, + retired: RETIRED_ROLLUP_QUEUES, + discovery, +}; diff --git a/src/lib/jobs/registry/status.ts b/src/lib/jobs/registry/status.ts new file mode 100644 index 000000000..104007bcd --- /dev/null +++ b/src/lib/jobs/registry/status.ts @@ -0,0 +1,260 @@ +/** + * The status, insight and computed-score queues: the nightly status batch, + * the comprehensive-insight pre-generation, the on-demand per-metric status, + * the morning digest refresh, the data-arrival spine with its reaction line + * and workout insight, the nightly scores, the period-narrative warm, the + * Coach memory refresh, nudge, reminder sweep and plan review, the low-stock + * pass and the daily-briefing fallback. + * + * Most queues here are keyless cron ticks, so they carry no policy. The + * LLM-bound warm queues (pre-generate, per-metric status, period narrative, + * Coach memory) pass `singletonSeconds` on every send, which pg-boss + * constrains whatever the policy, so they are left alone. The three that carry + * one key on the user's local date, which no wall-clock window can express. + */ +import type { Job } from "pg-boss"; + +import { COACH_MEMORY_REFRESH_QUEUE } from "@/lib/ai/coach/coach-memory-shared"; +import { DATA_ARRIVAL_QUEUE } from "@/lib/arrivals/emit-shared"; +import { REACTION_LINE_QUEUE } from "@/lib/arrivals/reaction-line-shared"; +import { + COACH_NUDGE_CRON, + COACH_NUDGE_QUEUE, +} from "@/lib/jobs/coach-nudge-queue"; +import { + COACH_PLAN_REVIEW_CRON, + COACH_PLAN_REVIEW_QUEUE, +} from "@/lib/jobs/coach-plan-review"; +import { + COACH_REMINDER_SWEEP_CRON, + COACH_REMINDER_SWEEP_QUEUE, +} from "@/lib/jobs/coach-reminder-sweep-queue"; +import { + DAILY_BRIEFING_CRON, + DAILY_BRIEFING_QUEUE, +} from "@/lib/jobs/daily-briefing-queue"; +import { + INSIGHT_PREGENERATE_CRON, + INSIGHT_PREGENERATE_EXPIRE_SECONDS, + INSIGHT_PREGENERATE_QUEUE, + type InsightPregeneratePayload, +} from "@/lib/jobs/insight-pregenerate-shared"; +import { + INSIGHT_STATUS_GENERATE_CONCURRENCY, + INSIGHT_STATUS_GENERATE_QUEUE, +} from "@/lib/jobs/insight-status-generate-queue"; +import { WHOLE_PASS } from "@/lib/jobs/long-pass"; +import { + MEDICATION_LOW_STOCK_CRON, + MEDICATION_LOW_STOCK_QUEUE, +} from "@/lib/jobs/medication-low-stock-queue"; +import { MORNING_DIGEST_REFRESH_QUEUE } from "@/lib/jobs/morning-digest-refresh-shared"; +import { + PERIOD_NARRATIVE_CRON, + PERIOD_NARRATIVE_EXPIRE_SECONDS, + PERIOD_NARRATIVE_QUEUE, +} from "@/lib/jobs/period-narrative-queue"; +import { + RECOVERY_SCORE_CRON, + RECOVERY_SCORE_QUEUE, +} from "@/lib/jobs/recovery-score-queue"; +import { + insightRetryOptions, + nightlyInsightPassOptions, +} from "@/lib/jobs/reminder/registrar-shared"; +import { + STRAIN_SCORE_CRON, + STRAIN_SCORE_QUEUE, +} from "@/lib/jobs/strain-score-queue"; +import { + STRESS_SCORE_CRON, + STRESS_SCORE_QUEUE, +} from "@/lib/jobs/stress-score-queue"; +import { + WORKOUT_INSIGHT_GENERATE_CONCURRENCY, + WORKOUT_INSIGHT_GENERATE_QUEUE, +} from "@/lib/jobs/workout-insight-generate-shared"; + +import { defineJob, type JobDefinition, type JobGroup } from "./define-job"; + +const GENERAL_STATUS_QUEUE = "insights-general-status"; +const GENERAL_STATUS_CRON = "0 2 * * *"; + +/** + * The six per-metric status crons that followed the 02:00 batch. The batch + * builds every card, so they only re-read cards it had just written; their + * schedules and queues are removed where they still exist. + */ +const RETIRED_STATUS_QUEUES = [ + "insights-blood-pressure-status", + "insights-weight-status", + "insights-pulse-status", + "insights-bmi-status", + "insights-mood-status", + "insights-medication-compliance-status", +] as const; + +const insights = () => import("@/lib/jobs/reminder/insights-handlers"); +const handlers = () => import("@/lib/jobs/reminder/status-handlers"); + +/** A pre-generation job that warms one account on a visit. */ +const isForcedWarm = (job: Job) => + Boolean(job.data?.force && job.data.userId); + +const jobs: readonly JobDefinition[] = [ + // Every per-metric status card, one provider call per account. The options + // ride on the schedule because the cron is the only sender. + defineJob({ + queue: GENERAL_STATUS_QUEUE, + cron: GENERAL_STATUS_CRON, + scheduleOptions: nightlyInsightPassOptions, + work: { localConcurrency: 1 }, + lock: () => WHOLE_PASS, + lane: true, + load: async () => (await insights()).handleGeneralStatusGenerate, + }), + // Two slots: the cohort walk and a visitor's forced warm. The walk is one + // pass at a time and runs in the lane; a forced warm is one per account and + // never waits behind the walk. + defineJob({ + queue: INSIGHT_PREGENERATE_QUEUE, + cron: INSIGHT_PREGENERATE_CRON, + scheduleOptions: { + ...insightRetryOptions, + expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS, + }, + work: { localConcurrency: 2 }, + lock: (job: Job) => + isForcedWarm(job) ? `user:${job.data.userId}` : WHOLE_PASS, + lane: (job: Job) => !isForcedWarm(job), + load: async () => (await insights()).handleInsightPregenerateJob, + }), + // Enqueued by the read-only status route on a cold card; low concurrency so + // a first visit cannot fan out unbounded provider calls. + defineJob({ + queue: INSIGHT_STATUS_GENERATE_QUEUE, + work: { localConcurrency: INSIGHT_STATUS_GENERATE_CONCURRENCY }, + load: async () => (await insights()).handleInsightStatusGenerate, + }), + // Seven sleep-write seams enqueue the same user + local-date key. + defineJob({ + queue: MORNING_DIGEST_REFRESH_QUEUE, + policy: { + policy: "exclusive", + reason: + "Seven sleep-write seams enqueue the same user+localDate key; exclusive makes the at-most-once-per-morning contract real. A wall-clock singletonSeconds window cannot express a local date.", + }, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleMorningDigestRefresh, + }), + // The arrival spine claims a durable unique row, so a second slot would only + // race the claim; it makes no provider call. + defineJob({ + queue: DATA_ARRIVAL_QUEUE, + policy: { + policy: "exclusive", + reason: + "Day-scoped arrival keys carry the user's local date; under standard they coalesce nothing and a chatty ingest queues a job per batch. The worker claims a durable unique row, so a concurrent second run is pure duplicated work.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/data-arrival")).handleDataArrival, + }), + // The spine's one provider call, behind the durable `generatedAt` claim. + defineJob({ + queue: REACTION_LINE_QUEUE, + policy: { + policy: "exclusive", + reason: + "Day-scoped per-kind keys carry the user's local date; under standard they coalesce nothing and a retried spine job could stack a second provider call behind the first.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reaction-line")).handleReactionLine, + }), + defineJob({ + queue: WORKOUT_INSIGHT_GENERATE_QUEUE, + work: { localConcurrency: WORKOUT_INSIGHT_GENERATE_CONCURRENCY }, + load: async () => + (await import("@/lib/jobs/workout-insight-generate")) + .handleWorkoutInsightGenerate, + }), + // The nightly scores, in order after the consolidation and drain they read: + // recovery 04:45, stress 04:50, strain 04:55. + defineJob({ + queue: RECOVERY_SCORE_QUEUE, + cron: RECOVERY_SCORE_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleRecoveryScore, + }), + defineJob({ + queue: STRESS_SCORE_QUEUE, + cron: STRESS_SCORE_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleStressScore, + }), + defineJob({ + queue: STRAIN_SCORE_QUEUE, + cron: STRAIN_SCORE_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleStrainScore, + }), + // The nightly fan-out runs on a week or month boundary, in the lane and one + // pass at a time; a single account's warm for the GET runs beside it. + defineJob({ + queue: PERIOD_NARRATIVE_QUEUE, + cron: PERIOD_NARRATIVE_CRON, + scheduleOptions: { + ...insightRetryOptions, + expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS, + }, + work: { localConcurrency: 1 }, + lock: (job: Job<{ userId?: string }>) => + job.data?.userId ? `user:${job.data.userId}` : WHOLE_PASS, + lane: (job: Job<{ userId?: string }>) => !job.data?.userId, + load: async () => (await handlers()).handlePeriodNarrative, + }), + // One slot, so a burst of long conversations cannot fan out provider calls. + defineJob({ + queue: COACH_MEMORY_REFRESH_QUEUE, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleCoachMemoryRefresh, + }), + defineJob({ + queue: COACH_NUDGE_QUEUE, + cron: COACH_NUDGE_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleCoachNudge, + }), + defineJob({ + queue: COACH_REMINDER_SWEEP_QUEUE, + cron: COACH_REMINDER_SWEEP_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleCoachReminderSweep, + }), + defineJob({ + queue: COACH_PLAN_REVIEW_QUEUE, + cron: COACH_PLAN_REVIEW_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleCoachPlanReview, + }), + defineJob({ + queue: MEDICATION_LOW_STOCK_QUEUE, + cron: MEDICATION_LOW_STOCK_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleMedicationLowStock, + }), + // Every 15 minutes; the per-user local-hour gate picks each user's slot. + defineJob({ + queue: DAILY_BRIEFING_QUEUE, + cron: DAILY_BRIEFING_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleDailyBriefing, + }), +]; + +export const STATUS_GROUP: JobGroup = { + name: "status", + jobs, + retired: RETIRED_STATUS_QUEUES, +}; diff --git a/src/lib/jobs/reminder-worker.ts b/src/lib/jobs/reminder-worker.ts index e486a92db..c882ca002 100644 --- a/src/lib/jobs/reminder-worker.ts +++ b/src/lib/jobs/reminder-worker.ts @@ -1,15 +1,12 @@ /** - * pg-boss based reminder worker — boot composition. + * pg-boss worker boot. * - * v1.18.1 — this file was a 2143-LOC monolith that declared every pg-boss - * queue name, listed them in one `allQueues` array, scheduled every cron, and - * bound every `boss.work` handler inline. It is now a thin registration table: - * `startReminderWorker()` boots pg-boss, runs the boot one-shots, then calls - * the domain registrars under `src/lib/jobs/reminder/`. Each registrar owns the - * four facts the v1.4.37 dead-queue guards pin — the queue-name constant, its - * `allQueues` membership, its `[QUEUE, CRON]` schedule tuple, and its - * `createAndWork(boss, QUEUE, …, handler)` binding — so a queue can never be - * declared without being provisioned, scheduled, and drained. + * `startReminderWorker()` starts pg-boss, registers every queue from the + * declarative registry (`./registry/`), runs the boot one-shots and the boot + * discovery. The registry declares each queue with its schedule, policy, + * concurrency and handler loader; a queue cannot be declared without being + * created, scheduled and drained. Handler modules load on their queue's first + * job, not at boot. * * Usage: Run as a standalone process or call startReminderWorker() from a * custom server setup. In dev, use: npx tsx src/lib/jobs/reminder-worker.ts @@ -20,23 +17,15 @@ import { reportJobsCutOffAtBoot } from "@/lib/jobs/job-observer"; import { beginBootStep, reportBootSteps } from "@/lib/boot/boot-steps"; import { assertSubsystemEnabled } from "@/lib/process-type"; import { DATABASE_URL, workerLog } from "./reminder/shared"; -import type { PgBoss } from "pg-boss"; /** - * Give the event loop a turn. Each registrar's module graph is evaluated in - * one synchronous stretch when it is first imported; loading the five one - * after another, with a turn in between, keeps the longest stall to the - * largest graph instead of the sum of all five, so requests and the health - * probe are answered in between. + * Give the event loop a turn around a module load, so requests and the + * health probe are answered between boot steps. */ const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve)); -/** - * Import one registrar module as its own boot step. The registrars and the - * handler graphs behind them are loaded here, after `boss.start()`, rather - * than when this module is imported. - */ -async function loadRegistrar( +/** Import one boot module as its own boot step. */ +async function loadBootModule( name: string, load: () => Promise, ): Promise { @@ -74,65 +63,25 @@ export async function startReminderWorker() { markWorkerStarted(); // Graceful shutdown: drain in-flight jobs on SIGTERM/SIGINT. - const lifecycle = await loadRegistrar( + const lifecycle = await loadBootModule( "worker_lifecycle", () => import("./reminder/worker-lifecycle"), ); lifecycle.registerShutdownHandlers(boss); - // Register every domain's queues (createQueue + schedule + boss.work). Each - // registrar returns the queue names it provisioned; the aggregate is the - // boot-level mirror of the per-registrar `allQueues` arrays the dead-queue - // guards pin. Order preserves the monolith's createQueue / schedule / work - // sequence (integration → status → rollup → reminders → maintenance). - const integration = await loadRegistrar( - "integration_sync", - () => import("./reminder/register-integration-sync"), + // Register every queue (createQueue + schedule + boss.work) from the + // declarative registry, one group per boot step. The registry holds + // declarations only; each handler module loads on its queue's first job. + const registry = await loadBootModule( + "registry", + () => import("./registry/index"), ); - const registeredQueues = await registerInStep( - "integration_sync", - boss, - integration.registerIntegrationSyncQueues, - ); - const status = await loadRegistrar( - "status", - () => import("./reminder/register-status"), - ); - registeredQueues.push( - ...(await registerInStep("status", boss, status.registerStatusQueues)), - ); - const rollup = await loadRegistrar( - "rollup", - () => import("./reminder/register-rollup"), - ); - registeredQueues.push( - ...(await registerInStep("rollup", boss, rollup.registerRollupQueues)), - ); - const reminders = await loadRegistrar( - "reminders", - () => import("./reminder/register-reminders"), - ); - registeredQueues.push( - ...(await registerInStep( - "reminders", - boss, - reminders.registerReminderQueues, - )), - ); - const maintenance = await loadRegistrar( - "maintenance", - () => import("./reminder/register-maintenance"), - ); - registeredQueues.push( - ...(await registerInStep( - "maintenance", - boss, - maintenance.registerMaintenanceQueues, - )), + const registeredQueues = await registry.registerAllJobGroups(boss, (group) => + beginBootStep(`register_${group.name}`), ); workerLog( "info", - `[reminder-worker] registered ${registeredQueues.length} queues across 5 domain registrars`, + `[reminder-worker] registered ${registeredQueues.length} queues across ${registry.JOB_GROUPS.length} groups`, ); // v1.4.34 — reconcile any `ImportJob` rows that were mid-parse when the @@ -146,26 +95,14 @@ export async function startReminderWorker() { // Fire-and-forget boot discovery for every self-converging backfill / // consolidation pass. Each is idempotent across reboots and never fails - // worker boot on a miss (errors come back through the helper's result). + // worker boot on a miss. beginBootStep("boot_discovery"); - await rollup.enqueueRollupBootDiscovery(); - await integration.enqueueIntegrationSyncBootDiscovery(); - await maintenance.enqueueMaintenanceBootDiscovery(); + await registry.runAllBootDiscovery(); await reportBootSteps({ boot_queue_count: registeredQueues.length }); return boss; } -/** Run one registrar as its own boot step. */ -async function registerInStep( - name: string, - boss: PgBoss, - register: (boss: PgBoss) => Promise, -): Promise { - beginBootStep(`register_${name}`); - return [...(await register(boss))]; -} - // Run standalone if ( process.argv[1]?.endsWith("reminder-worker.ts") || diff --git a/src/lib/jobs/reminder/climb-backfill-at-boot.ts b/src/lib/jobs/reminder/climb-backfill-at-boot.ts new file mode 100644 index 000000000..344b9c017 --- /dev/null +++ b/src/lib/jobs/reminder/climb-backfill-at-boot.ts @@ -0,0 +1,34 @@ +/** + * The one-time climb backfill for workouts stored with a track but no + * elevation (`workouts/climb-backfill.ts`), started at boot and left to run + * detached: it reads one track at a time with a pause between batches and + * resumes from its cursor on the next boot, so it never holds the boot or the + * heap. A finished pass returns at once. + */ +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; + +import { workerLog } from "./shared"; + +export async function startClimbBackfill(): Promise { + trackBackgroundTask( + (async () => { + const [{ prisma }, { runWorkoutClimbBackfill }] = await Promise.all([ + import("@/lib/db"), + import("@/lib/workouts/climb-backfill"), + ]); + const result = await runWorkoutClimbBackfill({ prisma }); + if (result.scanned > 0) { + workerLog( + "info", + `[workout-climb-backfill] scanned=${result.scanned} filled=${result.filled} done=${result.done}`, + ); + } + })().catch((err) => { + workerLog( + "error", + "[workout-climb-backfill] failed; the next boot resumes from the cursor", + err, + ); + }), + ); +} diff --git a/src/lib/jobs/reminder/document-handlers.ts b/src/lib/jobs/reminder/document-handlers.ts new file mode 100644 index 000000000..fa7c2d55c --- /dev/null +++ b/src/lib/jobs/reminder/document-handlers.ts @@ -0,0 +1,233 @@ +/** + * Handlers of the document vault queues that are not a module's own export: + * the content-index backfill, the per-document index, thumbnail and summary, + * the auto-read catch-up and the thumbnail backfill. A payload without its ids + * is counted as skipped rather than failed. The registry binds them + * (`registry/maintenance.ts`). + */ +import type { Job } from "pg-boss"; + +import { + runContentIndexBackfillForUser, + type ContentIndexBackfillPayload, +} from "@/lib/jobs/document-content-index-backfill"; +import { + runDocumentIndex, + type DocumentIndexPayload, +} from "@/lib/jobs/document-index"; +import { + runDocumentSummaryJob, + type DocumentSummaryPayload, +} from "@/lib/jobs/document-summary"; +import { + runSummaryCatchUpForUser, + type SummaryCatchUpPayload, +} from "@/lib/jobs/document-summary-catchup"; +import { + runDocumentThumbnail, + type DocumentThumbnailPayload, +} from "@/lib/jobs/document-thumbnail"; +import { + runThumbnailBackfillForUser, + type ThumbnailBackfillPayload, +} from "@/lib/jobs/document-thumbnail-backfill"; +import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { recordError } from "@/lib/jobs/worker-status"; + +import { workerLog } from "./shared"; + +/** + * The on-demand "index all documents" run for one account: consent and + * budget gated, bounded and resumable. + */ +export async function handleContentIndexBackfill( + jobs: Job[], +): Promise { + let users = 0; + let stoppedEarly = false; + const shouldStop = jobBudget(jobs); + let skipped = 0; + let documentsIndexed = 0; + let documentsRetokenised = 0; + let documentsSkipped = 0; + let documentsFailed = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + skipped++; + continue; + } + try { + const { + indexed, + retokenised, + skipped: docsSkipped, + failed: docsFailed, + reason, + stoppedEarly: stopped, + } = await runContentIndexBackfillForUser(userId, shouldStop); + users++; + stoppedEarly ||= stopped === true; + documentsIndexed += indexed; + documentsRetokenised += retokenised; + documentsSkipped += docsSkipped; + documentsFailed += docsFailed; + workerLog( + "info", + `[document-content-index-backfill] user=${userId} indexed=${indexed} retokenised=${retokenised} skipped=${docsSkipped} failed=${docsFailed} reason=${reason}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[document-content-index-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + skipped, + documents_indexed: documentsIndexed, + documents_retokenised: documentsRetokenised, + documents_skipped: documentsSkipped, + documents_failed: documentsFailed, + stopped_early: stoppedEarly, + }); +} + +/** One stored document's job, by the step that runs it. */ +function perDocument

( + name: string, + run: (payload: P) => Promise, +): (jobs: Job

[]) => Promise { + return async (jobs) => { + let documents = 0; + let skipped = 0; + for (const job of jobs) { + const { userId, documentId } = job.data; + if (!userId || !documentId) { + skipped++; + continue; + } + try { + await run(job.data); + documents++; + } catch (err) { + recordError(); + workerLog( + "error", + `[${name}] user=${userId} document=${documentId} failed`, + err, + ); + throw err; + } + } + return jobDone({ jobs: jobs.length, documents, skipped }); + }; +} + +/** + * Content indexing on upload: the vision provider when configured and + * consented, the local text layer otherwise. + */ +export const handleDocumentIndex = perDocument( + "document-index", + runDocumentIndex, +); + +/** The preview thumbnail on upload: local decode and downscale, no egress. */ +export const handleDocumentThumbnail = perDocument( + "document-thumbnail", + runDocumentThumbnail, +); + +/** + * The plain-language summary on upload, only with the auto-read opt-in on + * (consent and budget gated), stored encrypted. + */ +export const handleDocumentSummary = perDocument( + "document-summary", + runDocumentSummaryJob, +); + +/** + * After a genuine off-to-on flip of the auto-read opt-in: one summary job per + * stored document the upload-time enqueue skipped. Every gate still runs + * inside the summary job. + */ +export async function handleDocumentSummaryCatchUp( + jobs: Job[], +): Promise { + let users = 0; + let skipped = 0; + let summariesEnqueued = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + skipped++; + continue; + } + try { + const { enqueued } = await runSummaryCatchUpForUser(userId); + users++; + summariesEnqueued += enqueued; + } catch (err) { + recordError(); + workerLog( + "error", + `[document-summary-catchup] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + skipped, + summaries_enqueued: summariesEnqueued, + }); +} + +/** One thumbnail job per not-yet-previewed document of the account. */ +export async function handleDocumentThumbnailBackfill( + jobs: Job[], +): Promise { + let users = 0; + let skipped = 0; + let thumbnailsEnqueued = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + skipped++; + continue; + } + try { + const { enqueued } = await runThumbnailBackfillForUser(userId); + users++; + thumbnailsEnqueued += enqueued; + workerLog( + "info", + `[document-thumbnail-backfill] user=${userId} enqueued=${enqueued}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[document-thumbnail-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + skipped, + thumbnails_enqueued: thumbnailsEnqueued, + }); +} diff --git a/src/lib/jobs/reminder/encryption-backfill-handlers.ts b/src/lib/jobs/reminder/encryption-backfill-handlers.ts new file mode 100644 index 000000000..33a8bde60 --- /dev/null +++ b/src/lib/jobs/reminder/encryption-backfill-handlers.ts @@ -0,0 +1,214 @@ +/** + * Handlers of the three converging encryption backfills: the mood and + * measurement notes, the medication notes, and the remaining free text (Coach + * titles, custom-metric notes, practitioner contacts, workout tracks). A job + * with a `userId` migrates that account; the daily tick (no `userId`) re-runs + * the boot discovery and fans out one job per account still holding a + * readable value. The registry binds them (`registry/maintenance.ts`). + */ +import type { Job } from "pg-boss"; + +import { + enqueueBootTimeFreeTextEncryptionBackfill, + runFreeTextEncryptionBackfillForUser, + type FreeTextEncryptionBackfillPayload, +} from "@/lib/jobs/free-text-encryption-backfill"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { + enqueueBootTimeMedNotesEncryptionBackfill, + runMedNotesEncryptionBackfillForUser, + type MedNotesEncryptionBackfillPayload, +} from "@/lib/jobs/med-notes-encryption-backfill"; +import { + enqueueBootTimeNoteEncryptionBackfill, + runNoteEncryptionBackfillForUser, + type NoteEncryptionBackfillPayload, +} from "@/lib/jobs/note-encryption-backfill"; +import { recordError } from "@/lib/jobs/worker-status"; + +import { workerLog } from "./shared"; + +export async function handleNoteEncryptionBackfill( + jobs: Job>[], +): Promise { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + // The discovery helper returns its error instead of throwing, so a failed + // fan-out would otherwise leave no trace on the job at all. + let discoveryFailed = 0; + let measurements = 0; + let moodEntries = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + const result = await enqueueBootTimeNoteEncryptionBackfill(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; + workerLog( + "info", + `[note-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, + ); + continue; + } + try { + const { measurementsMigrated, moodEntriesMigrated } = + await runNoteEncryptionBackfillForUser(userId); + users++; + measurements += measurementsMigrated; + moodEntries += moodEntriesMigrated; + workerLog( + "info", + `[note-encryption-backfill] user=${userId} measurements=${measurementsMigrated} moodEntries=${moodEntriesMigrated}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[note-encryption-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + measurements_migrated: measurements, + mood_entries_migrated: moodEntries, + }); +} + +export async function handleMedNotesEncryptionBackfill( + jobs: Job>[], +): Promise { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + let discoveryFailed = 0; + let sideEffects = 0; + let doseChanges = 0; + let inventoryItems = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + const result = await enqueueBootTimeMedNotesEncryptionBackfill(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; + workerLog( + "info", + `[med-notes-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, + ); + continue; + } + try { + const { + sideEffectsMigrated, + doseChangesMigrated, + inventoryItemsMigrated, + } = await runMedNotesEncryptionBackfillForUser(userId); + users++; + sideEffects += sideEffectsMigrated; + doseChanges += doseChangesMigrated; + inventoryItems += inventoryItemsMigrated; + workerLog( + "info", + `[med-notes-encryption-backfill] user=${userId} sideEffects=${sideEffectsMigrated} doseChanges=${doseChangesMigrated} inventoryItems=${inventoryItemsMigrated}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[med-notes-encryption-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + side_effects_migrated: sideEffects, + dose_changes_migrated: doseChanges, + inventory_items_migrated: inventoryItems, + }); +} + +export async function handleFreeTextEncryptionBackfill( + jobs: Job>[], +): Promise { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + let discoveryFailed = 0; + let titles = 0; + let notes = 0; + let contacts = 0; + let routes = 0; + let appointmentAddresses = 0; + let contactAudits = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + const result = await enqueueBootTimeFreeTextEncryptionBackfill(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; + workerLog( + "info", + `[free-text-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, + ); + continue; + } + try { + const { + conversationTitlesMigrated, + metricNotesMigrated, + practitionerContactsMigrated, + routeGeometriesMigrated, + appointmentAddressesCleared, + contactAuditRowsScrubbed, + } = await runFreeTextEncryptionBackfillForUser(userId); + users++; + titles += conversationTitlesMigrated; + notes += metricNotesMigrated; + contacts += practitionerContactsMigrated; + routes += routeGeometriesMigrated; + appointmentAddresses += appointmentAddressesCleared; + contactAudits += contactAuditRowsScrubbed; + workerLog( + "info", + `[free-text-encryption-backfill] user=${userId} titles=${conversationTitlesMigrated} notes=${metricNotesMigrated} contacts=${practitionerContactsMigrated} routes=${routeGeometriesMigrated} appointment_addresses=${appointmentAddressesCleared} contact_audits=${contactAuditRowsScrubbed}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[free-text-encryption-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + conversation_titles_migrated: titles, + metric_notes_migrated: notes, + practitioner_contacts_migrated: contacts, + route_geometries_migrated: routes, + appointment_addresses_cleared: appointmentAddresses, + contact_audit_rows_scrubbed: contactAudits, + }); +} diff --git a/src/lib/jobs/reminder/maintenance-handlers.ts b/src/lib/jobs/reminder/maintenance-handlers.ts new file mode 100644 index 000000000..1f1facfd7 --- /dev/null +++ b/src/lib/jobs/reminder/maintenance-handlers.ts @@ -0,0 +1,186 @@ +/** + * Handlers of the maintenance queues that are not a module's own export: the + * Apple Health and medication-intake imports, the duplicate dose-slot dedup, + * and the boss-bound environment and Dawarich passes. The registry binds them + * (`registry/maintenance.ts`) and loads this module on the first job. + */ +import type { Job, JobWithMetadata, PgBoss } from "pg-boss"; + +import { + handleAppleHealthImport, + migrateLegacyAppleHealthImport, + type AppleHealthImportPayload, +} from "@/lib/jobs/apple-health-import-worker"; +import { + handleDawarichSync, + type DawarichSyncPayload, +} from "@/lib/jobs/dawarich-sync"; +import { + handleEnvironmentAqHistory, + type EnvironmentAqHistoryPayload, +} from "@/lib/jobs/environment-air-quality-history"; +import { + handleEnvironmentFetch, + type EnvironmentFetchPayload, +} from "@/lib/jobs/environment-fetch"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { + handleMedicationIntakeImport, + type MedicationIntakeImportQueuePayload, +} from "@/lib/jobs/medication-intake-import"; +import { recordError } from "@/lib/jobs/worker-status"; +import { + dedupeUserIntakeSlots, + enqueueBootTimeIntakeSlotDedup, + type IntakeSlotDedupPayload, +} from "@/lib/medications/intake-slot-dedup"; + +import { workerLog } from "./shared"; + +/** + * The Apple Health export ingest. Each import is independent work, so a + * failed one does not abort the rest of the batch; the first failure is what + * the job reports. + */ +export async function handleAppleHealthImportJobs( + jobs: Job[], +): Promise { + let failure: JobOutcome | null = null; + for (const job of jobs) { + const outcome = await handleAppleHealthImport(job); + if (!outcome.ok && failure === null) failure = outcome; + } + return failure ?? jobDone({ jobs: jobs.length }); +} + +/** + * Drains the pre-revision-2 backlog by moving each claimed legacy job onto + * the current queue, keeping the import's status row, so the current parser + * never runs under a revision-1 mirror. + */ +export async function handleLegacyAppleHealthImportJobs( + jobs: Job[], +): Promise { + for (const job of jobs) { + await migrateLegacyAppleHealthImport(job); + } + return jobDone({ jobs: jobs.length }); +} + +/** + * Independent per-delivery medication-intake imports: run them all and report + * the first failure rather than losing it behind a later success. Bound with + * metadata, so a retry resumes from the job's creation time. + */ +export async function handleMedicationIntakeImportJobs( + jobs: JobWithMetadata[], +): Promise { + let failure: JobOutcome | null = null; + for (const job of jobs) { + const outcome = await handleMedicationIntakeImport(job); + if (!outcome.ok && failure === null) failure = outcome; + } + return failure ?? jobDone({ jobs: jobs.length }); +} + +/** + * The duplicate dose-slot cleanup. A job with a `userId` keeps one account's + * winner per slot, soft-deletes the losers and recomputes the affected + * compliance days; the daily tick (no `userId`) runs the same discovery + * fan-out as the boot pass, so slots duplicated between deploys collapse + * within a day. + */ +export async function handleIntakeSlotDedup( + jobs: Job>[], +): Promise { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + // The discovery helper returns its error instead of throwing, so a failed + // fan-out would otherwise leave no trace on the job at all. + let discoveryFailed = 0; + let slotsCollapsed = 0; + let rowsSoftDeleted = 0; + let rowsNormalised = 0; + let daysRecomputed = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + const result = await enqueueBootTimeIntakeSlotDedup(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; + workerLog( + "info", + `[intake-slot-dedup] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, + ); + continue; + } + try { + const summary = await dedupeUserIntakeSlots(userId); + users++; + slotsCollapsed += summary.slotsCollapsed; + rowsSoftDeleted += summary.rowsSoftDeleted; + rowsNormalised += summary.rowsNormalised; + daysRecomputed += summary.daysRecomputed; + workerLog( + "info", + `[intake-slot-dedup] user=${userId} slotsCollapsed=${summary.slotsCollapsed} rowsSoftDeleted=${summary.rowsSoftDeleted} rowsNormalised=${summary.rowsNormalised} daysRecomputed=${summary.daysRecomputed}`, + ); + } catch (err) { + recordError(); + workerLog("error", `[intake-slot-dedup] user=${userId} failed`, err); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + slots_collapsed: slotsCollapsed, + rows_soft_deleted: rowsSoftDeleted, + rows_normalised: rowsNormalised, + days_recomputed: daysRecomputed, + }); +} + +/** The air-quality history backfill; the first failure is what the job reports. */ +export function environmentAqHistoryHandler( + boss: PgBoss, +): (jobs: Job[]) => Promise { + return async (jobs) => { + let failure: JobOutcome | null = null; + for (const job of jobs) { + const outcome = await handleEnvironmentAqHistory(boss, job.data); + if (!outcome.ok && failure === null) failure = outcome; + } + return failure ?? jobDone({ jobs: jobs.length }); + }; +} + +/** + * The environment fetch. A discovery tick and a per-user backfill share the + * queue and are independent, so the pass runs to the end and reports the + * first failure it saw. + */ +export function environmentFetchHandler( + boss: PgBoss, +): (jobs: Job[]) => Promise { + return async (jobs) => { + let failure: JobOutcome | null = null; + for (const job of jobs) { + const outcome = await handleEnvironmentFetch(boss, job.data); + if (!outcome.ok && failure === null) failure = outcome; + } + return failure ?? jobDone({ jobs: jobs.length }); + }; +} + +/** The Dawarich movement context, one account at a time. */ +export function dawarichSyncHandler( + boss: PgBoss, +): (jobs: Job[]) => Promise { + return (jobs) => handleDawarichSync(boss, jobs); +} diff --git a/src/lib/jobs/reminder/register-integration-sync.ts b/src/lib/jobs/reminder/register-integration-sync.ts deleted file mode 100644 index d96055632..000000000 --- a/src/lib/jobs/reminder/register-integration-sync.ts +++ /dev/null @@ -1,1051 +0,0 @@ -/** - * Integration-sync queue registrar. - * - * Owns the third-party-provider sync queues — Withings (fallback / activity / - * sleep), WHOOP (recovery / sleep / workout / cycle + backfill + OAuth-state - * cleanup), Fitbit (sync + backfill + sleep repair + OAuth-state cleanup), - * Nightscout, Polar, Oura, and the two one-shot backfills - * (sleep-timeline, lab-biomarker) — plus their boot-time discovery enqueues. - * - * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its - * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The - * queue-wiring guards (`withings-queues`, `whoop-queues`, `fitbit-queues`, - * `nightscout-queues`, `sleep-timeline-queues`) read THIS module. - */ -import { PgBoss } from "pg-boss"; -import { - WHOOP_BACKFILL_QUEUE, - WHOOP_BACKFILL_CONCURRENCY, - enqueueBootTimeWhoopBackfill, - type WhoopBackfillPayload, -} from "@/lib/jobs/whoop-backfill"; -import { - FITBIT_BACKFILL_QUEUE, - FITBIT_BACKFILL_CONCURRENCY, - enqueueBootTimeFitbitBackfill, - type FitbitBackfillPayload, -} from "@/lib/jobs/fitbit-backfill"; -import { - GOOGLE_HEALTH_BACKFILL_QUEUE, - GOOGLE_HEALTH_BACKFILL_CONCURRENCY, - enqueueBootTimeGoogleHealthBackfill, - type GoogleHealthBackfillPayload, -} from "@/lib/jobs/google-health-backfill"; -import { - GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, - GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY, - enqueueBootTimeGoogleHealthSleepRepair, - type GoogleHealthSleepRepairPayload, -} from "@/lib/jobs/google-health-sleep-repair"; -import { - FITBIT_SLEEP_REPAIR_QUEUE, - FITBIT_SLEEP_REPAIR_CONCURRENCY, - enqueueBootTimeFitbitSleepRepair, - type FitbitSleepRepairPayload, -} from "@/lib/jobs/fitbit-sleep-repair"; -import { - STRAVA_BACKFILL_QUEUE, - STRAVA_BACKFILL_CONCURRENCY, - enqueueBootTimeStravaBackfill, - type StravaBackfillPayload, -} from "@/lib/jobs/strava-backfill"; -import { - OURA_BACKFILL_QUEUE, - OURA_BACKFILL_CONCURRENCY, - enqueueBootTimeOuraBackfill, - type OuraBackfillPayload, -} from "@/lib/jobs/oura-backfill"; -import { - SLEEP_TIMELINE_BACKFILL_QUEUE, - SLEEP_TIMELINE_BACKFILL_CONCURRENCY, - enqueueBootTimeSleepTimelineBackfill, - type SleepTimelineBackfillPayload, -} from "@/lib/jobs/sleep-timeline-backfill"; -import { - LAB_BIOMARKER_BACKFILL_QUEUE, - LAB_BIOMARKER_BACKFILL_CONCURRENCY, - enqueueBootTimeLabBiomarkerBackfill, - type LabBiomarkerBackfillPayload, -} from "@/lib/jobs/lab-biomarker-backfill"; -import { - INTEGRATION_BACKFILL_ADMISSION_QUEUE, - INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, - bootStaggerSecondsFor, - enqueueIntegrationBackfillAdmission, - type IntegrationBackfillAdmissionPayload, -} from "@/lib/jobs/integration-backfill-admission"; -import { drainIntegrationBackfillAdmission } from "@/lib/jobs/integration-backfill-drain"; -import { heavyLane } from "@/lib/jobs/heavy-lane"; -import { - INTEGRATION_WATCHDOG_DAILY_CRON, - INTEGRATION_WATCHDOG_DAILY_QUEUE, - INTEGRATION_WATCHDOG_HOURLY_CRON, - INTEGRATION_WATCHDOG_HOURLY_QUEUE, - handleIntegrationWatchdogDaily, - handleIntegrationWatchdogHourly, - type IntegrationWatchdogPayload, -} from "@/lib/jobs/integration-watchdog"; -import { workerLog } from "./shared"; -import { jobDone } from "@/lib/jobs/job-outcome"; -import { - createAndSchedule, - createAndWork, - cronIsTheRetry, - type QueuePolicyTable, - type ScheduleEntry, -} from "./registrar-shared"; -import { WITHINGS_ECG_SYNC_QUEUE } from "@/lib/jobs/withings-ecg-queue"; -import { - NIGHTSCOUT_SYNC_QUEUE, - OURA_SYNC_QUEUE, - POLAR_SYNC_QUEUE, - STRAVA_SYNC_QUEUE, -} from "@/lib/jobs/integration-poll-queues"; -import { - WithingsSyncPayload, - WithingsActivitySyncPayload, - WithingsSleepSyncPayload, - WithingsEcgSyncPayload, - handleWithingsFallbackSync, - handleWithingsActivitySync, - handleWithingsSleepSync, - handleWithingsEcgSync, -} from "./withings-sync"; -import { - WhoopSyncPayload, - handleWhoopRecoverySync, - handleWhoopSleepSync, - handleWhoopWorkoutSync, - handleWhoopCycleSync, -} from "./whoop-sync"; -import { - WithingsOAuthStateCleanupPayload, - handleWithingsOAuthStateCleanup, - WhoopOAuthStateCleanupPayload, - handleWhoopOAuthStateCleanup, - OidcNativeHandoffCleanupPayload, - handleOidcNativeHandoffCleanup, -} from "./cleanup-handlers"; -import { NightscoutSyncPayload, handleNightscoutSync } from "./nightscout-sync"; -import { PolarSyncPayload, handlePolarSync } from "./polar-sync"; -import { OuraSyncPayload, handleOuraSync } from "./oura-sync"; -import { StravaSyncPayload, handleStravaSync } from "./strava-sync"; -import { - FitbitSyncPayload, - handleFitbitSync, - FitbitOAuthStateCleanupPayload, - handleFitbitOAuthStateCleanup, -} from "./fitbit-sync"; -import { - GoogleHealthSyncPayload, - handleGoogleHealthSync, - GoogleHealthOAuthStateCleanupPayload, - handleGoogleHealthOAuthStateCleanup, -} from "./google-health-sync"; - -const WITHINGS_SYNC_QUEUE = "withings-fallback-sync"; - -const WITHINGS_SYNC_CRON = "0 * * * *"; // every 60 minutes -// v1.4.25 — webhook-primary + cron-safety-net for activity and -// sleep v2. The webhook handler enqueues per-user jobs on appli=16 / 44 -// notifications; the crons below are the catch-net for the 1 % of -// notifications Withings drops. Offset 15-minute cadence per the -// research recommendation so the two queues don't lockstep against the -// existing measure cron at :00. - -const WITHINGS_ACTIVITY_QUEUE = "withings-activity-sync"; - -const WITHINGS_ACTIVITY_CRON = "0 * * * *"; // every hour at :00 - -const WITHINGS_SLEEP_QUEUE = "withings-sleep-sync"; - -const WITHINGS_SLEEP_CRON = "15 * * * *"; // every hour at :15 -// v1.32.23 — Withings ECG cron catch-net. The queue is webhook-primary -// (appli=54 → `enqueueWithingsEcgSync`), but appli 54 is not in the -// subscription set and the manual sync route measures only, so a ScanWatch-only -// account (no scale) never had its ECG strips pulled. An hourly cohort walk -// (one `v2/heart list` call per connected user, empty + cheap for non-ECG -// devices) closes the gap. :41 is a free minute (used: 0,5,8,11,13,15,17,18, -// 20,30,35,50) staggered off every other hourly sync tick. -const WITHINGS_ECG_SYNC_CRON = "41 * * * *"; // every hour at :41 - -// v1.4.47 — daily sweep for the Withings OAuth state ledger. Slots -// at :20 between the audit-log cleanup (:15) and the mood-reminder -// cleanup (:25), inside the existing 02:xx-03:xx maintenance window. -// Rows are normally consumed by the callback handler in single-use -// fashion; this sweep picks up the long tail where a user closed the -// Withings approval tab without bouncing back to the callback URL. - -const WITHINGS_OAUTH_STATE_CLEANUP_QUEUE = "withings-oauth-state-cleanup"; - -const WITHINGS_OAUTH_STATE_CLEANUP_CRON = "20 3 * * *"; -// v1.11.0 — WHOOP sync queues. Webhook-primary + cron-safety-net, mirroring -// the Withings activity/sleep crons. Recovery / sleep / workout each have a -// WHOOP webhook (`*.updated`) that enqueues the matching per-resource job; the -// crons below are the catch-net for dropped deliveries. Cycle has NO webhook, -// so its cron is the only driver. Minutes are staggered off the Withings crons -// (:00/:15) to spread DB load. - -const WHOOP_RECOVERY_SYNC_QUEUE = "whoop-recovery-sync"; - -const WHOOP_RECOVERY_SYNC_CRON = "5 * * * *"; // every hour at :05 - -const WHOOP_SLEEP_SYNC_QUEUE = "whoop-sleep-sync"; - -const WHOOP_SLEEP_SYNC_CRON = "20 * * * *"; // every hour at :20 - -const WHOOP_WORKOUT_SYNC_QUEUE = "whoop-workout-sync"; - -const WHOOP_WORKOUT_SYNC_CRON = "35 * * * *"; // every hour at :35 - -const WHOOP_CYCLE_SYNC_QUEUE = "whoop-cycle-sync"; - -const WHOOP_CYCLE_SYNC_CRON = "50 * * * *"; // every hour at :50 (poll-only) -// v1.11.0 — daily sweep for the WHOOP OAuth state ledger. Slots at 03:22, -// next to the Withings sweep (03:20), inside the maintenance window. - -const WHOOP_OAUTH_STATE_CLEANUP_QUEUE = "whoop-oauth-state-cleanup"; - -const WHOOP_OAUTH_STATE_CLEANUP_CRON = "22 3 * * *"; -// v1.12.0 — Fitbit / Google Health poll-only sync. There is no Fitbit webhook -// at launch (Pub/Sub deferred), so a single hourly cron drives the per-user -// `syncUserFitbit` driver across every connection. Minute staggered off the -// WHOOP slots (:05/:20/:35/:50) and the Withings slots (:00/:15) so the hourly -// ticks don't pile up on one boss poll. - -const FITBIT_SYNC_QUEUE = "fitbit-sync"; - -const FITBIT_SYNC_CRON = "8 * * * *"; // every hour at :08 -// v1.12.0 — daily sweep for the Fitbit OAuth state ledger. Slots at 03:24, next -// to the WHOOP sweep (03:22), inside the maintenance window. - -const FITBIT_OAUTH_STATE_CLEANUP_QUEUE = "fitbit-oauth-state-cleanup"; - -const FITBIT_OAUTH_STATE_CLEANUP_CRON = "24 3 * * *"; -// v1.26.0 — Google Health / Fitbit-Pixel-Wear-OS poll-only sync. There is no -// Google Health webhook at launch (Pub/Sub deferred), so a single hourly cron -// drives the per-user `syncUserGoogleHealth` driver across every connection. -// Minute staggered off the WHOOP (:05/:20/:35/:50), Fitbit (:08), Nightscout -// (:11), Polar (:13), and Oura (:15) slots so the hourly ticks don't pile up on -// one boss poll. - -const GOOGLE_HEALTH_SYNC_QUEUE = "google-health-sync"; - -const GOOGLE_HEALTH_SYNC_CRON = "18 * * * *"; // every hour at :18 -// v1.26.0 — daily sweep for the Google Health OAuth state ledger. Slots at -// 03:26, next to the Fitbit sweep (03:24), inside the maintenance window. - -const GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE = - "google-health-oauth-state-cleanup"; - -const GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON = "26 3 * * *"; -// v1.17.0 — Nightscout CGM poll sync. Poll-only (no webhook): one hourly tick -// pulls the recent SGV window per configured instance. :11 staggers off the -// WHOOP (:05), Fitbit (:08), and Withings sync ticks so the hourly polls don't -// pile up on one boss poll. -// -// The four poll-queue NAMES live in `@/lib/jobs/integration-poll-queues` (see -// the import above) so an OAuth callback can enqueue a connect-time first sync -// without importing this registrar and its whole handler graph. Creation, -// scheduling, and handler binding stay here. -const NIGHTSCOUT_SYNC_CRON = "11 * * * *"; // every hour at :11 -// v1.17.0 — Polar + Oura OAuth poll sync. Poll-only (one hourly tick per -// provider re-walks every connected user). :13 / :15 stagger off the other -// hourly sync ticks (WHOOP :05, Fitbit :08, Nightscout :11) so the polls don't -// pile up on one boss poll. The queues MUST be registered in `allQueues` below -// or pg-boss never provisions them and the schedule silently no-ops (the -// v1.4.37 dead-queue class). -const POLAR_SYNC_CRON = "13 * * * *"; // every hour at :13 -const OURA_SYNC_CRON = "15 * * * *"; // every hour at :15 -// v1.28.x — Strava OAuth poll sync. Poll-only (webhook deferred): one hourly -// tick re-walks every connected user. :17 staggers off the Polar (:13) / Oura -// (:15) ticks so the polls don't pile up on one boss poll. The queue MUST be -// registered in `allQueues` below or pg-boss never provisions it and the -// schedule silently no-ops (the v1.4.37 dead-queue class). -const STRAVA_SYNC_CRON = "17 * * * *"; // every hour at :17 -// v1.30.x — daily sweep for the native OIDC handoff ledger. Handoff codes are -// single-use + 90s-lived and expiry is enforced at read, so this is pure -// hygiene: it reaps rows the app never came back to exchange. Slots at 03:28, -// next to the Google Health OAuth-state sweep (03:26), inside the maintenance -// window. The queue MUST be registered in `allQueues` below or pg-boss never -// provisions it and the schedule silently no-ops (the v1.4.37 dead-queue class). -const OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE = "oidc-native-handoff-cleanup"; -const OIDC_NATIVE_HANDOFF_CLEANUP_CRON = "28 3 * * *"; - -// pg-boss v12 requires explicit queue creation before scheduling. Every queue -// MUST be registered here or pg-boss never provisions it and both the webhook -// enqueue AND the cron schedule below silently no-op (the v1.4.37 dead-queue -// class). -const allQueues = [ - WITHINGS_SYNC_QUEUE, - WITHINGS_ACTIVITY_QUEUE, - WITHINGS_SLEEP_QUEUE, - WITHINGS_ECG_SYNC_QUEUE, - WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, - // v1.11.0 — WHOOP sync queues. Webhook-primary + cron-safety-net for - // recovery / sleep / workout; cycle is poll-only (no WHOOP webhook). - WHOOP_RECOVERY_SYNC_QUEUE, - WHOOP_SLEEP_SYNC_QUEUE, - WHOOP_WORKOUT_SYNC_QUEUE, - WHOOP_CYCLE_SYNC_QUEUE, - // v1.11.0 — self-converging boot backfill for newly connected WHOOP - // accounts. Discovery enqueues one full-history sync per un-backfilled - // connection; idempotent across reboots. - WHOOP_BACKFILL_QUEUE, - // v1.11.0 — daily sweep for the WHOOP OAuth state ledger. - WHOOP_OAUTH_STATE_CLEANUP_QUEUE, - // v1.12.0 — Fitbit / Google Health poll-only sync (no webhook at launch), - // self-converging boot backfill, and the daily OAuth-state ledger sweep. - FITBIT_SYNC_QUEUE, - FITBIT_BACKFILL_QUEUE, - // One-shot Fitbit sleep duplicate repair. Discovery enqueues one bounded - // sleep re-read per connection not yet repaired; the replace-by-window write - // path collapses each night's leftovers. Idempotent across reboots. - FITBIT_SLEEP_REPAIR_QUEUE, - FITBIT_OAUTH_STATE_CLEANUP_QUEUE, - // v1.26.0 — Google Health poll-only sync (no webhook at launch), - // self-converging boot backfill, and the daily OAuth-state ledger sweep. - GOOGLE_HEALTH_SYNC_QUEUE, - GOOGLE_HEALTH_BACKFILL_QUEUE, - // v1.28.x — one-shot sleep duplicate repair. Discovery enqueues one full - // sleep re-read per connection not yet repaired; the replace-by-window write - // path collapses each night's duplicate rows. Idempotent across reboots. - GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, - GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, - // v1.17.1 — one-shot sleep-timeline backfill for WHOOP + Withings. - // Discovery enqueues one job per connection whose sleep rows predate the - // stamp/shape fix; the pass deletes the affected SLEEP_DURATION rows and - // re-syncs. Idempotent across reboots. - SLEEP_TIMELINE_BACKFILL_QUEUE, - // v1.18.1 — one-shot backfill that links legacy free-text lab readings to - // a user-scoped Biomarker catalog entry (group by `lower(analyte)`). - LAB_BIOMARKER_BACKFILL_QUEUE, - // Shared durable drain for every full-history integration job. Source queues - // retain their per-connection singleton keys; this queue supplies the one - // cross-provider execution lane. - INTEGRATION_BACKFILL_ADMISSION_QUEUE, - // v1.17.0 — Nightscout CGM poll sync. Poll-only (no webhook, no OAuth, no - // backfill queue — the hourly window walks the recent SGV set). - NIGHTSCOUT_SYNC_QUEUE, - // v1.17.0 — Polar + Oura OAuth poll sync. - POLAR_SYNC_QUEUE, - OURA_SYNC_QUEUE, - // v1.44 — Oura history backfill, sliced through the shared lane. - OURA_BACKFILL_QUEUE, - // v1.28.x — Strava OAuth poll sync + self-converging boot backfill. - STRAVA_SYNC_QUEUE, - STRAVA_BACKFILL_QUEUE, - // v1.30.x — daily sweep for the native OIDC handoff ledger. - OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, - // v1.42.4 — the integration watchdog: an hourly cheap pass and a daily - // pass that re-reads the history behind standing silence flags. - INTEGRATION_WATCHDOG_HOURLY_QUEUE, - INTEGRATION_WATCHDOG_DAILY_QUEUE, -]; - -const schedules: ScheduleEntry[] = [ - [WITHINGS_SYNC_QUEUE, WITHINGS_SYNC_CRON], - [WITHINGS_ACTIVITY_QUEUE, WITHINGS_ACTIVITY_CRON], - [WITHINGS_SLEEP_QUEUE, WITHINGS_SLEEP_CRON], - // v1.32.23 — hourly ECG catch-net (:41) so watch-only accounts pull their - // strips without a webhook or a scale. - [WITHINGS_ECG_SYNC_QUEUE, WITHINGS_ECG_SYNC_CRON], - // The five OAuth-state / handoff prunes carry `cronIsTheRetry`: their - // handlers used to warn and carry on, they now fail the job, and a DELETE - // that could not run will not run any better a second later. The daily tick - // is the retry. The sync crons above keep the default retries — a provider - // outage is transient and worth retrying inside the hour. - [ - WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, - WITHINGS_OAUTH_STATE_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.11.0 — WHOOP poll-fallback crons. Recovery/sleep/workout catch - // dropped webhooks; cycle is the sole driver (no webhook). Staggered off - // the Withings crons so the hourly ticks don't pile up on one boss poll. - [WHOOP_RECOVERY_SYNC_QUEUE, WHOOP_RECOVERY_SYNC_CRON], - [WHOOP_SLEEP_SYNC_QUEUE, WHOOP_SLEEP_SYNC_CRON], - [WHOOP_WORKOUT_SYNC_QUEUE, WHOOP_WORKOUT_SYNC_CRON], - [WHOOP_CYCLE_SYNC_QUEUE, WHOOP_CYCLE_SYNC_CRON], - // v1.11.0 — daily 03:22 Europe/Berlin prune for expired WHOOP OAuth states. - [ - WHOOP_OAUTH_STATE_CLEANUP_QUEUE, - WHOOP_OAUTH_STATE_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.12.0 — hourly Fitbit poll (:08, staggered off WHOOP/Withings) + the - // daily 03:24 Europe/Berlin prune for expired Fitbit OAuth states. - [FITBIT_SYNC_QUEUE, FITBIT_SYNC_CRON], - [ - FITBIT_OAUTH_STATE_CLEANUP_QUEUE, - FITBIT_OAUTH_STATE_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.26.0 — hourly Google Health poll (:18, staggered off WHOOP/Fitbit/ - // Nightscout/Polar/Oura) + the daily 03:26 Europe/Berlin prune for expired - // Google Health OAuth states. - [GOOGLE_HEALTH_SYNC_QUEUE, GOOGLE_HEALTH_SYNC_CRON], - [ - GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, - GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.17.0 — hourly Nightscout CGM poll (:11, staggered off the other sync - // ticks). - [NIGHTSCOUT_SYNC_QUEUE, NIGHTSCOUT_SYNC_CRON], - // v1.17.0 — hourly Polar (:13) + Oura (:15) OAuth polls. - [POLAR_SYNC_QUEUE, POLAR_SYNC_CRON], - [OURA_SYNC_QUEUE, OURA_SYNC_CRON], - // v1.28.x — hourly Strava (:17) OAuth poll. - [STRAVA_SYNC_QUEUE, STRAVA_SYNC_CRON], - // v1.42.4 — integration watchdog, hourly at :47 and daily at 04:52. The - // next tick is the retry: a missed pass only delays a flag by an hour. - [ - INTEGRATION_WATCHDOG_HOURLY_QUEUE, - INTEGRATION_WATCHDOG_HOURLY_CRON, - cronIsTheRetry, - ], - [ - INTEGRATION_WATCHDOG_DAILY_QUEUE, - INTEGRATION_WATCHDOG_DAILY_CRON, - cronIsTheRetry, - ], - // v1.30.x — daily 03:28 Europe/Berlin prune for expired native OIDC handoffs. - [ - OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, - OIDC_NATIVE_HANDOFF_CLEANUP_CRON, - cronIsTheRetry, - ], -]; - -/** - * De-duplication policy per integration-sync queue. - * - * The recurring poll queues (Withings / WHOOP / Fitbit / Google Health / - * Nightscout / Polar / Oura / Strava sync, and every OAuth-state cleanup) are - * keyless cron ticks — no `singletonKey` on any send — so a policy would only - * constrain the empty key. They are deliberately absent. - * - * The one-shot backfills and repairs below all share the same shape: one job - * per connection, keyed per user (per provider where a provider can differ), - * enqueued by a boot-discovery predicate that only stops offering the job once - * it has COMPLETED. That last property is what made the bug bite — a worker - * restarting during a heavy account's multi-hour full-history pass appended - * another identical job per restart, sustaining the boot storm the stagger - * deferrals exist to damp. `exclusive` covers queued, active, and retry-backoff, - * so the restart case is closed; and because discovery re-offers the job while - * the work is still outstanding, a suppressed send can never lose work. - */ -const queuePolicies: QueuePolicyTable = { - [WITHINGS_ECG_SYNC_QUEUE]: { - policy: "short", - reason: - "Webhook retries share one identity while queued; once work starts, a later replay remains admissible as a rescue if the original exhausts retries.", - }, - [WHOOP_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection full-history backfill; long-running, and boot discovery re-enqueues until the connection is marked backfilled.", - }, - [FITBIT_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection full-history backfill; boot-discovery driven and self-converging.", - }, - [GOOGLE_HEALTH_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection full-history backfill; boot-discovery driven and self-converging.", - }, - [STRAVA_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection full-history backfill; boot-discovery driven and self-converging.", - }, - [OURA_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection full-history backfill; boot-discovery driven and self-converging.", - }, - [GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection one-shot sleep re-read; discovery drops the connection once it is marked repaired.", - }, - [FITBIT_SLEEP_REPAIR_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection one-shot sleep re-read; discovery drops the connection once it is marked repaired.", - }, - [SLEEP_TIMELINE_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection one-shot timeline backfill, keyed per provider+user; discovery drops the connection once its rows carry the new shape.", - }, - [LAB_BIOMARKER_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user one-shot catalog link-up; discovery drops the user once no unlinked lab readings remain.", - }, - [INTEGRATION_BACKFILL_ADMISSION_QUEUE]: { - policy: "exclusive", - reason: - "Shared full-history drain; keyed per source job so boot rediscovery cannot duplicate queued, active, or retrying work.", - }, -}; - -/** - * Register every integration-sync queue: create, schedule, and bind handlers. - * Returns the queue names created (for the boot-level aggregate assertion). - */ -export async function registerIntegrationSyncQueues( - boss: PgBoss, -): Promise { - await createAndSchedule(boss, allQueues, schedules, queuePolicies); - // pg-boss scopes groupConcurrency to one queue name, so every provider - // reaches this shared durable queue before doing full-history work. The - // common group is enforced in PostgreSQL across worker processes/nodes; - // the heavy lane keeps an import off the heap while a backup, a restore - // or an upload is on it. - - await createAndWork( - boss, - INTEGRATION_BACKFILL_ADMISSION_QUEUE, - { - localConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, - groupConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, - }, - heavyLane( - INTEGRATION_BACKFILL_ADMISSION_QUEUE, - drainIntegrationBackfillAdmission, - ), - ); - - await createAndWork( - boss, - WITHINGS_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWithingsFallbackSync, - ); - await createAndWork( - boss, - WITHINGS_ACTIVITY_QUEUE, - { localConcurrency: 1 }, - handleWithingsActivitySync, - ); - await createAndWork( - boss, - WITHINGS_SLEEP_QUEUE, - { localConcurrency: 1 }, - handleWithingsSleepSync, - ); - await createAndWork( - boss, - WITHINGS_ECG_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWithingsEcgSync, - ); - await createAndWork( - boss, - WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleWithingsOAuthStateCleanup, - ); - // v1.11.0 — WHOOP per-resource sync handlers. Webhook-driven per-user + - // cron full-iteration. Serial concurrency so a backfill-heavy tick never - // crowds the request pool and stays inside WHOOP's 100 req/min app cap. - await createAndWork( - boss, - WHOOP_RECOVERY_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWhoopRecoverySync, - ); - await createAndWork( - boss, - WHOOP_SLEEP_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWhoopSleepSync, - ); - await createAndWork( - boss, - WHOOP_WORKOUT_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWhoopWorkoutSync, - ); - await createAndWork( - boss, - WHOOP_CYCLE_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWhoopCycleSync, - ); - // v1.11.0 — self-converging WHOOP backfill. This source queue retains the - // per-connection singleton and forwards admitted work to the shared drain. - await createAndWork( - boss, - WHOOP_BACKFILL_QUEUE, - { localConcurrency: WHOOP_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "whoop-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - await createAndWork( - boss, - WHOOP_OAUTH_STATE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleWhoopOAuthStateCleanup, - ); - // v1.30.x — daily sweep for the native OIDC handoff ledger. - await createAndWork( - boss, - OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleOidcNativeHandoffCleanup, - ); - // v1.12.0 — Fitbit poll-sync (cron full-iteration; no webhook). Serial - // concurrency so a backfill-heavy tick never crowds the request pool. - await createAndWork( - boss, - FITBIT_SYNC_QUEUE, - { localConcurrency: 1 }, - handleFitbitSync, - ); - // v1.12.0 — self-converging Fitbit backfill. This source queue retains the - // per-connection singleton and forwards admitted work to the shared drain. - await createAndWork( - boss, - FITBIT_BACKFILL_QUEUE, - { localConcurrency: FITBIT_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "fitbit-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // One-shot Fitbit sleep duplicate repair. This source queue retains the - // per-connection singleton and forwards the watermark-safe bounded sleep - // re-read to the shared drain. - await createAndWork( - boss, - FITBIT_SLEEP_REPAIR_QUEUE, - { localConcurrency: FITBIT_SLEEP_REPAIR_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "fitbit-sleep-repair", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - await createAndWork( - boss, - FITBIT_OAUTH_STATE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleFitbitOAuthStateCleanup, - ); - // v1.26.0 — Google Health poll-sync (cron full-iteration; no webhook). Serial - // concurrency so a backfill-heavy tick never crowds the request pool. - await createAndWork( - boss, - GOOGLE_HEALTH_SYNC_QUEUE, - { localConcurrency: 1 }, - handleGoogleHealthSync, - ); - // v1.26.0 — self-converging Google Health backfill. This source queue - // retains the per-connection singleton and forwards admitted work to the - // shared drain. - await createAndWork( - boss, - GOOGLE_HEALTH_BACKFILL_QUEUE, - { localConcurrency: GOOGLE_HEALTH_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "google-health-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // v1.28.x — one-shot Google Health sleep duplicate repair. This source - // queue retains the per-connection singleton and forwards the watermark-safe - // full sleep-history re-read to the shared drain. - await createAndWork( - boss, - GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, - { localConcurrency: GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "google-health-sleep-repair", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - await createAndWork( - boss, - GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleGoogleHealthOAuthStateCleanup, - ); - // v1.17.1 — one-shot sleep-timeline backfill. This source queue retains the - // per-(user, provider) singleton and forwards admitted work to the shared - // drain. - await createAndWork( - boss, - SLEEP_TIMELINE_BACKFILL_QUEUE, - { localConcurrency: SLEEP_TIMELINE_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "sleep-timeline-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // v1.18.1 — one-shot lab-biomarker backfill. This source queue retains the - // per-user singleton and forwards admitted work to the shared drain. - await createAndWork( - boss, - LAB_BIOMARKER_BACKFILL_QUEUE, - { localConcurrency: LAB_BIOMARKER_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "lab-biomarker-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // v1.17.0 — Nightscout CGM poll-cohort sync. The hourly cron tick (no - // `userId`) walks every configured instance; one user's unreachable host is - // warned, not fatal. - await createAndWork( - boss, - NIGHTSCOUT_SYNC_QUEUE, - { localConcurrency: 1 }, - handleNightscoutSync, - ); - // v1.17.0 — Polar + Oura OAuth poll-cohort sync. The hourly cron tick - // (no `userId`) re-walks every connected user; one user's revoked grant is - // warned, not fatal. - await createAndWork( - boss, - POLAR_SYNC_QUEUE, - { localConcurrency: 1 }, - handlePolarSync, - ); - await createAndWork( - boss, - OURA_SYNC_QUEUE, - { localConcurrency: 1 }, - handleOuraSync, - ); - // v1.28.x — Strava OAuth poll-cohort sync (poll-only; webhook deferred). The - // hourly cron tick (no `userId`) re-walks every connected user; one user's - // revoked grant is warned, not fatal. - await createAndWork( - boss, - STRAVA_SYNC_QUEUE, - { localConcurrency: 1 }, - handleStravaSync, - ); - // v1.42.4 — integration watchdog. Single-flight per process; both queues - // also share one advisory lock, so the daily and hourly passes never run - // side by side on any replica. - await createAndWork( - boss, - INTEGRATION_WATCHDOG_HOURLY_QUEUE, - { localConcurrency: 1 }, - handleIntegrationWatchdogHourly, - ); - await createAndWork( - boss, - INTEGRATION_WATCHDOG_DAILY_QUEUE, - { localConcurrency: 1 }, - handleIntegrationWatchdogDaily, - ); - // v1.28.x — self-converging Strava backfill. This source queue retains the - // per-connection singleton and forwards admitted work to the shared drain. - await createAndWork( - boss, - STRAVA_BACKFILL_QUEUE, - { localConcurrency: STRAVA_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "strava-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // v1.44 — Oura history backfill. The source queue keeps the per-account - // singleton and forwards admitted work to the shared drain, where it runs - // in resumable slices. - await createAndWork( - boss, - OURA_BACKFILL_QUEUE, - { localConcurrency: OURA_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "oura-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - return allQueues; -} - -/** - * Fire-and-forget boot discovery for the self-converging integration backfills. - * Each pass is idempotent across reboots and never fails worker boot on a miss - * (errors come back through the helper's result value). - */ -export async function enqueueIntegrationSyncBootDiscovery(): Promise { - // v1.11.0 — WHOOP backfill. Finds every WHOOP connection not yet backfilled - // and enqueues one full-history sync per account. A completed backfill stamps - // `backfillCompletedAt`, dropping the connection from the discovery set. - try { - const { enqueued, skipped, error } = await enqueueBootTimeWhoopBackfill( - bootStaggerSecondsFor("whoop-backfill"), - ); - if (error) { - workerLog("error", `[whoop-backfill] boot discovery failed: ${error}`); - } else { - workerLog( - "info", - `[whoop-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[whoop-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.12.0 — Fitbit backfill. Finds every Fitbit connection not yet - // backfilled and enqueues one full-history sync per account. - try { - const { enqueued, skipped, error } = await enqueueBootTimeFitbitBackfill( - bootStaggerSecondsFor("fitbit-backfill"), - ); - if (error) { - workerLog("error", `[fitbit-backfill] boot discovery failed: ${error}`); - } else { - workerLog( - "info", - `[fitbit-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[fitbit-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.26.0 — Google Health backfill. Finds every Google Health connection not - // yet backfilled and enqueues one full-history sync per account. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeGoogleHealthBackfill( - bootStaggerSecondsFor("google-health-backfill"), - ); - if (error) { - workerLog( - "error", - `[google-health-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[google-health-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[google-health-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.28.x — one-shot Google Health sleep duplicate repair. Finds every - // connection not yet repaired and enqueues one full sleep re-read per - // account; a completed pass stamps `sleepRepairedAt`. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeGoogleHealthSleepRepair( - bootStaggerSecondsFor("google-health-sleep-repair"), - ); - if (error) { - workerLog( - "error", - `[google-health-sleep-repair] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[google-health-sleep-repair] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[google-health-sleep-repair] boot discovery threw an unexpected error", - err, - ); - } - - // v1.17.1 — one-shot sleep-timeline backfill. Finds every WHOOP + Withings - // connection whose sleep rows predate the stamp/shape fix and enqueues one - // job per (user, provider). A completed pass stamps `sleepTimelineBackfillAt`. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeSleepTimelineBackfill( - bootStaggerSecondsFor("sleep-timeline-backfill"), - ); - if (error) { - workerLog( - "error", - `[sleep-timeline-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[sleep-timeline-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[sleep-timeline-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.28.x — Strava backfill. Finds every Strava connection not yet - // backfilled and enqueues one full-history sync per account. - try { - const { enqueued, skipped, error } = await enqueueBootTimeStravaBackfill( - bootStaggerSecondsFor("strava-backfill"), - ); - if (error) { - workerLog("error", `[strava-backfill] boot discovery failed: ${error}`); - } else { - workerLog( - "info", - `[strava-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[strava-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.44 — Oura history backfill. Finds every Oura connection not yet - // backfilled (and every account still holding the retired - // equivalent-distance rows) and enqueues one sliced backfill per account. - try { - const { enqueued, skipped, error } = await enqueueBootTimeOuraBackfill( - bootStaggerSecondsFor("oura-backfill"), - ); - if (error) { - workerLog("error", `[oura-backfill] boot discovery failed: ${error}`); - } else { - workerLog( - "info", - `[oura-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[oura-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.18.1 — one-shot lab-biomarker backfill. Finds every user holding an - // un-linked live lab reading and enqueues one job each. A completed pass - // links every reading, dropping the user from the discovery set. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeLabBiomarkerBackfill( - bootStaggerSecondsFor("lab-biomarker-backfill"), - ); - if (error) { - workerLog( - "error", - `[lab-biomarker-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[lab-biomarker-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[lab-biomarker-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // One-shot Fitbit sleep duplicate repair. Finds every connection not yet - // repaired and enqueues one bounded sleep re-read per account; a completed - // pass stamps `sleepRepairedAt`. - try { - const { enqueued, skipped, error } = await enqueueBootTimeFitbitSleepRepair( - bootStaggerSecondsFor("fitbit-sleep-repair"), - ); - if (error) { - workerLog( - "error", - `[fitbit-sleep-repair] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[fitbit-sleep-repair] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[fitbit-sleep-repair] boot discovery threw an unexpected error", - err, - ); - } -} diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts deleted file mode 100644 index b0e02c042..000000000 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ /dev/null @@ -1,2197 +0,0 @@ -/** - * Maintenance / ops / cleanup queue registrar. - * - * Owns the housekeeping tier: data backup (in-DB weekly + off-host nightly), - * the restore drill, every ledger cleanup (rate-limit, idempotency, audit-log, - * mood-reminder, push-attempt, measurement-tombstone), medication inventory - * expiry, intake auto-skip, the duplicate dose-slot dedup (boot + daily-cron), - * host-metric sampling, the rec-feedback aggregator, the geo backfill, the TLS - * pin monitor, PR detection, and the Apple Health export ingest. - * - * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its - * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The - * `measurement-tombstone-cleanup`, `withings-oauth-state-cleanup`, - * `intake-slot-dedup-queue`, `tls-pin-monitor-queue`, and `geo-backfill` - * guards read THIS module. - */ -import { trackBackgroundTask } from "@/lib/logging/background-tasks"; -import { PgBoss } from "pg-boss"; -import { GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON } from "@/lib/jobs/geo-backfill"; -import { - GEOLITE2_FETCH_QUEUE, - GEOLITE2_FETCH_CRON, - enqueueGeolite2FetchBootDiscovery, - type Geolite2FetchPayload, -} from "@/lib/jobs/geolite2-fetch"; -import { - TLS_PIN_MONITOR_QUEUE, - TLS_PIN_MONITOR_CRON, -} from "@/lib/jobs/tls-pin-monitor"; -import { - PR_DETECTION_QUEUE, - PR_DETECTION_CONCURRENCY, - PR_DETECTION_FALLBACK_CRON, - PR_DETECTION_EXPIRE_SECONDS, - type PrDetectionPayload, -} from "@/lib/jobs/pr-detection"; -import { jobBudget } from "@/lib/jobs/job-budget"; -import { lockedPass, WHOLE_PASS } from "@/lib/jobs/long-pass"; -import { heavyLane } from "@/lib/jobs/heavy-lane"; -import { - OFFHOST_BACKUP_QUEUE, - OFFHOST_BACKUP_SEND_OPTIONS, -} from "@/lib/jobs/offhost-backup"; -import { - MEDICATION_INVENTORY_EXPIRE_QUEUE, - MEDICATION_INVENTORY_EXPIRE_CRON, - type MedicationInventoryExpirePayload, -} from "@/lib/jobs/medication-inventory-expire"; -import { - INTAKE_AUTO_SKIP_QUEUE, - INTAKE_AUTO_SKIP_CRON, - type IntakeAutoSkipPayload, -} from "@/lib/jobs/intake-auto-skip"; -import { - INTAKE_AUTO_TAKE_QUEUE, - INTAKE_AUTO_TAKE_CRON, - type IntakeAutoTakePayload, -} from "@/lib/jobs/intake-auto-take"; -import { - APPLE_HEALTH_IMPORT_V2_QUEUE, - APPLE_HEALTH_IMPORT_LEGACY_QUEUE, - APPLE_HEALTH_IMPORT_CONCURRENCY, - IMPORT_JOB_RECONCILE_QUEUE, - IMPORT_JOB_RECONCILE_CRON, - handleAppleHealthImport, - handleImportJobReconcileTick, - migrateLegacyAppleHealthImport, - type AppleHealthImportPayload, -} from "@/lib/jobs/apple-health-import-worker"; -import { - COMPACTION_TOMBSTONE_PURGE_QUEUE, - enqueueBootTimeCompactionTombstonePurge, - handleCompactionTombstonePurge, - type CompactionTombstonePurgePayload, -} from "@/lib/jobs/compaction-tombstone-purge"; -import { - MEASUREMENT_FOLD_REPAIR_QUEUE, - enqueueBootTimeMeasurementFoldRepair, - handleMeasurementFoldRepair, - type MeasurementFoldRepairPayload, -} from "@/lib/jobs/measurement-fold-repair"; -import { - MEASUREMENT_MAINTENANCE_CRON, - MEASUREMENT_MAINTENANCE_QUEUE, - MEASUREMENT_MAINTENANCE_SEND_OPTIONS, - handleMeasurementMaintenance, - type MeasurementMaintenancePayload, -} from "@/lib/jobs/measurement-maintenance"; -import { - PULSE_DAY_MEAN_QUEUE, - handlePulseDayMean, - type PulseDayMeanPayload, -} from "@/lib/jobs/pulse-day-mean"; -import { - HEALTH_CONNECT_IMPORT_QUEUE, - HEALTH_CONNECT_IMPORT_CONCURRENCY, - handleHealthConnectImport, - type HealthConnectImportPayload, -} from "@/lib/jobs/health-connect-import-worker"; -import { - MEDICATION_INTAKE_IMPORT_QUEUE, - MEDICATION_INTAKE_IMPORT_CONCURRENCY, - handleMedicationIntakeImport, - type MedicationIntakeImportQueuePayload, -} from "@/lib/jobs/medication-intake-import"; -import { - INTAKE_SLOT_DEDUP_QUEUE, - INTAKE_SLOT_DEDUP_CONCURRENCY, - dedupeUserIntakeSlots, - enqueueBootTimeIntakeSlotDedup, - type IntakeSlotDedupPayload, -} from "@/lib/medications/intake-slot-dedup"; -import { - handleOffhostPurge, - OFFHOST_PURGE_CRON, - OFFHOST_PURGE_QUEUE, -} from "@/lib/jobs/offhost-purge"; -import { - handleRestoreDrill, - RESTORE_DRILL_CRON, - RESTORE_DRILL_QUEUE, - RESTORE_DRILL_SEND_OPTIONS, -} from "@/lib/jobs/restore-drill"; -import { - BACKUP_RESTORE_QUEUE, - enqueueBackupRestoreSweepAtBoot, - handleBackupRestore, - type BackupRestorePayload, -} from "@/lib/jobs/backup-restore"; -import { - NOTE_ENCRYPTION_BACKFILL_QUEUE, - NOTE_ENCRYPTION_BACKFILL_CONCURRENCY, - runNoteEncryptionBackfillForUser, - enqueueBootTimeNoteEncryptionBackfill, - type NoteEncryptionBackfillPayload, -} from "@/lib/jobs/note-encryption-backfill"; -import { - MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, - MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY, - runMedNotesEncryptionBackfillForUser, - enqueueBootTimeMedNotesEncryptionBackfill, - type MedNotesEncryptionBackfillPayload, -} from "@/lib/jobs/med-notes-encryption-backfill"; -import { - FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, - FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY, - runFreeTextEncryptionBackfillForUser, - enqueueBootTimeFreeTextEncryptionBackfill, - type FreeTextEncryptionBackfillPayload, -} from "@/lib/jobs/free-text-encryption-backfill"; -import { - CONTENT_INDEX_BACKFILL_QUEUE, - CONTENT_INDEX_BACKFILL_CONCURRENCY, - runContentIndexBackfillForUser, - type ContentIndexBackfillPayload, -} from "@/lib/jobs/document-content-index-backfill"; -import { - DOCUMENT_INDEX_QUEUE, - DOCUMENT_INDEX_CONCURRENCY, - runDocumentIndex, - type DocumentIndexPayload, -} from "@/lib/jobs/document-index"; -import { - DOCUMENT_AI_RUN_QUEUE, - DOCUMENT_AI_RUN_CONCURRENCY, - handleDocumentAiRunJobs, - type DocumentAiRunPayload, -} from "@/lib/jobs/document-ai-run"; -import { - DOCUMENT_AI_RUN_REAPER_QUEUE, - DOCUMENT_AI_RUN_REAPER_CRON, - handleDocumentAiRunReaper, - type DocumentAiRunReaperPayload, -} from "@/lib/jobs/document-ai-run-reaper"; -import { - DOCUMENT_THUMBNAIL_QUEUE, - DOCUMENT_THUMBNAIL_CONCURRENCY, - runDocumentThumbnail, - type DocumentThumbnailPayload, -} from "@/lib/jobs/document-thumbnail"; -import { - DOCUMENT_SUMMARY_QUEUE, - DOCUMENT_SUMMARY_CONCURRENCY, - runDocumentSummaryJob, - type DocumentSummaryPayload, -} from "@/lib/jobs/document-summary"; -import { - DOCUMENT_SUMMARY_CATCHUP_QUEUE, - DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY, - runSummaryCatchUpForUser, - type SummaryCatchUpPayload, -} from "@/lib/jobs/document-summary-catchup"; -import { - DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, - DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY, - runThumbnailBackfillForUser, - enqueueBootTimeThumbnailBackfill, - type ThumbnailBackfillPayload, -} from "@/lib/jobs/document-thumbnail-backfill"; -import { - ENCRYPTION_KEY_ROTATE_QUEUE, - ENCRYPTION_KEY_ROTATE_CONCURRENCY, - handleEncryptionKeyRotate, - type EncryptionKeyRotatePayload, -} from "@/lib/jobs/encryption-key-rotate"; -import { - ENVIRONMENT_FETCH_QUEUE, - ENVIRONMENT_FETCH_CRON, - ENVIRONMENT_FETCH_CONCURRENCY, - handleEnvironmentFetch, - type EnvironmentFetchPayload, -} from "@/lib/jobs/environment-fetch"; -import { - DAWARICH_SYNC_CRON, - DAWARICH_SYNC_QUEUE, - handleDawarichSync, - type DawarichSyncPayload, -} from "@/lib/jobs/dawarich-sync"; -import { - ENVIRONMENT_AQ_HISTORY_CRON, - ENVIRONMENT_AQ_HISTORY_QUEUE, - enqueueBootTimeAirQualityHistory, - handleEnvironmentAqHistory, - type EnvironmentAqHistoryPayload, -} from "@/lib/jobs/environment-air-quality-history"; -import { recordError } from "@/lib/jobs/worker-status"; -import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; -import { - DATA_BACKUP_QUEUE, - DATA_BACKUP_SEND_OPTIONS, -} from "@/lib/jobs/data-backup-policy"; -import { workerLog, BOOT_BACKFILL_STAGGER_SECONDS } from "./shared"; -import { - createAndSchedule, - createAndWork, - cronIsTheRetry, - type QueuePolicyTable, - type ScheduleEntry, -} from "./registrar-shared"; -import { - DataBackupPayload, - OffhostBackupPayload, - handleOffhostBackup, - handleDataBackup, -} from "./backup-handlers"; -import { - RateLimitCleanupPayload, - StepUpElevationCleanupPayload, - IdempotencyCleanupPayload, - AuditLogCleanupPayload, - MoodReminderCleanupPayload, - handleMoodReminderCleanup, - PushAttemptCleanupPayload, - handlePushAttemptCleanup, - ArrivalReactionCleanupPayload, - handleArrivalReactionCleanup, - WorkoutInsightClaimCleanupPayload, - handleWorkoutInsightClaimCleanup, - MeasurementTombstoneCleanupPayload, - handleMeasurementTombstoneCleanup, - handleRateLimitCleanup, - handleIdempotencyCleanup, - handleAuditLogCleanup, - handleStepUpElevationCleanup, - CoachMessageCleanupPayload, - handleCoachMessageCleanup, - McpTokenCleanupPayload, - handleMcpTokenCleanup, -} from "./cleanup-handlers"; -import { - HostMetricSamplePayload, - FeedbackAggregatorPayload, - GeoBackfillPayload, - TlsPinMonitorPayload, - handleHostMetricSample, - handleFeedbackAggregator, - handleGeoBackfill, - handleGeolite2Fetch, - handleTlsPinMonitor, - handlePrDetection, -} from "./ops-handlers"; -import { - handleMedicationInventoryExpire, - handleIntakeAutoSkip, - handleIntakeAutoTake, -} from "./medication-maintenance"; -import { - DOCUMENT_PURGE_QUEUE, - DOCUMENT_PURGE_CRON, - handleDocumentPurge, - type DocumentPurgePayload, -} from "@/lib/jobs/document-purge"; -import { - SOFT_DELETE_PURGE_QUEUE, - SOFT_DELETE_PURGE_CRON, - handleSoftDeletePurge, - type SoftDeletePurgePayload, -} from "@/lib/jobs/soft-delete-purge"; -import { - AUTH_LEDGER_CLEANUP_QUEUE, - AUTH_LEDGER_CLEANUP_CRON, - handleAuthLedgerCleanup, - type AuthLedgerCleanupPayload, -} from "@/lib/jobs/auth-ledger-cleanup"; -import { - DOCUMENT_SUMMARY_REAPER_QUEUE, - DOCUMENT_SUMMARY_REAPER_CRON, - handleDocumentSummaryReaper, - type DocumentSummaryReaperPayload, -} from "@/lib/jobs/document-summary-reaper"; -import { - CYCLE_PREDICTION_REFRESH_QUEUE, - CYCLE_PREDICTION_REFRESH_CRON, - handleCyclePredictionRefresh, - type CyclePredictionRefreshPayload, -} from "@/lib/jobs/cycle-prediction-refresh"; -import { - MOOD_PROGNOSIS_REFRESH_QUEUE, - MOOD_PROGNOSIS_REFRESH_CRON, - handleMoodPrognosisRefresh, - type MoodPrognosisRefreshPayload, -} from "@/lib/jobs/mood-prognosis-refresh"; -import { - ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, - ACHIEVEMENT_UNLOCK_SWEEP_CRON, - handleAchievementUnlockSweep, - type AchievementUnlockSweepPayload, -} from "@/lib/jobs/achievement-unlock-sweep"; - -const DATA_BACKUP_CRON = "0 3 * * 0"; // weekly Sunday at 03:00 - -const RATE_LIMIT_CLEANUP_QUEUE = "rate-limit-cleanup"; - -const RATE_LIMIT_CLEANUP_CRON = "*/5 * * * *"; // every 5 minutes - -const IDEMPOTENCY_CLEANUP_QUEUE = "idempotency-cleanup"; - -const IDEMPOTENCY_CLEANUP_CRON = "0 3 * * *"; // daily at 03:00 (Europe/Berlin) - -const STEP_UP_ELEVATION_CLEANUP_QUEUE = "step-up-elevation-cleanup"; - -// Every 15 minutes. An elevation lives five, so nothing lingers long; the -// cadence only has to be short enough that the table stays small. -const STEP_UP_ELEVATION_CLEANUP_CRON = "*/15 * * * *"; - -const AUDIT_LOG_CLEANUP_QUEUE = "audit-log-cleanup"; - -const AUDIT_LOG_CLEANUP_CRON = "15 3 * * *"; // daily at 03:15 (Europe/Berlin) -// v1.15.19 — daily duplicate dose-slot dedup discovery tick. The boot-time -// pass only ran on worker restart, so a cross-source duplicate slot created -// between deploys (a pending REMINDER row plus a standalone API/WEB row on -// the same instant) survived until the next reboot. The cron payload omits -// `userId`; the handler treats that as the discovery tick and fans out one -// per-user job (singletonKey-coalesced) exactly like the boot pass. Slots at -// 03:28 between the mood-reminder cleanup (03:25) and the inventory expire -// (03:30), inside the maintenance window. - -const INTAKE_SLOT_DEDUP_CRON = "28 3 * * *"; - -// 02:30 Europe/Berlin — runs after audit-log/idempotency cleanups so old -// rows are gone before they're snapshotted, but before the existing -// in-DB DATA_BACKUP at 03:00 (Sundays only) so the off-host copy is -// always at-or-ahead of the local one. - -const OFFHOST_BACKUP_CRON = "30 2 * * *"; - -const HOST_METRIC_QUEUE = "host-metric-sample"; -// v1.42 — every five minutes. A per-minute sample filled pg-boss's job -// table with one row a minute for a chart nobody reads at that resolution. -// The admin chart's polling interval (`host-metrics-chart.tsx`) has to match. - -const HOST_METRIC_CRON = "*/5 * * * *"; -// v1.4.16 — daily rec-feedback aggregator. 04:00 Europe/Berlin -// runs the slot AFTER all the cleanup jobs (rate-limit, idempotency, -// audit-log) so the previous-day's noise is gone before we aggregate. - -const FEEDBACK_AGGREGATOR_QUEUE = "feedback-aggregator"; - -const FEEDBACK_AGGREGATOR_CRON = "0 4 * * *"; -// v0.5.4 ios-coord — daily mood-reminder dispatch-ledger retention sweep. -// Rows older than 90 days are behavioural footprints of mood-log gaps; we -// keep them long enough to debug a duplicate-push report (~one billing -// cycle) but no longer. Slots between the audit-log cleanup (03:15) and the -// drain (03:45). - -const MOOD_REMINDER_CLEANUP_QUEUE = "mood-reminder-cleanup"; - -const MOOD_REMINDER_CLEANUP_CRON = "25 3 * * *"; - -const PUSH_ATTEMPT_CLEANUP_QUEUE = "push-attempt-cleanup"; - -const PUSH_ATTEMPT_CLEANUP_CRON = "35 3 * * *"; - -// v1.31.0 — daily prune for the data-arrival spine's reaction markers. -// Slots at 04:10, after the 03:xx retention block and the 04:00 MCP-token -// prune, so the maintenance window stays in a readable order. -const ARRIVAL_REACTION_CLEANUP_QUEUE = "arrival-reaction-cleanup"; - -const ARRIVAL_REACTION_CLEANUP_CRON = "10 4 * * *"; - -const MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE = "measurement-tombstone-cleanup"; - -const MEASUREMENT_TOMBSTONE_CLEANUP_CRON = "40 3 * * *"; -// v1.18.7 — daily prune for the encrypted Coach conversation history. -// coach_messages is append-only and grows forever; rows older than the -// configurable retention window (default 365 days) are hard-deleted. -// Slots at 03:50 Europe/Berlin after the measurement-tombstone cleanup -// (03:40) and the cumulative drain (03:45), staying inside the existing -// 03:xx maintenance window. - -const COACH_MESSAGE_CLEANUP_QUEUE = "coach-message-cleanup"; - -const COACH_MESSAGE_CLEANUP_CRON = "50 3 * * *"; -// v1.40 — daily prune of the workout-insight claim ledger. Claims outlive a -// deleted workout so the delete cannot free a daily slot; rows older than a -// week are never read by the cap again. - -const WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE = "workout-insight-claim-cleanup"; - -const WORKOUT_INSIGHT_CLAIM_CLEANUP_CRON = "52 3 * * *"; -// v1.23 — converging backfill that migrates the free-text health-note columns -// (mood + measurement) from plaintext to AES-256-GCM at rest. Boot discovery -// enqueues one per-user job; a daily 03:55 Europe/Berlin discovery tick (empty -// payload) re-fans for any rows that landed between worker reboots, so the -// migration converges without a restart. Idempotent + fail-closed per row. - -const NOTE_ENCRYPTION_BACKFILL_CRON = "55 3 * * *"; -// v1.24 — daily prune of expired/revoked MCP connector access tokens (every -// code exchange + hourly refresh mints a 60-minute row) and long-revoked OAuth -// connection anchors. Slots at 04:00 Europe/Berlin after the note-encryption -// backfill discovery (03:55) so the two don't pile up on the same boss poll. - -const MCP_TOKEN_CLEANUP_QUEUE = "mcp-token-cleanup"; - -const MCP_TOKEN_CLEANUP_CRON = "0 4 * * *"; -// v1.25 — converging backfill that migrates the three medication free-text -// note columns (side-effect note, dose-change note, inventory-item note) from -// plaintext to AES-256-GCM at rest. Boot discovery enqueues one per-user job; -// a daily 04:05 Europe/Berlin discovery tick (empty payload) re-fans for any -// rows that landed between worker reboots. Idempotent + fail-closed per row. -// Slots after the MCP-token prune (04:00) so the two don't pile up on the same -// boss poll. - -const MED_NOTES_ENCRYPTION_BACKFILL_CRON = "5 4 * * *"; -// v1.39.3 — converging backfill that moves the Coach conversation title and the -// custom-metric reading note to AES-256-GCM at rest. Boot discovery (staggered -// past the startup storm) enqueues one per-user job; a daily 04:07 -// Europe/Berlin discovery tick re-fans for rows written by a previous-release -// process after the upgrade. Two minutes after the medication-note tick so the -// two do not share a poll. v1.39.4 adds the practitioner phone + address and -// the workout GPS track to the same pass. -const FREE_TEXT_ENCRYPTION_BACKFILL_CRON = "7 4 * * *"; - -const allQueues = [ - // v1.25 — nightly environmental-context fetch. A daily discovery tick - // fans out one per-user job per opted-in account with a home location; the - // queue also serves the on-demand backfill from the Environment settings - // surface. Without this entry pg-boss never provisions it and both the cron - // and the backfill button silently no-op. - ENVIRONMENT_FETCH_QUEUE, - // v1.42 — the air-quality history backfill: per account, from boot and - // nightly discovery, the home being set and air quality switched on. - ENVIRONMENT_AQ_HISTORY_QUEUE, - // v1.43 — the Dawarich movement context: nightly discovery, per-account - // runs, and the connect / settings backfills. - DAWARICH_SYNC_QUEUE, - DATA_BACKUP_QUEUE, - RATE_LIMIT_CLEANUP_QUEUE, - IDEMPOTENCY_CLEANUP_QUEUE, - AUDIT_LOG_CLEANUP_QUEUE, - STEP_UP_ELEVATION_CLEANUP_QUEUE, - OFFHOST_BACKUP_QUEUE, - RESTORE_DRILL_QUEUE, - // v1.39.3 — deleting an account or wiping its data takes its off-host - // copies out of the bucket. Without this entry the requests would be - // written and never worked. - OFFHOST_PURGE_QUEUE, - // v1.39.1 — an admin's restore of a stored backup, run off the request, and - // the boot sweep that re-queues one a stopped worker left running. Without - // this entry every restore request would queue a job nobody works. - BACKUP_RESTORE_QUEUE, - HOST_METRIC_QUEUE, - FEEDBACK_AGGREGATOR_QUEUE, - GEO_BACKFILL_QUEUE, - // v1.37.7 (issue #659) — runtime GeoLite2 fetch. A self-hoster who sets - // MAXMIND_LICENSE_KEY on the published image gets the offline geo tier - // without a rebuild or a manual mount: boot discovery fetches when no - // database is present, a monthly cron keeps a keyed host current. Without - // this entry pg-boss never provisions the queue and both silently no-op. - GEOLITE2_FETCH_QUEUE, - // v1.12.2 ios-coord — TLS leaf SPKI-change monitor. The iOS client - // pins the served leaf certificate; this queue probes it every 6 h and - // alarms when the served SPKI leaves the operator's known-good set. - TLS_PIN_MONITOR_QUEUE, - PR_DETECTION_QUEUE, - MEDICATION_INVENTORY_EXPIRE_QUEUE, - // v1.4.46 — hourly auto-skip pass for stale unmarked intakes. Without - // this entry the schedule silently no-ops and pending rows older than - // 24 h pile up unflipped. - INTAKE_AUTO_SKIP_QUEUE, - // v1.42.2 — records the doses of a medication with auto-mark-taken on. - // Without this entry the schedule silently no-ops and those doses stay - // open. - INTAKE_AUTO_TAKE_QUEUE, - APPLE_HEALTH_IMPORT_V2_QUEUE, - APPLE_HEALTH_IMPORT_LEGACY_QUEUE, - // v1.32.1 (issue #588) — periodic sweep for ImportJob rows orphaned by a - // worker crash/restart mid-run. Without this entry the 15-minute cron - // below silently no-ops and a stuck "unpacking"/"parsing"/"upserting" row - // that survives past the next worker boot is never revisited. - IMPORT_JOB_RECONCILE_QUEUE, - MEDICATION_INTAKE_IMPORT_QUEUE, - // v1.42 (#972) — Android Health Connect export import, the Apple Health - // import's sibling. Without this entry every upload would queue a job - // nobody works. - HEALTH_CONNECT_IMPORT_QUEUE, - // v1.42 — backlog purge of compaction tombstones, queued once at boot. - // Without this entry the boot enqueue silently no-ops. - COMPACTION_TOMBSTONE_PURGE_QUEUE, - // v1.42 — one-time repair of the means older releases folded from part of - // their window; the purge above waits for it. Without this entry the boot - // enqueue silently no-ops and the purge never runs. - MEASUREMENT_FOLD_REPAIR_QUEUE, - // v1.42 — VACUUM / REINDEX of `measurements`: the admin route enqueues the - // operator's run, and a nightly check vacuums when dead rows pile up. - // Without this entry both silently no-op. - MEASUREMENT_MAINTENANCE_QUEUE, - // v1.43 — the pulse day-mean dry run and rebuild, admin-triggered only. - PULSE_DAY_MEAN_QUEUE, - // v1.8.2 — one-time duplicate dose-slot cleanup. Boot discovery enqueues - // one job per user holding two live intake rows that snap to the same - // canonical slot (the pre-fix REMINDER-pending + API-taken pair). Also - // carries a daily discovery cron (03:28). Without this entry pg-boss never - // provisions it and both the boot enqueue and the cron silently no-op. - INTAKE_SLOT_DEDUP_QUEUE, - // v0.5.4 ios-coord — mood-reminder dispatch-ledger retention sweep. - MOOD_REMINDER_CLEANUP_QUEUE, - // v1.4.49 — push-attempt ledger cleanup. The daily schedule below would - // silently no-op without this entry. - PUSH_ATTEMPT_CLEANUP_QUEUE, - // v1.31.0 — data-arrival reaction-marker prune. Without this entry the - // daily schedule silently no-ops and the markers accumulate forever. - ARRIVAL_REACTION_CLEANUP_QUEUE, - // v1.35.3 — nightly refresh of the cycle forecast rows the reminder cron - // scans. Without this entry the schedule silently no-ops and the forecast - // is only as fresh as the last time somebody opened the calendar. - CYCLE_PREDICTION_REFRESH_QUEUE, - // v1.38 — nightly rebuild of the mood forecast rows. Without this entry the - // schedule silently no-ops: nothing else writes them, so the surface would - // answer "not enough data yet" forever on an account that has plenty. - MOOD_PROGNOSIS_REFRESH_QUEUE, - // v1.35.3 — nightly pin for newly unlocked achievements. Without this entry - // the schedule silently no-ops and no unlock date is ever made durable. - ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, - // v1.7.0 — soft-deleted measurement tombstone prune. Without this entry - // the daily schedule silently no-ops and pruned-past-retention tombstones - // pile up forever. - MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, - // v1.18.7 — Coach conversation-history retention prune. Without this entry - // the daily schedule silently no-ops and the encrypted coach_messages - // table grows unbounded. - COACH_MESSAGE_CLEANUP_QUEUE, - // v1.40 — workout-insight claim ledger retention prune. Without this entry - // the daily schedule silently no-ops and the claim table keeps every row - // whose workout was deleted. - WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE, - // v1.23 — free-text health-note encryption backfill. Boot discovery + a - // daily discovery cron enqueue one per-user job per account still holding a - // plaintext note; without this entry pg-boss never provisions the queue and - // both the boot enqueue and the cron silently no-op. - NOTE_ENCRYPTION_BACKFILL_QUEUE, - // v1.23 — admin-triggered encryption-key rotation. On-demand only (no - // cron): the admin panel enqueues a singleton run that re-encrypts the - // corpus to the active key. Without this entry pg-boss never provisions the - // queue and the admin trigger silently no-ops. - ENCRYPTION_KEY_ROTATE_QUEUE, - // v1.24 — expired/revoked MCP access-token + connection prune. Without this - // entry the daily schedule silently no-ops and the api_tokens table grows - // unbounded with dead 60-minute connector rows. - MCP_TOKEN_CLEANUP_QUEUE, - // v1.25 — medication free-text note encryption backfill. Boot discovery + a - // daily discovery cron enqueue one per-user job per account still holding a - // plaintext medication note; without this entry pg-boss never provisions the - // queue and both the boot enqueue and the cron silently no-op. - MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, - // v1.39.3 — Coach title + custom-metric note encryption backfill. Without - // this entry pg-boss never provisions the queue and both the boot enqueue - // and the cron silently no-op. - FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, - // Document vault — daily physical purge for tombstones past the 30-day - // undo grace (returns the encrypted blob's TOAST space). Without this entry - // the daily schedule silently no-ops and "deleted" documents hold backup - // weight forever. - DOCUMENT_PURGE_QUEUE, - // Nightly purge of expired tombstones for every soft-delete model the - // retention register hands to it (`soft-delete-retention.ts`). - SOFT_DELETE_PURGE_QUEUE, - // Nightly removal of expired sessions, tokens, challenges and trusted - // devices. - AUTH_LEDGER_CLEANUP_QUEUE, - // Document vault — hourly reaper for summary rows stuck on PENDING (a - // job dead past its retry budget). Without this entry the schedule - // silently no-ops and a dead "wird generiert" claim never converges to - // the honest UNAVAILABLE state. - DOCUMENT_SUMMARY_REAPER_QUEUE, - // Document vault P2 — on-demand content-search index backfill. Fired by the - // "index all documents" action (no cron): indexes a user's not-yet-indexed - // documents via one provider transcription each, consent + budget gated. - // Without this entry pg-boss never provisions the queue and the trigger - // silently no-ops. - CONTENT_INDEX_BACKFILL_QUEUE, - // Document AI — automatic per-document content indexing, enqueued on upload. - // Provider-first (vision) when configured + consented, else local text-layer - // extraction. Without this entry pg-boss never provisions the queue and every - // upload enqueue silently no-ops. - DOCUMENT_INDEX_QUEUE, - // v1.40 — background document AI runs ("Read with AI", lab scans): the - // request answers 202 and this queue does the read. Without this entry - // pg-boss never provisions the queue and every run waits until the reaper - // fails it as worker-unavailable. - DOCUMENT_AI_RUN_QUEUE, - // v1.40 — five-minute reaper for those runs: fails a run no worker took or - // one past its deadline, deletes a finished one after an hour. - DOCUMENT_AI_RUN_REAPER_QUEUE, - // Document vault — automatic per-document preview thumbnail, enqueued on - // upload. Pure local compute (canvas/pdfjs downscale), no egress. Without - // this entry pg-boss never provisions the queue and every upload enqueue - // silently no-ops. - DOCUMENT_THUMBNAIL_QUEUE, - // Document vault — boot-time preview-thumbnail backfill. Discovers accounts - // holding thumbnailable documents without a preview and fans out per-document - // thumbnail jobs. Without this entry pg-boss never provisions the queue and - // the boot discovery silently no-ops. - DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, - // Document AI — automatic per-document plain-language summary, enqueued on - // upload. Provider (vision) only, gated on the `documentsAutoAiRead` opt-in + - // egress consent + budget; persists the summary encrypted. Without this entry - // pg-boss never provisions the queue and every upload enqueue silently no-ops. - DOCUMENT_SUMMARY_QUEUE, - // v1.30.31 — auto-read catch-up. Scheduled on a genuine OFF→ON flip of the - // `documentsAutoAiRead` opt-in; fans out summary jobs for documents that were - // uploaded while the flag was still OFF. Without this entry pg-boss never - // provisions the queue and the catch-up enqueue silently no-ops. - DOCUMENT_SUMMARY_CATCHUP_QUEUE, -]; - -// `cronIsTheRetry` (retryLimit 0) marks the passes whose handler now returns -// `jobFailed` where it used to warn and carry on, AND whose failure repeats: -// a bulk DELETE or UPDATE that timed out will time out again on an immediate -// retry, so pg-boss's default two would run the same doomed statement three -// times a night. The queues without it are the ones whose failure is -// transient — the TLS probe, the off-host upload, the per-user environment -// fetch — and those still want the default retries. The restore drill sends -// `retryLimit: 0` in its own options for the other reason: it pages each -// failed account itself, and a retry would page the same accounts again. -const schedules: ScheduleEntry[] = [ - // v1.43.7 — nightly 03:40 check of `measurements`: a plain VACUUM when the - // dead rows pass the threshold, never a rebuild. Same singleton key as the - // operator's run, so the two collapse. - [ - MEASUREMENT_MAINTENANCE_QUEUE, - MEASUREMENT_MAINTENANCE_CRON, - { ...MEASUREMENT_MAINTENANCE_SEND_OPTIONS }, - ], - // v1.25 — daily 02:10 Europe/Berlin discovery tick (empty payload) - // that fans out one per-user environment fetch per opted-in account. - [ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON], - // v1.42 — daily 02:40 discovery for the air-quality history backfill, after - // the night's environment fetch has taken its share of the budget. - [ENVIRONMENT_AQ_HISTORY_QUEUE, ENVIRONMENT_AQ_HISTORY_CRON], - // v1.43 — daily 03:20 discovery for the Dawarich movement context. A - // failed run is an outage at the instance; the default retries apply. - [DAWARICH_SYNC_QUEUE, DAWARICH_SYNC_CRON], - // The send options are the point of this tuple: without them the weekly - // snapshot inherits pg-boss's 15-minute expiration and is killed mid-run on - // any sizeable record. See `data-backup-policy.ts` for why the window rides - // the schedule rather than the queue. - [DATA_BACKUP_QUEUE, DATA_BACKUP_CRON, DATA_BACKUP_SEND_OPTIONS], - [RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, cronIsTheRetry], - [IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON, cronIsTheRetry], - [AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON], - [ - STEP_UP_ELEVATION_CLEANUP_QUEUE, - STEP_UP_ELEVATION_CLEANUP_CRON, - cronIsTheRetry, - ], - [OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON, OFFHOST_BACKUP_SEND_OPTIONS], - [RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON, RESTORE_DRILL_SEND_OPTIONS], - // Nightly backstop for the purge the deletion kicks straight away: a - // request the bucket refused, or one written while the queue was down. - [OFFHOST_PURGE_QUEUE, OFFHOST_PURGE_CRON, cronIsTheRetry], - [HOST_METRIC_QUEUE, HOST_METRIC_CRON, cronIsTheRetry], - [FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON, cronIsTheRetry], - // v1.4.37 — hourly geo backfill. The helper is idempotent + capped - // at 5 000 rows per pass; running it at :40 every hour catches the - // long tail of audit rows that landed with the offline MMDB - // missing or the online provider unreachable. - [GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON, cronIsTheRetry], - // v1.37.7 (issue #659) — monthly runtime GeoLite2 refresh (8th, 04:00 - // Europe/Berlin, past MaxMind's first-Tuesday reissue). Deliberately NOT - // cronIsTheRetry: a month between ticks is far too long to be the retry, so - // the queue keeps pg-boss's default retries for a transient MaxMind hiccup. - [GEOLITE2_FETCH_QUEUE, GEOLITE2_FETCH_CRON], - // v1.12.2 ios-coord — every-6-hour TLS leaf SPKI probe (:07 off the - // hourly sync crons). Surfaces a pinned-leaf rotation well inside the - // ≥11-day re-pin window the iOS release owner needs. - [TLS_PIN_MONITOR_QUEUE, TLS_PIN_MONITOR_CRON], - // Fallback rescan every 30 minutes — protects against ingest paths - // that ship measurements without enqueueing a per-user job. The - // cron payload deliberately omits a `userId` so the handler iterates - // every user; per-user push-suppression cannot apply on the cron - // path (the silent flag is set by the ingest hooks). - [ - PR_DETECTION_QUEUE, - PR_DETECTION_FALLBACK_CRON, - { expireInSeconds: PR_DETECTION_EXPIRE_SECONDS }, - ], - // v1.4.25 — daily expire-stale pass for the per-pen inventory - // entities. Flips IN_USE rows whose 30-day clock has blown to - // EXPIRED at 03:30 Europe/Berlin (in the existing 02:xx–03:xx - // maintenance window, right after idempotency-cleanup). - [ - MEDICATION_INVENTORY_EXPIRE_QUEUE, - MEDICATION_INVENTORY_EXPIRE_CRON, - cronIsTheRetry, - ], - // v1.4.46 — hourly auto-skip for medication intakes the user never - // marked. Cron `5 * * * *` slots off the top-of-the-hour - // reminder-check (:00) so the hourly ticks don't pile up on the same - // boss poll. - [INTAKE_AUTO_SKIP_QUEUE, INTAKE_AUTO_SKIP_CRON, cronIsTheRetry], - // v1.42.2 — every 15 minutes, off the reminder tick and the auto-miss - // pass, so a dose of an auto-mark-taken medication reads taken within - // minutes of its time. - [INTAKE_AUTO_TAKE_QUEUE, INTAKE_AUTO_TAKE_CRON, cronIsTheRetry], - // v0.5.4 ios-coord — daily mood-reminder dispatch-ledger retention sweep. - [MOOD_REMINDER_CLEANUP_QUEUE, MOOD_REMINDER_CLEANUP_CRON, cronIsTheRetry], - // v1.15.19 — daily 03:28 Europe/Berlin duplicate dose-slot dedup - // discovery. The empty cron payload (no `userId`) is the handler's - // signal to run the discovery fan-out instead of a per-user pass, so - // cross-source duplicate slots created between deploys collapse within - // a day instead of waiting for the next worker reboot. - [INTAKE_SLOT_DEDUP_QUEUE, INTAKE_SLOT_DEDUP_CRON], - // v1.4.49 — daily 03:35 Europe/Berlin prune for push_attempts. - [PUSH_ATTEMPT_CLEANUP_QUEUE, PUSH_ATTEMPT_CLEANUP_CRON, cronIsTheRetry], - // v1.31.0 — daily 04:10 Europe/Berlin prune for arrival_reactions. - [ - ARRIVAL_REACTION_CLEANUP_QUEUE, - ARRIVAL_REACTION_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.7.0 — daily 03:40 Europe/Berlin prune for expired measurement - // tombstones. - [MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, MEASUREMENT_TOMBSTONE_CLEANUP_CRON], - // v1.18.7 — daily 03:50 Europe/Berlin prune for stale Coach history. - [COACH_MESSAGE_CLEANUP_QUEUE, COACH_MESSAGE_CLEANUP_CRON, cronIsTheRetry], - // v1.40 — daily 03:52 Europe/Berlin prune for the workout-insight claims. - [ - WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE, - WORKOUT_INSIGHT_CLAIM_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.23 — daily 03:55 Europe/Berlin note-encryption backfill discovery. - // The empty cron payload (no `userId`) is the handler's signal to fan out - // one per-user job per account still holding a plaintext note. - [NOTE_ENCRYPTION_BACKFILL_QUEUE, NOTE_ENCRYPTION_BACKFILL_CRON], - // v1.24 — daily 04:00 Europe/Berlin prune for dead MCP connector tokens. - [MCP_TOKEN_CLEANUP_QUEUE, MCP_TOKEN_CLEANUP_CRON, cronIsTheRetry], - // v1.35.3 — daily 04:20 Europe/Berlin refresh of the cycle forecast rows - // the reminder cron scans. Tomorrow's tick is the retry: the forecast moves - // on a day scale and the reminder windows are days wide. - [ - CYCLE_PREDICTION_REFRESH_QUEUE, - CYCLE_PREDICTION_REFRESH_CRON, - cronIsTheRetry, - ], - // v1.38 — daily 04:35 Europe/Berlin rebuild of the mood forecast rows. - // Tomorrow's tick is the retry: the fit moves on a day scale and a night - // late writes the same rows from the same days. - [MOOD_PROGNOSIS_REFRESH_QUEUE, MOOD_PROGNOSIS_REFRESH_CRON, cronIsTheRetry], - // v1.35.3 — daily 04:25 Europe/Berlin pin for newly unlocked achievements. - // Tomorrow's tick is the retry: the date written is the computed completion - // date, so a night late writes the identical row. - [ - ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, - ACHIEVEMENT_UNLOCK_SWEEP_CRON, - cronIsTheRetry, - ], - // v1.25 — daily 04:05 Europe/Berlin medication-note encryption backfill - // discovery. The empty cron payload (no `userId`) is the handler's signal to - // fan out one per-user job per account still holding a plaintext medication - // note. - [MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, MED_NOTES_ENCRYPTION_BACKFILL_CRON], - // v1.39.3 — daily 04:07 Europe/Berlin discovery for the Coach title + - // custom-metric note backfill. Empty payload = fan out per user. - [FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, FREE_TEXT_ENCRYPTION_BACKFILL_CRON], - // Document vault — daily 04:10 Europe/Berlin purge for tombstoned - // documents past the 30-day undo grace. - [DOCUMENT_PURGE_QUEUE, DOCUMENT_PURGE_CRON, cronIsTheRetry], - // Daily 03:42 Europe/Berlin, after the sync-domain tombstone cleanup. The - // next night is the retry: the purge picks up whatever a failed run left. - [SOFT_DELETE_PURGE_QUEUE, SOFT_DELETE_PURGE_CRON, cronIsTheRetry], - // Daily 03:35 Europe/Berlin. Same: the next night deletes what is left. - [AUTH_LEDGER_CLEANUP_QUEUE, AUTH_LEDGER_CLEANUP_CRON, cronIsTheRetry], - // Document vault — hourly heal of summary rows stuck on PENDING. The next - // tick is the retry: the updateMany is idempotent and the TTL predicate - // re-selects anything a failed tick left behind. - [DOCUMENT_SUMMARY_REAPER_QUEUE, DOCUMENT_SUMMARY_REAPER_CRON, cronIsTheRetry], - // v1.40 — five-minute pass over background document AI runs. The next tick - // is the retry: every write is conditional on the state it leaves. - [DOCUMENT_AI_RUN_REAPER_QUEUE, DOCUMENT_AI_RUN_REAPER_CRON, cronIsTheRetry], - // v1.32.1 (issue #588) — every-15-minute orphan-ImportJob sweep. Re-runs - // the same reconcile the boot path uses, so a stuck "unpacking" row - // whose worker crashed/restarted without the boot-time pass catching it - // (heartbeat not yet stale, pg-boss job not yet expired) still converges - // to a visible `failed` state within two ticks instead of staying stuck - // forever on an otherwise-healthy worker. - [IMPORT_JOB_RECONCILE_QUEUE, IMPORT_JOB_RECONCILE_CRON, cronIsTheRetry], -]; - -/** - * De-duplication policy per maintenance queue. - * - * Every retention / cleanup cron here (rate-limit, idempotency, audit-log, - * push-attempt, tombstone, coach-message, MCP-token, document-purge, …) is a - * keyless tick with no `singletonKey` on any send, so a policy would only - * constrain the empty key. Those are deliberately absent from this table. - * - * Two deliberate omissions worth naming, because both look like candidates: - * - * - ENVIRONMENT_FETCH_QUEUE is LEFT ALONE. It is genuinely ambiguous: the - * lookback refresh sends a per-user key, but an explicit-range backfill - * sends with NO options at all, deliberately, so that "it always runs". - * Any policy would collapse every keyless explicit-range backfill onto the - * shared empty key, so two different requested date ranges would silently - * become one. That is exactly the class of silent work-dropping a policy is - * supposed to prevent, so the queue keeps `standard`. The settings - * backfill (v1.42) is held per account by a keyed time slot - * (`singletonSeconds`), which pg-boss enforces under any policy; the - * travel-period refresh stays keyless so two periods never merge. - * - APPLE_HEALTH_IMPORT_V2_QUEUE, APPLE_HEALTH_IMPORT_LEGACY_QUEUE, - * DATA_BACKUP_QUEUE and PR_DETECTION_QUEUE send keylessly by design (each - * import / backup / detection run is a distinct unit of work). A policy - * would coalesce independent runs. - */ -const queuePolicies: QueuePolicyTable = { - // Per-user, boot- and cron-discovery driven, self-converging one-shots. The - // discovery predicate re-offers the job while the work is outstanding, so - // suppressing a duplicate of a queued OR active pass cannot lose work. - [INTAKE_SLOT_DEDUP_QUEUE]: { - policy: "exclusive", - reason: - "Per-user duplicate-slot cleanup; discovery drops the user once no colliding intake pair remains.", - }, - [NOTE_ENCRYPTION_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user note-encryption backfill; discovery drops the user once no plaintext note remains.", - }, - [MED_NOTES_ENCRYPTION_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user medication-note encryption backfill; discovery drops the user once no plaintext note remains.", - }, - [FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user encryption backfill (Coach title, custom-metric note, practitioner phone + address, workout GPS track); discovery drops the user once no readable value remains.", - }, - [DOCUMENT_THUMBNAIL_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user thumbnail backfill; discovery drops the user once every thumbnailable document has a preview.", - }, - [CONTENT_INDEX_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user index-all run, triggered from the documents surface. Idempotent (it indexes only not-yet-indexed documents), so a second press while one runs is correctly a no-op.", - }, - [ENCRYPTION_KEY_ROTATE_QUEUE]: { - policy: "exclusive", - reason: - "Fixed singleton key, admin-triggered. Two concurrent corpus rotations must never overlap; the route already reports a suppressed send back as alreadyQueued.", - }, - // v1.42 — VACUUM / REINDEX of `measurements`, operator or nightly. - [MEASUREMENT_MAINTENANCE_QUEUE]: { - policy: "exclusive", - reason: - "Fixed singleton key on the admin send and the nightly tick alike. Two maintenance passes would vacuum or rebuild side by side; the route reports a suppressed send back as enqueued: false.", - }, - // v1.43 — the pulse day-mean dry run and rebuild. - [PULSE_DAY_MEAN_QUEUE]: { - policy: "exclusive", - reason: - "Fixed singleton key, admin-triggered. A dry run beside a rebuild would compare against rollups being rewritten; the route answers 409 busy instead.", - }, - // v1.42 — compaction-tombstone backlog purge. `short`, not `exclusive`: - // a run that stops early sends its own follow-up while it is still active, - // and `exclusive` would swallow that send. A boot's send while a run is - // queued collapses into it; the delete predicate is idempotent, so a boot - // that lands beside an active run costs a duplicate scan, never a wrong row. - [COMPACTION_TOMBSTONE_PURGE_QUEUE]: { - policy: "short", - reason: - "Fixed singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", - }, - // v1.42 — per-account fold repair. `short` for the same reason: a run the - // budget stopped sends its own follow-up for the account while it is still - // active. A boot's send for a queued account collapses into it; a duplicate - // beside an active run rewrites nothing, since only differing means are - // written. - // v1.42 — the air-quality history backfill. `short` for the same reason: a - // run with work left sends its own follow-up while it is still active. A - // send for an account whose run is queued collapses into it; one beside an - // active run only re-reads what is done, since done days are never fetched. - [ENVIRONMENT_AQ_HISTORY_QUEUE]: { - policy: "short", - reason: - "Per-account singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", - }, - [MEASUREMENT_FOLD_REPAIR_QUEUE]: { - policy: "short", - reason: - "Per-account singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", - }, - - // Per-document, enqueued on upload. `short`, NOT `exclusive`: each handler - // re-reads the document when it starts, so collapsing sends that arrive while - // an identical job is still queued is safe. `exclusive` would additionally - // suppress a re-process requested after the current job had already read the - // old bytes, leaving a stale summary/thumbnail/index behind with no discovery - // pass to re-offer it — these queues have no cron to re-converge. - [DOCUMENT_INDEX_QUEUE]: { - policy: "short", - reason: - "Per-document, no re-converging discovery pass. Collapse queued duplicates only, so a re-index after a content change is never dropped.", - }, - [DOCUMENT_THUMBNAIL_QUEUE]: { - policy: "short", - reason: - "Per-document, no re-converging discovery pass. Collapse queued duplicates only.", - }, - [DOCUMENT_SUMMARY_QUEUE]: { - policy: "short", - reason: - "Per-document, no re-converging discovery pass. Collapse queued duplicates only.", - }, - [DOCUMENT_SUMMARY_CATCHUP_QUEUE]: { - policy: "short", - reason: - "Per-user, keyed on the opt-in flip. Collapse a double toggle into one pass; the pass re-reads its candidate set when it starts.", - }, -}; - -/** - * Register every maintenance / ops / cleanup queue. Returns the queue names - * created (for the boot-level aggregate assertion). - */ -export async function registerMaintenanceQueues( - boss: PgBoss, -): Promise { - await createAndSchedule(boss, allQueues, schedules, queuePolicies); - - await createAndWork( - boss, - DATA_BACKUP_QUEUE, - { localConcurrency: 1 }, - lockedPass( - DATA_BACKUP_QUEUE, - () => WHOLE_PASS, - heavyLane(DATA_BACKUP_QUEUE, handleDataBackup), - ), - ); - await createAndWork( - boss, - RATE_LIMIT_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleRateLimitCleanup, - ); - await createAndWork( - boss, - IDEMPOTENCY_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleIdempotencyCleanup, - ); - await createAndWork( - boss, - AUDIT_LOG_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleAuditLogCleanup, - ); - await createAndWork( - boss, - STEP_UP_ELEVATION_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleStepUpElevationCleanup, - ); - // With metadata: the handler resumes a retry from the job's creation time. - await createAndWork( - boss, - OFFHOST_BACKUP_QUEUE, - { localConcurrency: 1, includeMetadata: true }, - lockedPass( - OFFHOST_BACKUP_QUEUE, - () => WHOLE_PASS, - heavyLane(OFFHOST_BACKUP_QUEUE, handleOffhostBackup), - ), - ); - // Locked: a drill that outlives a restart or a manual run never reads the - // bucket beside another one. - await createAndWork( - boss, - RESTORE_DRILL_QUEUE, - { localConcurrency: 1 }, - lockedPass( - RESTORE_DRILL_QUEUE, - () => WHOLE_PASS, - heavyLane(RESTORE_DRILL_QUEUE, handleRestoreDrill), - ), - ); - await createAndWork( - boss, - OFFHOST_PURGE_QUEUE, - { localConcurrency: 1 }, - handleOffhostPurge, - ); - // One restore at a time per worker: each holds a transaction over a whole - // account and a batch of readings in memory, and two at once on a small host - // is how a restore that fits alone would run out of heap. - await createAndWork( - boss, - BACKUP_RESTORE_QUEUE, - { localConcurrency: 1 }, - heavyLane(BACKUP_RESTORE_QUEUE, handleBackupRestore), - ); - // A restore the previous process was running is picked up again once its - // heartbeat has gone stale; see `sweepInterruptedRestores`. - await enqueueBackupRestoreSweepAtBoot(boss); - await createAndWork( - boss, - HOST_METRIC_QUEUE, - { localConcurrency: 1 }, - handleHostMetricSample, - ); - await createAndWork( - boss, - FEEDBACK_AGGREGATOR_QUEUE, - { localConcurrency: 1 }, - handleFeedbackAggregator, - ); - await createAndWork( - boss, - GEO_BACKFILL_QUEUE, - { localConcurrency: 1 }, - handleGeoBackfill, - ); - // v1.37.7 (issue #659) — runtime GeoLite2 fetch. Single-flight: one download - // at a time, and two ticks racing the same atomic place is wasted bandwidth. - await createAndWork( - boss, - GEOLITE2_FETCH_QUEUE, - { localConcurrency: 1 }, - handleGeolite2Fetch, - ); - // v1.12.2 ios-coord — TLS leaf SPKI-change monitor. Single-flight: one - // short outbound TLS handshake per tick, no benefit to overlapping ticks. - await createAndWork( - boss, - TLS_PIN_MONITOR_QUEUE, - { localConcurrency: 1 }, - handleTlsPinMonitor, - ); - await createAndWork( - boss, - MOOD_REMINDER_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleMoodReminderCleanup, - ); - // v1.4.49 — daily prune of the push-attempt ledger. Single-flight - // matches every other cleanup queue; two ticks racing on the same - // DELETE statement is wasted work and the second tick's payload - // would be a no-op anyway. - await createAndWork( - boss, - PUSH_ATTEMPT_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handlePushAttemptCleanup, - ); - // v1.31.0 — daily prune of the data-arrival reaction markers. Single-flight - // like every other cleanup queue: two ticks racing the same DELETE is - // wasted work. - await createAndWork( - boss, - ARRIVAL_REACTION_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleArrivalReactionCleanup, - ); - // v1.35.3 — nightly cycle-forecast refresh. Single-flight: two ticks - // recomputing the same cohort is wasted work and the second upsert is a - // no-op against the debounce. - await createAndWork( - boss, - CYCLE_PREDICTION_REFRESH_QUEUE, - { localConcurrency: 1 }, - handleCyclePredictionRefresh, - ); - // v1.38 — nightly mood-forecast rebuild. Single-flight: two ticks would fit - // the same windows twice and the second pass would write the identical rows. - await createAndWork( - boss, - MOOD_PROGNOSIS_REFRESH_QUEUE, - { localConcurrency: 1 }, - handleMoodPrognosisRefresh, - ); - // v1.35.3 — nightly achievement-unlock pin. Single-flight: the underlying - // createMany is idempotent on the (userId, achievementId) unique, so a - // second concurrent tick would only re-walk the same accounts. - await createAndWork( - boss, - ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, - { localConcurrency: 1 }, - handleAchievementUnlockSweep, - ); - // v1.7.0 — daily prune of expired measurement tombstones. Single-flight - // like every other cleanup queue. - await createAndWork( - boss, - MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleMeasurementTombstoneCleanup, - ); - // v1.18.7 — daily prune of the encrypted Coach conversation history. - // Single-flight like every other cleanup queue; two ticks racing on the - // same DELETE is wasted work and the second is a no-op. - await createAndWork( - boss, - COACH_MESSAGE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleCoachMessageCleanup, - ); - // v1.40 — daily prune of the workout-insight claim ledger. Single-flight - // like every other cleanup queue. - await createAndWork( - boss, - WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleWorkoutInsightClaimCleanup, - ); - // MCP Phase 3 (M2) — daily prune of dead MCP connector access tokens + - // long-revoked connection anchors. Single-flight like every other cleanup. - await createAndWork( - boss, - MCP_TOKEN_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleMcpTokenCleanup, - ); - // Document vault — daily purge of tombstoned documents past the 30-day - // undo grace. Single-flight like every other cleanup queue; the underlying - // deleteMany is idempotent so a duplicate tick is a no-op. - await createAndWork( - boss, - DOCUMENT_PURGE_QUEUE, - { localConcurrency: 1 }, - handleDocumentPurge, - ); - // Single-flight like every other cleanup queue; both deletes are bounded - // and idempotent, so a duplicate tick finds nothing. - await createAndWork( - boss, - SOFT_DELETE_PURGE_QUEUE, - { localConcurrency: 1 }, - handleSoftDeletePurge, - ); - await createAndWork( - boss, - AUTH_LEDGER_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleAuthLedgerCleanup, - ); - // v1.40 — background document AI runs. Each job is one run, under the lock - // `lockedPass` takes per run id, so a redelivery never reads beside the - // first delivery; the claim inside is conditional as well. - await createAndWork( - boss, - DOCUMENT_AI_RUN_QUEUE, - { localConcurrency: DOCUMENT_AI_RUN_CONCURRENCY }, - lockedPass( - DOCUMENT_AI_RUN_QUEUE, - (job) => `run:${job.data.runId ?? ""}`, - handleDocumentAiRunJobs, - ), - ); - // v1.40 — the five-minute run reaper. Single-flight; every write in it is - // conditional, so a duplicate tick is a no-op. - await createAndWork( - boss, - DOCUMENT_AI_RUN_REAPER_QUEUE, - { localConcurrency: 1 }, - handleDocumentAiRunReaper, - ); - // Document vault — hourly stale-PENDING summary reaper. Single-flight; - // the underlying updateMany is idempotent so a duplicate tick is a no-op. - await createAndWork( - boss, - DOCUMENT_SUMMARY_REAPER_QUEUE, - { localConcurrency: 1 }, - handleDocumentSummaryReaper, - ); - await createAndWork( - boss, - PR_DETECTION_QUEUE, - { localConcurrency: PR_DETECTION_CONCURRENCY }, - // An ingest job is one account; the cron pass is every account, one run - // at a time. - lockedPass( - PR_DETECTION_QUEUE, - (job) => { - const userId = (job.data as PrDetectionPayload | undefined)?.userId; - return userId ? `user:${userId}` : WHOLE_PASS; - }, - handlePrDetection, - ), - ); - await createAndWork( - boss, - MEDICATION_INVENTORY_EXPIRE_QUEUE, - { localConcurrency: 1 }, - handleMedicationInventoryExpire, - ); - // v1.4.46 — hourly auto-skip for stale unmarked intakes. - // Single-flight: two ticks racing against the same row pile is wasted - // work, and the underlying `updateMany` is the canonical idempotent - // shape anyway. - await createAndWork( - boss, - INTAKE_AUTO_SKIP_QUEUE, - { localConcurrency: 1 }, - handleIntakeAutoSkip, - ); - // v1.42.2 — auto-mark-taken pass. Single-flight: two passes racing over - // the same slots would only collide on the conditional convert. - await createAndWork( - boss, - INTAKE_AUTO_TAKE_QUEUE, - { localConcurrency: 1 }, - handleIntakeAutoTake, - ); - // v1.4.34 — Apple Health export.zip ingest worker. localConcurrency - // caps at 1 because the parse loop is CPU-bound and a concurrent - // second 1 GB import would race the first for RSS. - await createAndWork( - boss, - APPLE_HEALTH_IMPORT_V2_QUEUE, - { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, - heavyLane(APPLE_HEALTH_IMPORT_V2_QUEUE, async (jobs) => { - // pg-boss v12 work callbacks always receive an array (batched - // worker mode); for our concurrency-1 + batchSize-1 case we just - // process each job sequentially. - // - // Each import is independent work, so a failed one does not abort the - // rest of the batch; the first failure is what the job reports. - let failure: JobOutcome | null = null; - for (const job of jobs) { - const outcome = await handleAppleHealthImport(job); - if (!outcome.ok && failure === null) failure = outcome; - } - return failure ?? jobDone({ jobs: jobs.length }); - }), - ); - // Drain pre-revision-2 backlog by moving each claimed legacy job onto the - // isolated v2 queue. This preserves the ImportJob status id without letting - // the current parser execute under a revision-1 mirror. - await createAndWork( - boss, - APPLE_HEALTH_IMPORT_LEGACY_QUEUE, - { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await migrateLegacyAppleHealthImport(job); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // v1.42 (#972) — Health Connect import. One at a time per worker, like the - // Apple Health import: each run reads a whole export and consumes its - // staged upload. - await createAndWork( - boss, - HEALTH_CONNECT_IMPORT_QUEUE, - { localConcurrency: HEALTH_CONNECT_IMPORT_CONCURRENCY }, - heavyLane(HEALTH_CONNECT_IMPORT_QUEUE, handleHealthConnectImport), - ); - // v1.42 — compaction-tombstone backlog purge. Serial: each batch holds - // an account against a concurrent restore. - await createAndWork( - boss, - COMPACTION_TOMBSTONE_PURGE_QUEUE, - { localConcurrency: 1 }, - handleCompactionTombstonePurge, - ); - // v1.42 — fold repair. Serial: each day holds the account against a - // concurrent restore, and the purge waits for it. - await createAndWork( - boss, - MEASUREMENT_FOLD_REPAIR_QUEUE, - { localConcurrency: 1 }, - handleMeasurementFoldRepair, - ); - // v1.42 — measurement table maintenance, operator-triggered only. - await createAndWork( - boss, - MEASUREMENT_MAINTENANCE_QUEUE, - { localConcurrency: 1 }, - handleMeasurementMaintenance, - ); - // v1.43 — pulse day-mean dry run and rebuild, operator-triggered only. - await createAndWork( - boss, - PULSE_DAY_MEAN_QUEUE, - { localConcurrency: 1 }, - handlePulseDayMean, - ); - // v1.32.1 (issue #588) — periodic orphan-ImportJob sweep. Single-flight: - // the underlying `updateMany` is idempotent and two ticks racing the same - // read-then-write pass would just be wasted work. - await createAndWork( - boss, - IMPORT_JOB_RECONCILE_QUEUE, - { localConcurrency: 1 }, - handleImportJobReconcileTick, - ); - await createAndWork( - boss, - MEDICATION_INTAKE_IMPORT_QUEUE, - { - localConcurrency: MEDICATION_INTAKE_IMPORT_CONCURRENCY, - includeMetadata: true, - }, - async (jobs) => { - // Independent per-delivery imports: run them all, report the first - // failure rather than losing it behind a later success. - let failure: JobOutcome | null = null; - for (const job of jobs) { - const outcome = await handleMedicationIntakeImport(job); - if (!outcome.ok && failure === null) failure = outcome; - } - return failure ?? jobDone({ jobs: jobs.length }); - }, - ); - - // v1.8.2 — duplicate dose-slot cleanup worker. The boot enqueue helper - // below sends one job per user holding two live intake rows that snap - // to the same canonical slot; this handler keeps the winner, soft- - // deletes the losers, normalises the winner's scheduledFor, and - // recomputes the affected compliance rollups. Serial concurrency so the - // pass never crowds the request pool. - // - // v1.15.19 — the queue also carries a daily cron tick (03:28, scheduled - // above) whose payload omits `userId`. That tick runs the SAME discovery - // fan-out the boot pass uses, so duplicate slots created between deploys - // collapse within a day instead of waiting for the next worker reboot. - await createAndWork>( - boss, - INTAKE_SLOT_DEDUP_QUEUE, - { localConcurrency: INTAKE_SLOT_DEDUP_CONCURRENCY }, - async (jobs) => { - let users = 0; - let discoveryEnqueued = 0; - let discoverySkipped = 0; - // The discovery helper returns its error instead of throwing, so a - // failed fan-out would otherwise leave no trace on the job at all. - let discoveryFailed = 0; - let slotsCollapsed = 0; - let rowsSoftDeleted = 0; - let rowsNormalised = 0; - let daysRecomputed = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - // Daily discovery tick — fan out one per-user job per account - // still holding duplicate-slot candidates (singletonKey-coalesced, - // identical to the boot pass). - const result = await enqueueBootTimeIntakeSlotDedup(); - discoveryEnqueued += result.enqueued; - discoverySkipped += result.skipped; - if (result.error) discoveryFailed++; - workerLog( - "info", - `[intake-slot-dedup] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, - ); - continue; - } - try { - const summary = await dedupeUserIntakeSlots(userId); - users++; - slotsCollapsed += summary.slotsCollapsed; - rowsSoftDeleted += summary.rowsSoftDeleted; - rowsNormalised += summary.rowsNormalised; - daysRecomputed += summary.daysRecomputed; - workerLog( - "info", - `[intake-slot-dedup] user=${userId} slotsCollapsed=${summary.slotsCollapsed} rowsSoftDeleted=${summary.rowsSoftDeleted} rowsNormalised=${summary.rowsNormalised} daysRecomputed=${summary.daysRecomputed}`, - ); - } catch (err) { - recordError(); - workerLog("error", `[intake-slot-dedup] user=${userId} failed`, err); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - discovery_enqueued: discoveryEnqueued, - discovery_skipped: discoverySkipped, - discovery_failed: discoveryFailed, - slots_collapsed: slotsCollapsed, - rows_soft_deleted: rowsSoftDeleted, - rows_normalised: rowsNormalised, - days_recomputed: daysRecomputed, - }); - }, - ); - - // v1.23 — note-encryption backfill worker. The boot enqueue helper (and the - // daily 03:55 discovery tick) send one job per user still holding a plaintext - // note; this handler migrates that user's rows to the encrypted columns. An - // empty-userId payload is the daily discovery tick and re-fans the per-user - // jobs (singletonKey-coalesced, identical to the boot pass). Serial - // concurrency so the migration never crowds the request pool. - await createAndWork>( - boss, - NOTE_ENCRYPTION_BACKFILL_QUEUE, - { localConcurrency: NOTE_ENCRYPTION_BACKFILL_CONCURRENCY }, - async (jobs) => { - let users = 0; - let discoveryEnqueued = 0; - let discoverySkipped = 0; - // The discovery helper returns its error instead of throwing, so a - // failed fan-out would otherwise leave no trace on the job at all. - let discoveryFailed = 0; - let measurements = 0; - let moodEntries = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - const result = await enqueueBootTimeNoteEncryptionBackfill(); - discoveryEnqueued += result.enqueued; - discoverySkipped += result.skipped; - if (result.error) discoveryFailed++; - workerLog( - "info", - `[note-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, - ); - continue; - } - try { - const { measurementsMigrated, moodEntriesMigrated } = - await runNoteEncryptionBackfillForUser(userId); - users++; - measurements += measurementsMigrated; - moodEntries += moodEntriesMigrated; - workerLog( - "info", - `[note-encryption-backfill] user=${userId} measurements=${measurementsMigrated} moodEntries=${moodEntriesMigrated}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[note-encryption-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - discovery_enqueued: discoveryEnqueued, - discovery_skipped: discoverySkipped, - discovery_failed: discoveryFailed, - measurements_migrated: measurements, - mood_entries_migrated: moodEntries, - }); - }, - ); - - // v1.23 — admin-triggered encryption-key rotation (on-demand, singleton). - await createAndWork( - boss, - ENCRYPTION_KEY_ROTATE_QUEUE, - { localConcurrency: ENCRYPTION_KEY_ROTATE_CONCURRENCY }, - lockedPass( - ENCRYPTION_KEY_ROTATE_QUEUE, - () => WHOLE_PASS, - handleEncryptionKeyRotate, - ), - ); - - // v1.25 — medication-note encryption backfill worker. The boot enqueue - // helper (and the daily 04:05 discovery tick) send one job per user still - // holding a plaintext medication note; this handler migrates that user's - // rows across all three tables to the encrypted columns. An empty-userId - // payload is the daily discovery tick and re-fans the per-user jobs - // (singletonKey-coalesced, identical to the boot pass). Serial concurrency - // so the migration never crowds the request pool. - await createAndWork>( - boss, - MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, - { localConcurrency: MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY }, - async (jobs) => { - let users = 0; - let discoveryEnqueued = 0; - let discoverySkipped = 0; - // The discovery helper returns its error instead of throwing, so a - // failed fan-out would otherwise leave no trace on the job at all. - let discoveryFailed = 0; - let sideEffects = 0; - let doseChanges = 0; - let inventoryItems = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - const result = await enqueueBootTimeMedNotesEncryptionBackfill(); - discoveryEnqueued += result.enqueued; - discoverySkipped += result.skipped; - if (result.error) discoveryFailed++; - workerLog( - "info", - `[med-notes-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, - ); - continue; - } - try { - const { - sideEffectsMigrated, - doseChangesMigrated, - inventoryItemsMigrated, - } = await runMedNotesEncryptionBackfillForUser(userId); - users++; - sideEffects += sideEffectsMigrated; - doseChanges += doseChangesMigrated; - inventoryItems += inventoryItemsMigrated; - workerLog( - "info", - `[med-notes-encryption-backfill] user=${userId} sideEffects=${sideEffectsMigrated} doseChanges=${doseChangesMigrated} inventoryItems=${inventoryItemsMigrated}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[med-notes-encryption-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - discovery_enqueued: discoveryEnqueued, - discovery_skipped: discoverySkipped, - discovery_failed: discoveryFailed, - side_effects_migrated: sideEffects, - dose_changes_migrated: doseChanges, - inventory_items_migrated: inventoryItems, - }); - }, - ); - - // v1.39.3 — Coach title + custom-metric note encryption backfill worker. - // Same shape as the medication-note worker above: an empty-userId payload is - // the daily discovery tick, a userId payload migrates that account's rows. - await createAndWork>( - boss, - FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, - { localConcurrency: FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY }, - async (jobs) => { - let users = 0; - let discoveryEnqueued = 0; - let discoverySkipped = 0; - let discoveryFailed = 0; - let titles = 0; - let notes = 0; - let contacts = 0; - let routes = 0; - let appointmentAddresses = 0; - let contactAudits = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - const result = await enqueueBootTimeFreeTextEncryptionBackfill(); - discoveryEnqueued += result.enqueued; - discoverySkipped += result.skipped; - if (result.error) discoveryFailed++; - workerLog( - "info", - `[free-text-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, - ); - continue; - } - try { - const { - conversationTitlesMigrated, - metricNotesMigrated, - practitionerContactsMigrated, - routeGeometriesMigrated, - appointmentAddressesCleared, - contactAuditRowsScrubbed, - } = await runFreeTextEncryptionBackfillForUser(userId); - users++; - titles += conversationTitlesMigrated; - notes += metricNotesMigrated; - contacts += practitionerContactsMigrated; - routes += routeGeometriesMigrated; - appointmentAddresses += appointmentAddressesCleared; - contactAudits += contactAuditRowsScrubbed; - workerLog( - "info", - `[free-text-encryption-backfill] user=${userId} titles=${conversationTitlesMigrated} notes=${metricNotesMigrated} contacts=${practitionerContactsMigrated} routes=${routeGeometriesMigrated} appointment_addresses=${appointmentAddressesCleared} contact_audits=${contactAuditRowsScrubbed}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[free-text-encryption-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - discovery_enqueued: discoveryEnqueued, - discovery_skipped: discoverySkipped, - discovery_failed: discoveryFailed, - conversation_titles_migrated: titles, - metric_notes_migrated: notes, - practitioner_contacts_migrated: contacts, - route_geometries_migrated: routes, - appointment_addresses_cleared: appointmentAddresses, - contact_audit_rows_scrubbed: contactAudits, - }); - }, - ); - - // Document vault P2 — on-demand content-search index backfill worker. The - // trigger endpoint sends one per-user job; this handler indexes that user's - // not-yet-indexed documents (consent + budget gated, bounded + resumable). - // Serial concurrency so the provider calls never crowd the request pool. - await createAndWork( - boss, - CONTENT_INDEX_BACKFILL_QUEUE, - { localConcurrency: CONTENT_INDEX_BACKFILL_CONCURRENCY }, - lockedPass( - CONTENT_INDEX_BACKFILL_QUEUE, - (job) => `user:${job.data.userId ?? ""}`, - async (jobs) => { - let users = 0; - let stoppedEarly = false; - const shouldStop = jobBudget(jobs); - let skipped = 0; - let documentsIndexed = 0; - let documentsRetokenised = 0; - let documentsSkipped = 0; - let documentsFailed = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - skipped++; - continue; - } - try { - const { - indexed, - retokenised, - skipped: docsSkipped, - failed: docsFailed, - reason, - stoppedEarly: stopped, - } = await runContentIndexBackfillForUser(userId, shouldStop); - users++; - stoppedEarly ||= stopped === true; - documentsIndexed += indexed; - documentsRetokenised += retokenised; - documentsSkipped += docsSkipped; - documentsFailed += docsFailed; - workerLog( - "info", - `[document-content-index-backfill] user=${userId} indexed=${indexed} retokenised=${retokenised} skipped=${docsSkipped} failed=${docsFailed} reason=${reason}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[document-content-index-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - skipped, - documents_indexed: documentsIndexed, - documents_retokenised: documentsRetokenised, - documents_skipped: documentsSkipped, - documents_failed: documentsFailed, - stopped_early: stoppedEarly, - }); - }, - ), - ); - - // Document AI — automatic per-document content indexing. Enqueued on upload - // (one job per stored document); provider-first (vision) when configured + - // consented, else local text-layer extraction. Serial concurrency so the - // provider calls + PDF parse never crowd the request pool. - await createAndWork( - boss, - DOCUMENT_INDEX_QUEUE, - { localConcurrency: DOCUMENT_INDEX_CONCURRENCY }, - async (jobs) => { - let documents = 0; - let skipped = 0; - for (const job of jobs) { - const { userId, documentId } = job.data; - if (!userId || !documentId) { - skipped++; - continue; - } - try { - await runDocumentIndex(job.data); - documents++; - } catch (err) { - recordError(); - workerLog( - "error", - `[document-index] user=${userId} document=${documentId} failed`, - err, - ); - throw err; - } - } - return jobDone({ jobs: jobs.length, documents, skipped }); - }, - ); - - // Document vault — automatic per-document preview thumbnail. Enqueued on - // upload (one job per stored document); decodes + downscales the original to - // a small encrypted JPEG. Pure local compute; serial concurrency so the - // canvas/pdfjs decode never crowds the request pool. - await createAndWork( - boss, - DOCUMENT_THUMBNAIL_QUEUE, - { localConcurrency: DOCUMENT_THUMBNAIL_CONCURRENCY }, - async (jobs) => { - let documents = 0; - let skipped = 0; - for (const job of jobs) { - const { userId, documentId } = job.data; - if (!userId || !documentId) { - skipped++; - continue; - } - try { - await runDocumentThumbnail(job.data); - documents++; - } catch (err) { - recordError(); - workerLog( - "error", - `[document-thumbnail] user=${userId} document=${documentId} failed`, - err, - ); - throw err; - } - } - return jobDone({ jobs: jobs.length, documents, skipped }); - }, - ); - - // Document AI — automatic per-document plain-language summary. Enqueued on - // upload (one job per stored document); runs the vision provider ONLY when - // the `documentsAutoAiRead` opt-in is ON (egress consent + budget gated) and - // persists the summary encrypted. Serial concurrency so the provider call - // never crowds the request pool. - await createAndWork( - boss, - DOCUMENT_SUMMARY_QUEUE, - { localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY }, - async (jobs) => { - let documents = 0; - let skipped = 0; - for (const job of jobs) { - const { userId, documentId } = job.data; - if (!userId || !documentId) { - skipped++; - continue; - } - try { - await runDocumentSummaryJob(job.data); - documents++; - } catch (err) { - recordError(); - workerLog( - "error", - `[document-summary] user=${userId} document=${documentId} failed`, - err, - ); - throw err; - } - } - return jobDone({ jobs: jobs.length, documents, skipped }); - }, - ); - - // Document AI — auto-read catch-up worker. A genuine OFF→ON flip of the - // `documentsAutoAiRead` opt-in sends one per-user job; this handler fans out - // per-document summary jobs for that user's already-stored, un-summarised - // documents (the upload-time enqueue no-opped while the flag was OFF). It - // only enqueues — the opt-in, egress consent and budget gates all still run - // per document inside the summary job. Serial concurrency. - await createAndWork( - boss, - DOCUMENT_SUMMARY_CATCHUP_QUEUE, - { localConcurrency: DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY }, - async (jobs) => { - let users = 0; - let skipped = 0; - let summariesEnqueued = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - skipped++; - continue; - } - try { - const { enqueued } = await runSummaryCatchUpForUser(userId); - users++; - summariesEnqueued += enqueued; - } catch (err) { - recordError(); - workerLog( - "error", - `[document-summary-catchup] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - skipped, - summaries_enqueued: summariesEnqueued, - }); - }, - ); - - // Document vault — boot-time preview-thumbnail backfill worker. The boot - // discovery sends one per-user job; this handler fans out per-document - // thumbnail jobs for that user's not-yet-thumbnailed documents. Serial - // concurrency; cheap discovery + enqueue only. - await createAndWork( - boss, - DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, - { localConcurrency: DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY }, - async (jobs) => { - let users = 0; - let skipped = 0; - let thumbnailsEnqueued = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - skipped++; - continue; - } - try { - const { enqueued } = await runThumbnailBackfillForUser(userId); - users++; - thumbnailsEnqueued += enqueued; - workerLog( - "info", - `[document-thumbnail-backfill] user=${userId} enqueued=${enqueued}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[document-thumbnail-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - skipped, - thumbnails_enqueued: thumbnailsEnqueued, - }); - }, - ); - - // v1.42 — the air-quality history backfill. Serial: every run draws on the - // same instance-wide Open-Meteo budget as the nightly fetch. - await createAndWork( - boss, - ENVIRONMENT_AQ_HISTORY_QUEUE, - { localConcurrency: 1 }, - async (jobs) => { - let failure: JobOutcome | null = null; - for (const job of jobs) { - const outcome = await handleEnvironmentAqHistory(boss, job.data); - if (!outcome.ok && failure === null) failure = outcome; - } - return failure ?? jobDone({ jobs: jobs.length }); - }, - ); - - // v1.25 — nightly environment fetch. The daily discovery tick (empty - // payload) fans out one per-user job per opted-in account; the queue also - // serves the on-demand backfill payloads from the settings surface. Serial - // concurrency so the staggered outbound fetches never crowd the request pool. - await createAndWork( - boss, - ENVIRONMENT_FETCH_QUEUE, - { localConcurrency: ENVIRONMENT_FETCH_CONCURRENCY }, - async (jobs) => { - // A discovery tick and a per-user backfill share this queue and are - // independent of each other, so the pass runs to the end and reports - // the first failure it saw. - let failure: JobOutcome | null = null; - for (const job of jobs) { - const outcome = await handleEnvironmentFetch(boss, job.data); - if (!outcome.ok && failure === null) failure = outcome; - } - return failure ?? jobDone({ jobs: jobs.length }); - }, - ); - - // v1.43 — Dawarich movement context. Serial: one account at a time, so a - // slow instance never crowds the request pool. - await createAndWork( - boss, - DAWARICH_SYNC_QUEUE, - { localConcurrency: 1 }, - (jobs) => handleDawarichSync(boss, jobs), - ); - - return allQueues; -} - -/** - * Fire-and-forget boot discovery for the duplicate dose-slot cleanup. Finds - * every user holding two live intake rows within the drift window on the same - * medication and enqueues one dedup job per account. Idempotent across reboots: - * collapsed losers are soft-deleted so the `deleted_at IS NULL` discovery - * predicate drops them. Errors are returned through the helper's result value — - * the worker boot never fails because of a dedup miss. - */ -export async function enqueueMaintenanceBootDiscovery(): Promise { - try { - const { enqueued, skipped, error } = await enqueueBootTimeIntakeSlotDedup(); - if (error) { - workerLog("error", `[intake-slot-dedup] boot discovery failed: ${error}`); - } else { - workerLog( - "info", - `[intake-slot-dedup] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[intake-slot-dedup] boot discovery threw an unexpected error", - err, - ); - } - - // v1.23 — note-encryption backfill. Finds every user still holding a - // plaintext mood/measurement note (no ciphertext yet) and enqueues one - // migration job per account. Idempotent across reboots: a migrated row nulls - // its plaintext column and drops off the discovery predicate. - try { - // Stage 5 — last of the staggered boot backfills. The heavy full-history - // loads each `localConcurrency: 1` used to drain in parallel onto one heavy - // tenant at boot; threading an increasing `startAfter` delay spreads them - // across a window. The daily cron tick (above) passes no offset and stays - // immediate. - const { enqueued, skipped, error } = - await enqueueBootTimeNoteEncryptionBackfill( - BOOT_BACKFILL_STAGGER_SECONDS * 5, - ); - if (error) { - workerLog( - "error", - `[note-encryption-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[note-encryption-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[note-encryption-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.25 — medication free-text note encryption backfill. Finds every user - // still holding a plaintext side-effect / dose-change / inventory-item note - // (no ciphertext yet) and enqueues one migration job per account. Idempotent - // across reboots: a migrated row nulls its plaintext column and drops off the - // discovery predicate. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeMedNotesEncryptionBackfill(); - if (error) { - workerLog( - "error", - `[med-notes-encryption-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[med-notes-encryption-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[med-notes-encryption-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.39.3 — Coach title + custom-metric note encryption backfill. Staggered - // one step past the note-encryption backfill so it starts after the boot - // storm rather than on the first poll. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeFreeTextEncryptionBackfill( - BOOT_BACKFILL_STAGGER_SECONDS * 6, - ); - if (error) { - workerLog( - "error", - `[free-text-encryption-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[free-text-encryption-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[free-text-encryption-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // Document vault — preview-thumbnail backfill. Finds every account holding a - // thumbnailable document (image or PDF) without a preview and enqueues one - // per-user backfill pass. Idempotent across reboots: a rendered thumbnail - // drops its document off the discovery predicate. - try { - const { enqueued } = await enqueueBootTimeThumbnailBackfill(); - workerLog( - "info", - `[document-thumbnail-backfill] boot discovery: enqueued=${enqueued}`, - ); - } catch (err) { - workerLog( - "error", - "[document-thumbnail-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.42 — the air-quality history backfill after an update, staggered past - // the boot storm. Accounts whose history is through are not offered. - try { - const { enqueued, skipped, error } = await enqueueBootTimeAirQualityHistory( - BOOT_BACKFILL_STAGGER_SECONDS * 7, - ); - if (error) { - workerLog( - "error", - `[environment-aq-history] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[environment-aq-history] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[environment-aq-history] boot discovery threw an unexpected error", - err, - ); - } - - // v1.42 — the fold repair, once per account; the purge below waits for it. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeMeasurementFoldRepair(); - if (error) { - workerLog( - "error", - `[measurement-fold-repair] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[measurement-fold-repair] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[measurement-fold-repair] boot discovery threw an unexpected error", - err, - ); - } - - // v1.42 — one compaction-tombstone purge run per boot. The run is - // self-limiting and idempotent, so a boot with no backlog costs one query. - // It sends nothing until the fold repair has finished an account. - try { - const { enqueued } = await enqueueBootTimeCompactionTombstonePurge(); - workerLog( - "info", - `[compaction-tombstone-purge] boot discovery: enqueued=${enqueued}`, - ); - } catch (err) { - workerLog( - "error", - "[compaction-tombstone-purge] boot discovery threw an unexpected error", - err, - ); - } - - // v1.37.7 (issue #659) — runtime GeoLite2 fetch. Only enqueues when a - // MAXMIND_LICENSE_KEY is set, egress is not disabled, and the offline tier - // is not already present — so a self-hoster who sets the key on the - // published image gets the databases fetched on the next worker boot without - // a rebuild or a manual mount. - try { - const { enqueued } = await enqueueGeolite2FetchBootDiscovery(); - workerLog("info", `[geolite2-fetch] boot discovery: enqueued=${enqueued}`); - } catch (err) { - workerLog( - "error", - "[geolite2-fetch] boot discovery threw an unexpected error", - err, - ); - } - - // v1.44 — one-time climb backfill for workouts stored with a track but no - // elevation (`climb-backfill.ts`). Detached: it reads one track at a time - // with a pause between batches and resumes from its cursor on the next - // boot, so it never holds the boot or the heap. A finished pass returns at - // once. - trackBackgroundTask( - (async () => { - const [{ prisma }, { runWorkoutClimbBackfill }] = await Promise.all([ - import("@/lib/db"), - import("@/lib/workouts/climb-backfill"), - ]); - const result = await runWorkoutClimbBackfill({ prisma }); - if (result.scanned > 0) { - workerLog( - "info", - `[workout-climb-backfill] scanned=${result.scanned} filled=${result.filled} done=${result.done}`, - ); - } - })().catch((err) => { - workerLog( - "error", - "[workout-climb-backfill] failed; the next boot resumes from the cursor", - err, - ); - }), - ); -} diff --git a/src/lib/jobs/reminder/register-reminders.ts b/src/lib/jobs/reminder/register-reminders.ts deleted file mode 100644 index 581a81358..000000000 --- a/src/lib/jobs/reminder/register-reminders.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Reminder-dispatch queue registrar. - * - * Owns the user-facing reminder ticks: the medication overdue check, the daily - * mood reminder, the cycle reminder (period-soon + period-start-confirm), the - * Vorsorge (measurement) reminder, and the eventful reminder-satisfy queue. - * - * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its - * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The - * `cycle-reminder-queue` and `measurement-reminder-queue` guards read THIS - * module. - * - * Retry policy: every queue here keeps the pg-boss default. These ticks are - * fan-out passes whose per-user failures never fail the job, so a job-level - * failure means the pass itself could not run — a transient the default two - * retries are the right answer to. The satisfy queue carries its own - * `retryLimit` at send time (see `reminder-satisfy.ts`). - */ -import { PgBoss } from "pg-boss"; -import { - REMINDER_SATISFY_QUEUE, - REMINDER_SATISFY_CONCURRENCY, - type ReminderSatisfyPayload, -} from "@/lib/jobs/reminder-satisfy"; -import { - createAndSchedule, - createAndWork, - type ScheduleEntry, -} from "./registrar-shared"; -import { - ReminderCheckPayload, - handleReminderCheck, -} from "./medication-reminder-check"; -import { - MoodReminderPayload, - CycleReminderPayload, - MeasurementReminderPayload, - handleMoodReminderCheck, - handleCycleReminderCheck, - handleMeasurementReminderCheck, - handleReminderSatisfy, -} from "./mood-cycle-checks"; - -const QUEUE_NAME = "medication-reminder-check"; - -const CHECK_INTERVAL_CRON = "*/15 * * * *"; // every 15 minutes -// v0.5.4 ios-coord — daily mood-reminder cron. -// -// Runs every 15 minutes so the handler can pick up users whose local -// time has just crossed 22:00 across any IANA timezone without having -// to schedule one cron entry per zone. The handler short-circuits when -// the user's local hour isn't 22, so the 15-min cadence translates to -// ~4 ticks-per-hour × 1 actual-dispatch-window-per-user = at most one -// push per user per day. Idempotency is enforced by the -// `MoodReminderDispatch` ledger inside the handler. - -const MOOD_REMINDER_QUEUE = "mood-reminder-check"; - -const MOOD_REMINDER_CRON = "*/15 * * * *"; -// v1.15 — daily cycle reminder cron (period-soon + period-start-confirm). -// -// Runs every 15 minutes for the same reason as the mood reminder: the -// handler short-circuits unless the candidate user's local time is the -// cycle-reminder hour (09:00), so the 15-min cadence picks up every IANA -// timezone crossing that hour without one cron entry per zone. At most one -// push per event per user per local day — the `push_attempts` ledger is the -// idempotency anchor inside the handler. - -const CYCLE_REMINDER_QUEUE = "cycle-reminder-check"; - -const CYCLE_REMINDER_CRON = "*/15 * * * *"; - -// v1.17.1 — every-15-min tick for Vorsorge (measurement) reminders. -// Same cadence + short-circuit shape as the mood / cycle reminders: the -// handler only fires a reminder whose `nextDueAt` is past AND whose local -// time is the reminder's `notifyHour`, so the 15-min cadence picks up -// every IANA timezone crossing that hour without one cron entry per zone. -// Dedup is the reminder's own `nextDueAt` advance — no ledger table. - -const MEASUREMENT_REMINDER_QUEUE = "measurement-reminder-check"; - -const MEASUREMENT_REMINDER_CRON = "*/15 * * * *"; - -const allQueues = [ - QUEUE_NAME, - // v0.5.4 ios-coord — mood-reminder cron tick. Same pg-boss v12 - // createQueue contract as the drain queue; without this entry the - // every-15-min schedule silently no-ops and the dispatcher never fires. - MOOD_REMINDER_QUEUE, - // v1.15 — cycle-reminder cron tick (period-soon + period-start-confirm). - // Without this entry the every-15-min schedule silently no-ops and the - // cycle dispatcher never fires (the v1.4.37 dead-queue class). - CYCLE_REMINDER_QUEUE, - // v1.17.1 — Vorsorge (measurement) reminder cron tick. Without this entry - // pg-boss never provisions the queue and the every-15-min schedule - // silently no-ops (the v1.4.37 dead-queue class). - MEASUREMENT_REMINDER_QUEUE, - // v1.18.1 — eventful Vorsorge satisfaction. No cron of its own (the - // 15-min measurement-reminder check is the safety-net); enqueued by the - // ingest paths. Must still be registered here or the worker binding - // below never provisions the queue (v1.4.37 dead-queue lesson). - REMINDER_SATISFY_QUEUE, -]; - -const schedules: ScheduleEntry[] = [ - [QUEUE_NAME, CHECK_INTERVAL_CRON], - // v0.5.4 ios-coord — every-15-min tick for the daily mood reminder. - // The handler short-circuits unless the candidate user's local - // time is the 22:00 hour, so the cron costs ~one user-row scan - // per tick for the entire opted-in cohort. - [MOOD_REMINDER_QUEUE, MOOD_REMINDER_CRON], - // v1.15 — every-15-min tick for the daily cycle reminder. The handler - // short-circuits unless the candidate user's local time is the 09:00 - // hour, so the cron costs ~one prediction-row scan per tick for the - // opted-in cohort. - [CYCLE_REMINDER_QUEUE, CYCLE_REMINDER_CRON], - // v1.17.1 — every-15-min tick for the Vorsorge (measurement) reminder. - // The handler short-circuits unless a reminder is past-due AND the - // user's local time matches the reminder's notifyHour, so the cron - // costs ~one reminder-row scan per tick for the active cohort. - [MEASUREMENT_REMINDER_QUEUE, MEASUREMENT_REMINDER_CRON], -]; - -/** - * Register every reminder-dispatch queue. Returns the queue names created (for - * the boot-level aggregate assertion). - */ -export async function registerReminderQueues( - boss: PgBoss, -): Promise { - await createAndSchedule(boss, allQueues, schedules); - - await createAndWork( - boss, - QUEUE_NAME, - { localConcurrency: 1 }, - handleReminderCheck, - ); - // v0.5.4 ios-coord — single-flight worker. localConcurrency=1 keeps - // two reminder ticks from interleaving against the same user row; - // the dedup ledger would still save us, but skipping the race here - // avoids spurious P2002 errors in the wide-event log. - await createAndWork( - boss, - MOOD_REMINDER_QUEUE, - { localConcurrency: 1 }, - handleMoodReminderCheck, - ); - // v1.15 — single-flight cycle-reminder worker. localConcurrency=1 keeps - // two ticks from racing the fire-and-forget `push_attempts` ledger that - // anchors the per-day idempotency, exactly like the mood-reminder worker. - await createAndWork( - boss, - CYCLE_REMINDER_QUEUE, - { localConcurrency: 1 }, - handleCycleReminderCheck, - ); - // v1.17.1 — single-flight Vorsorge (measurement) reminder worker. - // localConcurrency=1 keeps two ticks from racing the `nextDueAt` - // advance that anchors the per-cycle idempotency, exactly like the - // mood / cycle reminder workers. - await createAndWork( - boss, - MEASUREMENT_REMINDER_QUEUE, - { localConcurrency: 1 }, - handleMeasurementReminderCheck, - ); - // v1.18.1 — eventful Vorsorge satisfaction. Resolves a user's reminders - // against their just-landed measurement / lab. Read-heavy on the user's - // own data; the same small concurrency budget as PR detection. - await createAndWork( - boss, - REMINDER_SATISFY_QUEUE, - { localConcurrency: REMINDER_SATISFY_CONCURRENCY }, - handleReminderSatisfy, - ); - - return allQueues; -} diff --git a/src/lib/jobs/reminder/register-rollup.ts b/src/lib/jobs/reminder/register-rollup.ts deleted file mode 100644 index f4ed00269..000000000 --- a/src/lib/jobs/reminder/register-rollup.ts +++ /dev/null @@ -1,1156 +0,0 @@ -/** - * Rollup / consolidation queue registrar. - * - * Owns the pre-aggregation + consolidation tier: measurement-rollup recompute - * + boot-backfill, mood-rollup recompute + boot-backfill, medication-compliance - * boot-backfill, legacy step consolidation, daily-mean consolidation, dense - * intra-day retention, and the nightly per-sample cumulative drain (which also - * folds the daily-mean + dense-retention passes onto the same tick). Includes - * the boot-time discovery enqueues for the self-converging backfills. - * - * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its - * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The - * `drain-cumulative-queue`, `mean-consolidation-queue`, and - * `stress-strain-retention-queue` (dense-retention facet) guards read THIS - * module. - */ -import { PgBoss } from "pg-boss"; -import { reportWorkerError } from "@/lib/jobs/report-worker-error"; -import { recordError } from "@/lib/jobs/worker-status"; -import { - ROLLUP_FULL_BACKFILL_QUEUE, - ROLLUP_FULL_BACKFILL_CONCURRENCY, - ROLLUP_RECOMPUTE_QUEUE, - ROLLUP_RECOMPUTE_CONCURRENCY, - enqueueBootTimeRollupBackfill, - recomputeUserRollups, - type RollupFullBackfillPayload, - type RollupRecomputePayload, -} from "@/lib/rollups/measurement-rollups"; -import { - MOOD_ROLLUP_FULL_BACKFILL_QUEUE, - MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY, - MOOD_ROLLUP_RECOMPUTE_QUEUE, - MOOD_ROLLUP_RECOMPUTE_CONCURRENCY, - enqueueBootTimeMoodRollupBackfill, - recomputeUserMoodRollups, - type MoodRollupFullBackfillPayload, - type MoodRollupRecomputePayload, -} from "@/lib/rollups/mood-rollups"; -import { - SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE, - SUPPLEMENT_NUTRIENT_RECOMPUTE_CONCURRENCY, - runSupplementNutrientRecompute, - type SupplementNutrientRecomputePayload, -} from "@/lib/medications/ingredients/recompute-job"; -import { - drainPerSampleCumulative, - DRAIN_CUMULATIVE_CUTOFF_HOURS, -} from "@/lib/measurements/drain-per-sample-cumulative"; -import { - consolidateDailyMean, - MEAN_CONSOLIDATION_CUTOFF_HOURS, -} from "@/lib/measurements/consolidate-daily-mean"; -import { - STEP_CONSOLIDATION_QUEUE, - STEP_CONSOLIDATION_CONCURRENCY, - runStepConsolidationForUser, - enqueueBootTimeStepConsolidation, - type StepConsolidationPayload, -} from "@/lib/jobs/step-consolidation"; -import { - STEP_CONSOLIDATION_REPAIR_QUEUE, - STEP_CONSOLIDATION_REPAIR_CONCURRENCY, - runStepConsolidationRepairForUser, - enqueueBootTimeStepConsolidationRepair, - type StepConsolidationRepairPayload, -} from "@/lib/jobs/step-consolidation-repair"; -import { - CUMULATIVE_PR_REDERIVE_QUEUE, - CUMULATIVE_PR_REDERIVE_CONCURRENCY, - runCumulativePrRederivationForUser, - enqueueBootTimeCumulativePrRederivation, - type CumulativePrRederivePayload, -} from "@/lib/personal-records/cumulative-pr-rederivation"; -import { - MEAN_CONSOLIDATION_QUEUE, - MEAN_CONSOLIDATION_CONCURRENCY, - runMeanConsolidationForUser, - enqueueBootTimeMeanConsolidation, - type MeanConsolidationPayload, -} from "@/lib/jobs/mean-consolidation"; -import { - DENSE_INTRADAY_RETENTION_QUEUE, - DENSE_INTRADAY_RETENTION_CONCURRENCY, - runDenseIntradayRetentionForUser, - enqueueBootTimeDenseIntradayRetention, - DENSE_INTRADAY_RETENTION_ENABLED, - type DenseIntradayRetentionPayload, -} from "@/lib/jobs/dense-intraday-retention"; -import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention"; -import { - DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, - DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY, - runDenseIntradayHourlyRebuildForUser, - enqueueBootTimeDenseIntradayHourlyRebuild, - type DenseIntradayHourlyRebuildPayload, -} from "@/lib/jobs/dense-intraday-hourly-rebuild"; -import { - getWorkerPrisma, - workerLog, - BOOT_BACKFILL_STAGGER_SECONDS, -} from "./shared"; -import { heavyLane } from "@/lib/jobs/heavy-lane"; -import { lockedPass } from "@/lib/jobs/long-pass"; -import { - createAndSchedule, - createAndWork, - cronIsTheRetry, - retireQueues, - type QueuePolicyTable, - type ScheduleEntry, -} from "./registrar-shared"; -import { jobDone, jobFailed } from "@/lib/jobs/job-outcome"; -import { jobBudget } from "@/lib/jobs/job-budget"; -import { - DRAIN_CONTINUATION_DELAY_SECONDS, - nextDrainContinuation, -} from "@/lib/jobs/drain-continuation"; -// v1.4.37 — nightly drain of per-sample APPLE_HEALTH cumulative rows. -// Collapses each user × cumulative-type × calendar-day bucket into one -// `stats:…` row so the list view stops painting hundreds of step chunks per -// day. 03:45 Europe/Berlin slots in between the 03:15 audit-log cleanup and -// the 04:00 feedback aggregator. The 36-hour grace window keeps today + the -// trailing watch-sync window intact for real-time visibility; only -// completed-and-stable days fall to the drain. - -const DRAIN_CUMULATIVE_QUEUE = "drain-per-sample-cumulative"; - -const DRAIN_CUMULATIVE_CRON = "45 3 * * *"; - -interface DrainCumulativePayload { - triggeredAt?: string; - /** - * How many times this tick has already handed itself on. Absent on the - * cron's own job. See `nextDrainContinuation`. - */ - continuation?: number; -} - -const allQueues = [ - ROLLUP_RECOMPUTE_QUEUE, - ROLLUP_FULL_BACKFILL_QUEUE, - // v1.4.39 — per-bucket WEEK/MONTH/YEAR fold queue for the - // mood rollup tier. The DAY pass runs inline in the write hook; - // these enqueue paths cover the cross-granularity buckets the - // worker materialises off the request path. - MOOD_ROLLUP_RECOMPUTE_QUEUE, - // v1.4.39 — boot-time fold queue for the mood rollup tier. - MOOD_ROLLUP_FULL_BACKFILL_QUEUE, - // v1.5.6 — boot-time legacy step consolidation. Discovery enqueues - // one job per user still holding live pre-v1.5.0 granular step rows. - STEP_CONSOLIDATION_QUEUE, - // v1.28.37 — one-shot repair for the provider step rows the pre-fix - // consolidation swept. Boot-discovery driven, self-converging. - STEP_CONSOLIDATION_REPAIR_QUEUE, - // v1.30.3 — one-shot repair for the cumulative-type PersonalRecord rows - // the pre-fix multi-source SUM inflated. Boot-discovery driven, - // self-converging (QA F4). - CUMULATIVE_PR_REDERIVE_QUEUE, - // v1.7.0 — daily-mean consolidation for high-frequency spot HealthKit - // metrics (walking speed/step length, respiratory rate, audio exposure). - MEAN_CONSOLIDATION_QUEUE, - // v1.10.0 — computed scores (WX-E). Dense intra-day retention drain of - // daytime HRV / HR samples (per-user backfill queue, boot-discovery - // driven like mean-consolidation; the steady-state nightly walk folds - // onto the drain-cumulative tick). - DENSE_INTRADAY_RETENTION_QUEUE, - // v1.28.31 — one-shot hourly history rebuild for pre-hourly folded - // dense-tier days. Boot-discovery driven and self-converging (a rebuilt - // day's retired daily row drops it from the discovery pairing). - DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, - // v1.4.37 — explicit createQueue is required before the nightly - // schedule below registers (pg-boss v12 contract). Without this entry the - // drain schedule silently no-ops and the per-sample APPLE_HEALTH rows - // never collapse. - DRAIN_CUMULATIVE_QUEUE, - // v1.43 — one account's supplement doses recomputed into the day's - // micronutrients after a change that spans more than one day. - SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE, -]; - -const schedules: ScheduleEntry[] = [ - // v1.4.37 — nightly fold of per-sample APPLE_HEALTH cumulative - // rows into one row per day per type. Slots between the - // audit-log cleanup (03:15) and the feedback aggregator (04:00). - [DRAIN_CUMULATIVE_QUEUE, DRAIN_CUMULATIVE_CRON, cronIsTheRetry], -]; - -/** - * De-duplication policy per rollup queue. Until this table existed every queue - * here ran under pg-boss's `standard` policy, under which no unique index - * constrains `singleton_key` — so the `singletonKey` each enqueue site passes - * was inert and de-duplicated nothing. - * - * The split below is deliberate and the two halves are NOT interchangeable. - */ -const queuePolicies: QueuePolicyTable = { - // `short`, not `exclusive`. A recompute reads the bucket's live rows when it - // STARTS, so collapsing sends that pile up while an identical job is still - // queued is safe — the queued job will observe every write that landed in the - // meantime. This is what kills the storm: a sync posting a few hundred batch - // requests fanned out three jobs per type-day per request, thousands of jobs - // re-deriving byte-identical rows. Under `short` they collapse to one queued - // job per bucket. `exclusive` would be wrong here: it would also suppress a - // send issued after the recompute had already started reading, stranding the - // newer measurement in a stale bucket until the nightly pass. - [ROLLUP_RECOMPUTE_QUEUE]: { - policy: "short", - reason: - "Burst coalescing for the write-path fan-out. The handler re-reads the bucket at run time, so collapsing queued duplicates cannot strand a write; exclusive could.", - }, - [MOOD_ROLLUP_RECOMPUTE_QUEUE]: { - policy: "short", - reason: - "Same shape as the measurement recompute: re-reads the bucket at run time, so only queued duplicates may be collapsed.", - }, - [SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE]: { - policy: "short", - reason: - "Per-account recompute of supplement nutrient rows; the pass reads the live doses and composition when it starts, so a queued duplicate may be collapsed, while a send during a running pass must still queue the next one.", - }, - - // The rest are per-user, self-converging boot/discovery backfills. A second - // concurrent run is pure duplicated work, and the discovery predicate - // re-enqueues on the next boot or cron tick while work remains outstanding — - // so `exclusive` (queued OR active OR awaiting retry) can never lose work it - // does not re-offer. `short` would be too weak: these passes run for minutes - // to hours, and the observed failure was precisely a worker restarting - // mid-pass and appending another identical full-history job per restart. - [ROLLUP_FULL_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user full-history backfill; long-running. Boot discovery re-enqueues while coverage is still missing, so suppressing a duplicate of an ACTIVE pass is safe.", - }, - [MOOD_ROLLUP_FULL_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user mood backfill; boot-discovery driven and self-converging on rollup coverage.", - }, - [STEP_CONSOLIDATION_QUEUE]: { - policy: "exclusive", - reason: - "Per-user legacy step consolidation; discovery drops the user once no live granular rows remain.", - }, - [STEP_CONSOLIDATION_REPAIR_QUEUE]: { - policy: "exclusive", - reason: - "Per-user one-shot repair; discovery drops the user once the affected provider rows are repaired.", - }, - [CUMULATIVE_PR_REDERIVE_QUEUE]: { - policy: "exclusive", - reason: - "Per-user one-shot re-derivation; discovery drops the user once the inflated records are rewritten.", - }, - [MEAN_CONSOLIDATION_QUEUE]: { - policy: "exclusive", - reason: - "Per-user daily-mean consolidation; discovery drops the user once the spot rows are folded.", - }, - [DENSE_INTRADAY_RETENTION_QUEUE]: { - policy: "exclusive", - reason: - "Per-user dense-tier drain; the multi-hour pass on a heavy account is exactly the case where a restart used to append a duplicate full-history job.", - }, - [DENSE_INTRADAY_HOURLY_REBUILD_QUEUE]: { - policy: "exclusive", - reason: - "Per-user one-shot hourly rebuild; discovery drops the day once its retired daily row is gone.", - }, - - // DRAIN_CUMULATIVE_QUEUE is deliberately absent: it is a keyless nightly cron - // tick with no singletonKey on any send, so a policy would only constrain the - // empty key and buy nothing. -}; - -/** - * The boot backfill of the per-day medication-compliance rollup, whose table - * had no reader left and was dropped. Its queue, and any job still waiting in - * it, is removed on an instance that still has it. - */ -const RETIRED_ROLLUP_QUEUES = ["medication-compliance-full-backfill"] as const; - -/** - * Register every rollup / consolidation queue. Returns the queue names created - * (for the boot-level aggregate assertion). - */ -export async function registerRollupQueues( - boss: PgBoss, -): Promise { - await createAndSchedule(boss, allQueues, schedules, queuePolicies); - await retireQueues(boss, RETIRED_ROLLUP_QUEUES); - - // v1.5.0 — persistent measurement rollup worker. Folds the - // WEEK / MONTH / YEAR buckets that the write-path hooks enqueue; - // the DAY bucket is already recomputed synchronously by the hook - // itself. Concurrency-2 keeps two recomputes in flight without - // crowding the dashboard request pool. - await createAndWork( - boss, - ROLLUP_RECOMPUTE_QUEUE, - { localConcurrency: ROLLUP_RECOMPUTE_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - const payload = job.data; - await recomputeUserRollups(payload.userId, { - types: [payload.type], - granularities: [payload.granularity], - from: new Date(payload.from), - to: new Date(payload.to), - }); - } - // No per-bucket isolation here on purpose: a recompute that throws - // fails the job, so reaching this line means every bucket in the batch - // was folded. - return jobDone({ buckets_recomputed: jobs.length }); - }, - ); - - // v1.4.35.1 — boot-time full-fold worker. The boot enqueue helper - // below sends one job per uncovered user; this handler runs the - // full `recomputeUserRollups` against the default 5-year window - // across every granularity. Serial concurrency so the populator - // never crowds the dashboard request pool. - await createAndWork( - boss, - ROLLUP_FULL_BACKFILL_QUEUE, - { localConcurrency: ROLLUP_FULL_BACKFILL_CONCURRENCY }, - // The lock sits outside the lane: a retry pg-boss starts after the expiry - // while the first fold still runs finds the account's lock and settles, - // instead of waiting in the lane and folding the account again. - lockedPass( - ROLLUP_FULL_BACKFILL_QUEUE, - (job) => job.data.userId, - heavyLane(ROLLUP_FULL_BACKFILL_QUEUE, async (jobs) => { - const shouldStop = jobBudget(jobs); - let rowsUpsertedTotal = 0; - for (const job of jobs) { - const { userId } = job.data; - const { rowsUpserted, durationMs, stopped } = - await recomputeUserRollups(userId, { shouldStop }); - if (stopped) { - // Each granularity committed on its own; the retry folds again. - throw new Error( - `rollup-full-backfill for user ${userId} stopped on its job budget; the retry folds the rest`, - ); - } - rowsUpsertedTotal += rowsUpserted; - workerLog( - "info", - `[rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, - ); - } - return jobDone({ - users: jobs.length, - rows_upserted: rowsUpsertedTotal, - }); - }), - ), - ); - - // v1.5.6 — legacy step consolidation worker. The boot enqueue helper - // below sends one job per user still holding live pre-v1.5.0 granular - // step rows; this handler collapses them into one daily-total row per - // calendar day and soft-deletes the originals. Serial concurrency so - // the populator never crowds the dashboard request pool. - await createAndWork( - boss, - STEP_CONSOLIDATION_QUEUE, - { localConcurrency: STEP_CONSOLIDATION_CONCURRENCY }, - async (jobs) => { - let daysTotal = 0; - let legacyRowsTotal = 0; - for (const job of jobs) { - const { userId } = job.data; - const { daysConsolidated, legacyRowsSoftDeleted } = - await runStepConsolidationForUser(userId); - daysTotal += daysConsolidated; - legacyRowsTotal += legacyRowsSoftDeleted; - workerLog( - "info", - `[step-consolidation] user=${userId} days=${daysConsolidated} legacyRowsSoftDeleted=${legacyRowsSoftDeleted}`, - ); - } - return jobDone({ - users: jobs.length, - days_consolidated: daysTotal, - legacy_rows_soft_deleted: legacyRowsTotal, - }); - }, - ); - - // v1.28.37 — step-consolidation repair worker. Resurrects the - // GOOGLE_HEALTH / FITBIT daily-total step rows the pre-fix consolidation - // swept and removes the shadow MANUAL totals it minted. Serial - // concurrency so the repair never crowds the dashboard request pool. - await createAndWork( - boss, - STEP_CONSOLIDATION_REPAIR_QUEUE, - { localConcurrency: STEP_CONSOLIDATION_REPAIR_CONCURRENCY }, - async (jobs) => { - let rowsResurrected = 0; - let wedgeSkipped = 0; - let manualMintsRemoved = 0; - let daysRecomputed = 0; - let failures = 0; - for (const job of jobs) { - const { userId } = job.data; - const summary = await runStepConsolidationRepairForUser(userId); - rowsResurrected += summary.rowsResurrected; - wedgeSkipped += summary.wedgeSkipped; - manualMintsRemoved += summary.manualMintsRemoved; - daysRecomputed += summary.daysRecomputed; - // The repair isolates its own per-day failures and reports them as a - // count, so they ride out as a fact rather than failing the job. - failures += summary.failures; - workerLog( - "info", - `[step-consolidation-repair] user=${userId} resurrected=${summary.rowsResurrected} wedgeSkipped=${summary.wedgeSkipped} manualMintsRemoved=${summary.manualMintsRemoved} daysRecomputed=${summary.daysRecomputed} failures=${summary.failures}`, - ); - } - return jobDone({ - users: jobs.length, - rows_resurrected: rowsResurrected, - wedge_skipped: wedgeSkipped, - manual_mints_removed: manualMintsRemoved, - days_recomputed: daysRecomputed, - failures, - }); - }, - ); - - // v1.43 — supplement nutrient recompute. One account per job, serial. - await createAndWork( - boss, - SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE, - { localConcurrency: SUPPLEMENT_NUTRIENT_RECOMPUTE_CONCURRENCY }, - async (jobs) => { - let rowsWritten = 0; - for (const job of jobs) { - const result = await runSupplementNutrientRecompute(job.data); - rowsWritten += result.rowsWritten; - } - return jobDone({ users: jobs.length, rows_written: rowsWritten }); - }, - ); - - // v1.30.3 — cumulative-PersonalRecord rederivation worker (QA F4). The - // boot enqueue helper below sends one job per user still holding a - // suspect pre-fix inflated cumulative-type record; this handler deletes - // it and re-runs detection silently so the honest re-derived best takes - // its place. Serial concurrency so the repair never crowds the - // dashboard request pool. - await createAndWork( - boss, - CUMULATIVE_PR_REDERIVE_QUEUE, - { localConcurrency: CUMULATIVE_PR_REDERIVE_CONCURRENCY }, - async (jobs) => { - let rowsDeleted = 0; - let rowsReinserted = 0; - for (const job of jobs) { - const { userId } = job.data; - const summary = await runCumulativePrRederivationForUser(userId); - rowsDeleted += summary.rowsDeleted; - rowsReinserted += summary.rowsReinserted; - workerLog( - "info", - `[cumulative-pr-rederive] user=${userId} rowsDeleted=${summary.rowsDeleted} rowsReinserted=${summary.rowsReinserted}`, - ); - } - return jobDone({ - users: jobs.length, - rows_deleted: rowsDeleted, - rows_reinserted: rowsReinserted, - }); - }, - ); - - // v1.7.0 — daily-mean consolidation worker. The boot enqueue helper - // below sends one job per user holding live per-sample high-frequency - // mean-type rows; this handler collapses each completed day to its - // mean and soft-deletes the originals. Serial concurrency so the - // populator never crowds the dashboard request pool. - await createAndWork( - boss, - MEAN_CONSOLIDATION_QUEUE, - { localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY }, - async (jobs) => { - const shouldStop = jobBudget(jobs); - let daysTotal = 0; - let perSampleRowsTotal = 0; - let stoppedEarly = false; - for (const job of jobs) { - const { userId } = job.data; - const summary = await runMeanConsolidationForUser(userId, shouldStop); - const { daysConsolidated, perSampleRowsSoftDeleted } = summary; - stoppedEarly ||= summary.stoppedEarly; - daysTotal += daysConsolidated; - perSampleRowsTotal += perSampleRowsSoftDeleted; - workerLog( - "info", - `[mean-consolidation] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted}`, - ); - } - // Out of budget is a completed run, not a failed one: the folded days - // are committed, and the nightly tick carries on from the first day - // this run did not reach. - return jobDone({ - users: jobs.length, - days_consolidated: daysTotal, - per_sample_rows_soft_deleted: perSampleRowsTotal, - stopped_early: stoppedEarly, - }); - }, - ); - - // v1.10.0 WX-E — dense intra-day retention per-user backfill worker. The - // boot enqueue helper below sends one job per user holding live per-sample - // dense-tier (HRV / HR / SpO2) rows older than the retention window; this - // handler folds those out-of-window samples to hourly means (v1.28.31) and - // soft-deletes the originals, keeping the in-window intra-day shape intact - // for the Stress engine. Serial concurrency so the backfill never crowds - // the request pool. - await createAndWork( - boss, - DENSE_INTRADAY_RETENTION_QUEUE, - { localConcurrency: DENSE_INTRADAY_RETENTION_CONCURRENCY }, - async (jobs) => { - const shouldStop = jobBudget(jobs); - let daysTotal = 0; - let perSampleRowsTotal = 0; - let derivedRestingRowsTotal = 0; - let stoppedEarly = false; - for (const job of jobs) { - const { userId } = job.data; - try { - const summary = await runDenseIntradayRetentionForUser( - userId, - shouldStop, - ); - const { - daysConsolidated, - perSampleRowsSoftDeleted, - derivedRestingRowsUpserted, - } = summary; - stoppedEarly ||= summary.stoppedEarly; - daysTotal += daysConsolidated; - perSampleRowsTotal += perSampleRowsSoftDeleted; - derivedRestingRowsTotal += derivedRestingRowsUpserted; - workerLog( - "info", - `[dense-intraday-retention] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted} derivedRestingRowsUpserted=${derivedRestingRowsUpserted}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[dense-intraday-retention] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - users: jobs.length, - days_consolidated: daysTotal, - per_sample_rows_soft_deleted: perSampleRowsTotal, - derived_resting_rows_upserted: derivedRestingRowsTotal, - // Out of budget completes the job: the folded days are committed and - // the nightly tick, which hands itself on while work remains, - // continues from the first day this run did not reach. - stopped_early: stoppedEarly, - }); - }, - ); - - // v1.28.31 — one-shot hourly history rebuild worker. The boot enqueue - // helper below sends one job per user still holding a pre-hourly daily - // `stats:` row paired with tombstoned raw rows; this handler reconstructs - // hourly means from the tombstones and retires the daily row atomically. - // Self-converging: a rebuilt day drops out of the discovery pairing, so - // the pass runs once per install. Serial concurrency so the rebuild never - // crowds the request pool. - await createAndWork( - boss, - DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, - { localConcurrency: DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY }, - async (jobs) => { - const shouldStop = jobBudget(jobs); - let stoppedEarly = false; - let daysRebuiltTotal = 0; - let hourlyRowsTotal = 0; - let dailyRowsRetiredTotal = 0; - let daysSkippedTotal = 0; - for (const job of jobs) { - const { userId } = job.data; - try { - const summary = await runDenseIntradayHourlyRebuildForUser( - userId, - shouldStop, - ); - const { - daysRebuilt, - hourlyRowsUpserted, - dailyRowsRetired, - daysSkippedNoTombstones, - } = summary; - stoppedEarly ||= summary.stoppedEarly; - daysRebuiltTotal += daysRebuilt; - hourlyRowsTotal += hourlyRowsUpserted; - dailyRowsRetiredTotal += dailyRowsRetired; - daysSkippedTotal += daysSkippedNoTombstones; - workerLog( - "info", - `[dense-intraday-hourly-rebuild] user=${userId} daysRebuilt=${daysRebuilt} hourlyRowsUpserted=${hourlyRowsUpserted} dailyRowsRetired=${dailyRowsRetired} daysSkippedNoTombstones=${daysSkippedNoTombstones}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[dense-intraday-hourly-rebuild] user=${userId} failed`, - err, - ); - throw err; - } - } - const did = { - users: jobs.length, - stopped_early: stoppedEarly, - days_rebuilt: daysRebuiltTotal, - hourly_rows_upserted: hourlyRowsTotal, - daily_rows_retired: dailyRowsRetiredTotal, - days_skipped_no_tombstones: daysSkippedTotal, - }; - // Out of budget: fail so pg-boss retries (the boot send allows three - // retries), and the retry starts at the first day not yet rebuilt. The - // queue is `exclusive`, so the job cannot hand itself on while active. - if (stoppedEarly) { - return jobFailed( - "stopped before every day was rebuilt; the retry resumes", - undefined, - did, - ); - } - return jobDone(did); - }, - ); - - // v1.4.39 — mood-rollup per-bucket worker. Folds the - // WEEK / MONTH / YEAR buckets that the mood-entry write hooks - // enqueue; the DAY bucket runs synchronously in the hook itself. - // No current read path consumes these buckets — they exist so a - // future cross-granularity reader can ship without a backfill - // step. Concurrency-2 mirrors the measurement-rollup worker. - await createAndWork( - boss, - MOOD_ROLLUP_RECOMPUTE_QUEUE, - { localConcurrency: MOOD_ROLLUP_RECOMPUTE_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - const payload = job.data; - await recomputeUserMoodRollups(payload.userId, { - granularities: [payload.granularity], - from: new Date(payload.from), - to: new Date(payload.to), - }); - } - return jobDone({ buckets_recomputed: jobs.length }); - }, - ); - - // v1.4.39 — mood-rollup boot-time fold worker. The boot - // discovery helper below sends one job per user with mood entries - // but zero rollup rows; this handler folds the full 5-year window - // across every granularity. Concurrency-1 so the populator never - // crowds the request pool. - await createAndWork( - boss, - MOOD_ROLLUP_FULL_BACKFILL_QUEUE, - { localConcurrency: MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY }, - async (jobs) => { - let rowsUpsertedTotal = 0; - for (const job of jobs) { - const { userId } = job.data; - try { - // v1.32.12 — delete-then-refold. Empties the user's rollup - // rows before refolding so the boot backfill is self-cleaning - // against any row minted under an older keying (e.g. a - // rolling-deploy old-container write during the swap window). - // The brief empty window serves correct data via the live - // `entry.date` fallback. - const { rowsUpserted, durationMs } = await recomputeUserMoodRollups( - userId, - { replace: true }, - ); - rowsUpsertedTotal += rowsUpserted; - workerLog( - "info", - `[mood-rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[mood-rollup-full-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - users: jobs.length, - rows_upserted: rowsUpsertedTotal, - }); - }, - ); - - // v1.4.37 — nightly drain worker. Walks every user × cumulative - // type and folds per-sample APPLE_HEALTH rows older than the cutoff - // into one `stats:…` row per calendar day. Idempotent — a second run - // collapses zero buckets once every day is in the `stats:` shape. - // Concurrency-1 so the drain never crowds the dashboard request pool - // and a long backfill on the maintainer's account (300 k+ measurement rows) - // stays a single sequential walk. - await createAndWork( - boss, - DRAIN_CUMULATIVE_QUEUE, - { localConcurrency: 1 }, - async (jobs) => { - // One budget shared by all three passes, run in turn. - const shouldStop = jobBudget(jobs); - let stoppedEarly = false; - // The three drains below are independent passes sharing one nightly - // tick, so one failing pass must not stop the other two from running. - // The first failure is held back and reported once the tick is over — - // that keeps the existing order and still fails the job, which is what - // the swallowed errors never did. - let firstFailure: { reason: string; cause: unknown } | undefined; - let passesFailed = 0; - let bucketsCollapsed = 0; - let perSampleRowsDeleted = 0; - let meanDaysConsolidated = 0; - let meanRowsSoftDeleted = 0; - let denseDaysConsolidated = 0; - let denseRowsSoftDeleted = 0; - let denseSkipped = false; - - for (const job of jobs) { - try { - const summary = await drainPerSampleCumulative(getWorkerPrisma(), { - dryRun: false, - cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS, - shouldStop, - log: (line) => workerLog("info", line), - }); - stoppedEarly ||= summary.stoppedEarly; - bucketsCollapsed += summary.totals.bucketsCollapsed; - perSampleRowsDeleted += summary.totals.perSampleRowsDeleted; - workerLog( - "info", - `[drain-cumulative] triggeredAt=${job.data.triggeredAt} usersScanned=${summary.totals.usersScanned} bucketsCollapsed=${summary.totals.bucketsCollapsed} perSampleRowsDeleted=${summary.totals.perSampleRowsDeleted} dailyRowsUpserted=${summary.totals.dailyRowsUpserted}`, - ); - } catch (err) { - recordError(); - await reportWorkerError("drain-cumulative", err); - passesFailed++; - firstFailure ??= { reason: "cumulative drain failed", cause: err }; - } - - // v1.8.5 — fold the daily-MEAN drain onto the same nightly tick as - // the cumulative (SUM) drain. The mean drain was previously - // boot-discovery only, so between worker reboots new high-frequency - // spot samples (walking speed, respiratory rate, gait/mobility, - // audio exposure) accumulated raw. Running both passes on one - // concurrency-1 cron keeps the maintenance window a single - // sequential walk and never crowds the request pool. The global - // (no `userId`) signature drains every user; the 36-hour grace - // cutoff keeps today's in-flight watch syncs raw. Boot discovery - // stays as the back-fill for accounts that accumulated raw rows - // before this cron shipped. - try { - const meanSummary = await consolidateDailyMean(getWorkerPrisma(), { - dryRun: false, - cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS, - shouldStop, - log: (line) => workerLog("info", line), - }); - stoppedEarly ||= meanSummary.stoppedEarly; - meanDaysConsolidated += meanSummary.totals.daysConsolidated; - meanRowsSoftDeleted += meanSummary.totals.perSampleRowsSoftDeleted; - workerLog( - "info", - `[mean-consolidation] triggeredAt=${job.data.triggeredAt} usersScanned=${meanSummary.totals.usersScanned} daysConsolidated=${meanSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${meanSummary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${meanSummary.totals.dailyRowsUpserted} daysFailed=${meanSummary.totals.daysFailed}`, - ); - } catch (err) { - recordError(); - await reportWorkerError("mean-consolidation", err, { - tick: "nightly", - }); - passesFailed++; - firstFailure ??= { reason: "mean consolidation failed", cause: err }; - } - - // v1.10.0 WX-E — fold the dense intra-day retention drain onto the - // same nightly tick. Unlike the daily-mean drain, this scopes to the - // dense-tier types (HEART_RATE_VARIABILITY, PULSE) and keeps the last - // DENSE_INTRADAY_RETENTION_DAYS of raw per-sample rows so the Stress - // engine still sees the intra-day SDNN shape; only out-of-window - // samples fold to a daily mean. These two types are NEVER in the - // destructive HIGH_FREQUENCY_MEAN_TYPES allowlist — the drain - // exemption the intra-day shape depends on. The global (no `userId`) - // signature drains every user. Boot discovery (below) back-fills - // accounts that accumulated out-of-window raw rows before this - // shipped. - try { - if (!DENSE_INTRADAY_RETENTION_ENABLED) { - denseSkipped = true; - workerLog( - "info", - "[dense-intraday-retention] disabled via DENSE_INTRADAY_RETENTION_ENABLED — skipping the nightly walk (operator kill-switch)", - ); - } else { - const denseSummary = await runDenseIntradayRetention( - getWorkerPrisma(), - { - dryRun: false, - shouldStop, - log: (line) => workerLog("info", line), - }, - ); - stoppedEarly ||= denseSummary.stoppedEarly; - denseDaysConsolidated += denseSummary.totals.daysConsolidated; - denseRowsSoftDeleted += - denseSummary.totals.perSampleRowsSoftDeleted; - workerLog( - "info", - `[dense-intraday-retention] triggeredAt=${job.data.triggeredAt} usersScanned=${denseSummary.totals.usersScanned} daysConsolidated=${denseSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${denseSummary.totals.perSampleRowsSoftDeleted} hourlyRowsUpserted=${denseSummary.totals.hourlyRowsUpserted} dailyRowsRetired=${denseSummary.totals.dailyRowsRetired}`, - ); - } - } catch (err) { - recordError(); - await reportWorkerError("dense-intraday-retention", err, { - tick: "nightly", - }); - passesFailed++; - firstFailure ??= { reason: "dense retention failed", cause: err }; - } - } - - // Hand the remaining work to a follow-up job when this one ran out of - // budget (bounded; see `nextDrainContinuation`). - const continuation = Math.max( - 0, - ...jobs.map((job) => job.data.continuation ?? 0), - ); - const next = nextDrainContinuation({ - stoppedEarly, - daysFolded: - bucketsCollapsed + meanDaysConsolidated + denseDaysConsolidated, - continuation, - }); - let continued = false; - if (next !== null) { - try { - const followUp: DrainCumulativePayload = { - triggeredAt: new Date().toISOString(), - continuation: next, - }; - continued = - (await boss.send(DRAIN_CUMULATIVE_QUEUE, followUp, { - ...cronIsTheRetry, - startAfter: DRAIN_CONTINUATION_DELAY_SECONDS, - })) !== null; - } catch (err) { - // The next nightly tick picks the work up regardless; a lost - // follow-up only delays it. - workerLog("error", "[drain-cumulative] follow-up send failed", err); - } - } - - const did = { - stopped_early: stoppedEarly, - continuation, - continued, - passes_failed: passesFailed, - buckets_collapsed: bucketsCollapsed, - per_sample_rows_deleted: perSampleRowsDeleted, - mean_days_consolidated: meanDaysConsolidated, - mean_rows_soft_deleted: meanRowsSoftDeleted, - dense_days_consolidated: denseDaysConsolidated, - dense_rows_soft_deleted: denseRowsSoftDeleted, - dense_skipped: denseSkipped, - }; - - if (firstFailure) { - return jobFailed(firstFailure.reason, firstFailure.cause, did); - } - return jobDone(did); - }, - ); - - return allQueues; -} - -/** - * Fire-and-forget boot discovery for the self-converging rollup / consolidation - * backfills. Each pass is idempotent across reboots and never fails worker boot - * on a miss (errors come back through the helper's result value). - * - * Each backfill is `localConcurrency: 1` on its own, but every one of them used - * to drain from the first pg-boss poll at boot — on a heavy tenant that meant - * several full-history loads contending for the connection pool at once (a boot - * storm / crash-loop risk). Each type now gets an increasing - * `BOOT_BACKFILL_STAGGER_SECONDS` multiple as a `startAfter` delay so their - * loads spread across a window instead of all landing on the same poll. Dense - * intra-day retention keeps its own larger boot defer (the established P2028 - * pool-exhaustion fix) and stays the furthest-out stage. - */ -export async function enqueueRollupBootDiscovery(): Promise { - // v1.4.35.1 — measurement-rollup backfill. Finds every user with - // measurements but no rollup coverage and enqueues a full-fold per account. - // The discovery query only matches accounts with zero rollup rows, so once - // a fold completes the user drops off the list. - try { - // Stage 0 — first off the line; no stagger delay. - const { enqueued, skipped, error } = await enqueueBootTimeRollupBackfill( - BOOT_BACKFILL_STAGGER_SECONDS * 0, - ); - if (error) { - workerLog( - "error", - `[rollup-full-backfill] boot discovery failed: ${error}`, - ); - } else { - // v1.4.38.7 — log the discovery result on every boot, including - // the silent `enqueued=0 skipped=0` case. Without this, an - // operator chasing "analytics is slow" cannot tell whether the - // discovery query ran successfully (and found nothing to fold) - // vs. silently no-op'd. The line is one row per worker boot, so - // the log cost is negligible. - workerLog( - "info", - `[rollup-full-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[rollup-full-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.5.6 — legacy step consolidation. Finds every user still holding live - // pre-v1.5.0 granular step rows and enqueues one consolidation job per - // account. Consolidated legacy rows are soft-deleted, so the - // `deleted_at IS NULL` discovery predicate drops them. - try { - // Stage 1. - const { enqueued, skipped, error } = await enqueueBootTimeStepConsolidation( - BOOT_BACKFILL_STAGGER_SECONDS * 1, - ); - if (error) { - workerLog( - "error", - `[step-consolidation] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[step-consolidation] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[step-consolidation] boot discovery threw an unexpected error", - err, - ); - } - - // v1.7.0 — daily-mean consolidation. Finds every user holding live - // per-sample high-frequency mean-type rows and enqueues one job per account. - // Consolidated rows are soft-deleted, so the discovery predicate drops them. - try { - // Stage 2. - const { enqueued, skipped, error } = await enqueueBootTimeMeanConsolidation( - BOOT_BACKFILL_STAGGER_SECONDS * 2, - ); - if (error) { - workerLog( - "error", - `[mean-consolidation] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[mean-consolidation] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[mean-consolidation] boot discovery threw an unexpected error", - err, - ); - } - - // v1.10.0 WX-E — dense intra-day retention drain. Finds every user holding - // live per-sample dense-tier (HRV / HR) rows OLDER than the retention window - // and enqueues one job per account. Folded rows are soft-deleted, so the - // discovery predicate drops them. - try { - // Furthest-out stage — the helper self-defers past the boot window via its - // own larger constant (the P2028 pool-exhaustion fix), so no stagger - // argument is threaded here. - const { enqueued, skipped, error } = - await enqueueBootTimeDenseIntradayRetention(); - if (error) { - workerLog( - "error", - `[dense-intraday-retention] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[dense-intraday-retention] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[dense-intraday-retention] boot discovery threw an unexpected error", - err, - ); - } - - // v1.28.31 — one-shot hourly history rebuild. Finds every user still - // holding a pre-hourly daily dense-tier `stats:` row paired with tombstoned - // raw rows and enqueues one rebuild job per account. Rebuilt days retire - // their daily row, so the pairing predicate converges to zero across - // boots — the retired row is the durable once-per-install marker. - try { - // Self-defers past both the boot window and the retention drain's own - // 600 s stage via its larger constant; no stagger argument threaded. - const { enqueued, skipped, error } = - await enqueueBootTimeDenseIntradayHourlyRebuild(); - if (error) { - workerLog( - "error", - `[dense-intraday-hourly-rebuild] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[dense-intraday-hourly-rebuild] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[dense-intraday-hourly-rebuild] boot discovery threw an unexpected error", - err, - ); - } - - // v1.4.39 — mood rollup tier. Mirrors the v1.4.35.1 - // measurement-rollup pattern: one job per user with mood entries but no - // rollup coverage. Idempotent across reboots and singleton-keyed inside - // pg-boss so a fast restart while a backfill is queued doesn't double up. - try { - // Stage 3. - const { enqueued, skipped, error } = - await enqueueBootTimeMoodRollupBackfill( - BOOT_BACKFILL_STAGGER_SECONDS * 3, - ); - if (error) { - workerLog( - "error", - `[mood-rollup-full-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[mood-rollup-full-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[mood-rollup-full-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.28.37 — step-consolidation repair. Finds every account still holding - // a resurrectable tombstoned GOOGLE_HEALTH / FITBIT daily-total step row - // (swept by the pre-fix consolidation) and enqueues one repair job per - // account. Self-converging: a resurrected row goes live and drops out of - // the discovery predicate. - try { - // Stage 5. - const { enqueued, skipped, error } = - await enqueueBootTimeStepConsolidationRepair( - BOOT_BACKFILL_STAGGER_SECONDS * 5, - ); - if (error) { - workerLog( - "error", - `[step-consolidation-repair] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[step-consolidation-repair] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[step-consolidation-repair] boot discovery threw an unexpected error", - err, - ); - } - - // v1.30.3 — cumulative-PersonalRecord rederivation. Finds every - // account still holding a suspect measurement-driven cumulative-type - // record created before the source-collapse fix landed and enqueues one - // repair job per account. Self-converging: a deleted-and-redetected - // record's `createdAt` sits after the fix cutoff, so the account drops - // out of the discovery predicate for good. - try { - // Stage 6. - const { enqueued, skipped, error } = - await enqueueBootTimeCumulativePrRederivation( - BOOT_BACKFILL_STAGGER_SECONDS * 6, - ); - if (error) { - workerLog( - "error", - `[cumulative-pr-rederive] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[cumulative-pr-rederive] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[cumulative-pr-rederive] boot discovery threw an unexpected error", - err, - ); - } -} diff --git a/src/lib/jobs/reminder/register-status.ts b/src/lib/jobs/reminder/register-status.ts deleted file mode 100644 index 81f241b10..000000000 --- a/src/lib/jobs/reminder/register-status.ts +++ /dev/null @@ -1,878 +0,0 @@ -/** - * Status / insight / computed-score queue registrar. - * - * Owns the nightly status batch (every per-metric status card, one provider - * call per account), the comprehensive-insight - * pre-generation, the on-demand per-metric status generation, the nightly - * computed scores (recovery / stress / strain), the proactive Coach nudge, the - * Coach memory refresh, the period-narrative warm, and the medication - * low-stock pass. - * - * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its - * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The - * status-wiring guards (`status-cron-candidates`, `insight-pregenerate`, - * `insight-status-generate`, `recovery-score-queue`, - * `stress-strain-retention-queue`, `period-narrative-warm`, - * `coach-memory-refresh-queue`) read THIS module. - */ -import { PgBoss } from "pg-boss"; -import { reportWorkerError } from "@/lib/jobs/report-worker-error"; -import { recordError } from "@/lib/jobs/worker-status"; -import { - INSIGHT_PREGENERATE_QUEUE, - INSIGHT_PREGENERATE_CRON, - type InsightPregeneratePayload, -} from "@/lib/jobs/insight-pregenerate"; -import { - MORNING_DIGEST_REFRESH_QUEUE, - runMorningDigestRefresh, - type MorningDigestRefreshPayload, -} from "@/lib/jobs/morning-digest-refresh"; -import { DATA_ARRIVAL_QUEUE, handleDataArrival } from "@/lib/jobs/data-arrival"; -import { - REACTION_LINE_QUEUE, - handleReactionLine, -} from "@/lib/jobs/reaction-line"; -import type { ReactionLineJob } from "@/lib/arrivals/reaction-line-shared"; -import { - handleWorkoutInsightGenerate, - WORKOUT_INSIGHT_GENERATE_CONCURRENCY, - WORKOUT_INSIGHT_GENERATE_QUEUE, - type WorkoutInsightGeneratePayload, -} from "@/lib/jobs/workout-insight-generate"; -import type { DataArrival } from "@/lib/arrivals/types"; -import { - DAILY_BRIEFING_QUEUE, - DAILY_BRIEFING_CRON, - runDailyBriefingTick, -} from "@/lib/jobs/daily-briefing"; -import { maybeDispatchDailyBriefing } from "@/lib/daily/daily-briefing-push"; -import { - RECOVERY_SCORE_QUEUE, - RECOVERY_SCORE_CRON, - runRecoveryScore, -} from "@/lib/jobs/recovery-score"; -import { - COACH_NUDGE_QUEUE, - COACH_NUDGE_CRON, - runCoachNudgeTick, -} from "@/lib/jobs/coach-nudge"; -import { - COACH_REMINDER_SWEEP_QUEUE, - COACH_REMINDER_SWEEP_CRON, - runCoachReminderSweep, -} from "@/lib/jobs/coach-reminder-sweep"; -import { - COACH_PLAN_REVIEW_QUEUE, - COACH_PLAN_REVIEW_CRON, - runCoachPlanReviewTick, -} from "@/lib/jobs/coach-plan-review"; -import { - MEDICATION_LOW_STOCK_QUEUE, - MEDICATION_LOW_STOCK_CRON, - runMedicationLowStockTick, -} from "@/lib/jobs/medication-low-stock"; -import { - STRESS_SCORE_QUEUE, - STRESS_SCORE_CRON, - runStressScore, -} from "@/lib/jobs/stress-score"; -import { - STRAIN_SCORE_QUEUE, - STRAIN_SCORE_CRON, - runStrainScore, -} from "@/lib/jobs/strain-score"; -import { - PERIOD_NARRATIVE_QUEUE, - PERIOD_NARRATIVE_CRON, - runPeriodNarrativeWarm, - warmOneNarrative, - type PeriodNarrativePayload, -} from "@/lib/jobs/period-narrative-warm"; -import { - COACH_MEMORY_REFRESH_QUEUE, - type CoachMemoryRefreshPayload, -} from "@/lib/ai/coach/coach-memory-shared"; -import { runCoachMemoryRefresh } from "@/lib/ai/coach/coach-memory-refresh-worker"; -import { - INSIGHT_STATUS_GENERATE_QUEUE, - INSIGHT_STATUS_GENERATE_CONCURRENCY, - type InsightStatusGeneratePayload, -} from "@/lib/jobs/insight-status-generate"; -import { withBackgroundEvent } from "@/lib/logging/background"; -import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; -import { jobBudget } from "@/lib/jobs/job-budget"; -import { lockedPass, WHOLE_PASS } from "@/lib/jobs/long-pass"; -import { heavyLane } from "@/lib/jobs/heavy-lane"; -import { INSIGHT_PREGENERATE_EXPIRE_SECONDS } from "@/lib/jobs/insight-pregenerate-shared"; -import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "@/lib/jobs/period-narrative-shared"; -import { getWorkerPrisma, workerLog } from "./shared"; -import { - createAndSchedule, - createAndWork, - insightRetryOptions, - nightlyInsightPassOptions, - retireQueues, - type QueuePolicyTable, - type ScheduleEntry, -} from "./registrar-shared"; -import { - GeneralStatusPayload, - handleGeneralStatusGenerate, - handleInsightPregenerateJob, - handleInsightStatusGenerate, -} from "./insights-handlers"; - -const GENERAL_STATUS_QUEUE = "insights-general-status"; - -const GENERAL_STATUS_CRON = "0 2 * * *"; // daily at 02:00 - -/** - * The six per-metric status crons that followed the 02:00 batch (02:05 to - * 02:30). The batch builds every card and falls each metric it could not - * cover back to that card's own path, so they only re-read cards the batch - * had just written; a user who joins later is warmed by the visit or the next - * night. Their schedules and queues are removed on an instance that still has - * them. - */ -const RETIRED_STATUS_QUEUES = [ - "insights-blood-pressure-status", - "insights-weight-status", - "insights-pulse-status", - "insights-bmi-status", - "insights-mood-status", - "insights-medication-compliance-status", -] as const; - -const allQueues = [ - GENERAL_STATUS_QUEUE, - // v1.7.0 — nightly comprehensive-insight pre-generation so the daily - // briefing is warm before the user opens /insights or the dashboard - // snapshot. Without this entry the 04:30 schedule silently no-ops. - INSIGHT_PREGENERATE_QUEUE, - // v1.8.3 — on-demand per-metric status generation. The read-only status - // route enqueues here on a cold card; without this entry pg-boss never - // provisions the queue and the enqueue silently drops. No cron schedule — - // a send-only queue driven by navigation. - INSIGHT_STATUS_GENERATE_QUEUE, - // Event-driven morning digest refresh, enqueued by the sleep-arrival - // trigger when last night's completed sleep lands. No cron schedule — a - // send-only queue driven by sleep ingest; without this entry pg-boss never - // provisions the queue and every enqueue silently drops. - MORNING_DIGEST_REFRESH_QUEUE, - // v1.31.0 — the data-arrival spine. Every salient ingest emits here; the - // worker claims the day's reaction marker and fans out to the reaction - // surfaces. No cron schedule — a send-only queue driven by ingest; without - // this entry pg-boss never provisions the queue and every emit silently - // drops (the v1.4.37 dead-queue class). - DATA_ARRIVAL_QUEUE, - // v1.31.0 — the day's single arrival reaction line. Send-only, enqueued by - // the spine worker on the pass that claimed the day's marker. Without this - // entry pg-boss never provisions the queue and every enqueue silently drops. - REACTION_LINE_QUEUE, - // v1.31.0 — the per-workout Activity Insight, dispatched by the spine when a - // workout lands. No cron schedule — nothing warms it and no read path - // triggers it, so this entry is the only thing that provisions the queue. - WORKOUT_INSIGHT_GENERATE_QUEUE, - // v1.10.0 — computed scores (WX-C / WX-E). Nightly Recovery / Stress / - // Strain compute + store. The queue MUST be registered here or pg-boss - // never provisions it and the nightly schedule silently never fires. - RECOVERY_SCORE_QUEUE, - STRESS_SCORE_QUEUE, - STRAIN_SCORE_QUEUE, - // v1.11.0 — nightly period-narrative warm + single-user warm enqueued by - // the read-only narrative GET. - PERIOD_NARRATIVE_QUEUE, - // v1.11.1 — combined Coach memory-refresh (rolling conversation summary + - // durable fact extraction), enqueued fire-and-forget from a long chat turn. - COACH_MEMORY_REFRESH_QUEUE, - // v1.15.20 — proactive Coach nudge. Without this entry the daily 05:15 - // schedule silently no-ops and no nudge ever fires. - COACH_NUDGE_QUEUE, - // v1.22 — daily Coach-reminder sweep. Without this entry the 05:20 - // schedule silently no-ops and no "remind me" reminder ever surfaces. - COACH_REMINDER_SWEEP_QUEUE, - // v1.22 — daily n-of-1 experiment review worker. Without this entry - // the 05:25 schedule silently no-ops and no experiment read-back is written. - COACH_PLAN_REVIEW_QUEUE, - // v1.16.11 — daily medication low-stock pass. Without this entry the 09:00 - // schedule silently no-ops and no low-stock alert ever fires. - MEDICATION_LOW_STOCK_QUEUE, - // Daily-briefing fallback cron. The primary morning push rides the - // sleep-arrival finalisation hook below; this queue is the fixed local-morning - // slot for the no-sleep-arrived case. Without this entry the every-15-min - // schedule silently no-ops and the fallback push never fires. - DAILY_BRIEFING_QUEUE, -]; - -const schedules: ScheduleEntry[] = [ - [GENERAL_STATUS_QUEUE, GENERAL_STATUS_CRON, nightlyInsightPassOptions], - // v1.7.0 — nightly 04:30 Europe/Berlin comprehensive-insight - // pre-generation. Budget-gated per user inside the handler. - [ - INSIGHT_PREGENERATE_QUEUE, - INSIGHT_PREGENERATE_CRON, - { - ...insightRetryOptions, - expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS, - }, - ], - // v1.10.0 — computed scores (WX-C). Nightly 04:45 Europe/Berlin - // Recovery-score compute + store, after the rollup-feeding consolidation - // + drain so the signals it reads are already folded. - [RECOVERY_SCORE_QUEUE, RECOVERY_SCORE_CRON], - // v1.10.0 — computed scores (WX-E). Nightly 04:50 Europe/Berlin - // Stress-score compute + store, after the dense intra-day retention - // drain so the HRV inputs it reads are settled. - [STRESS_SCORE_QUEUE, STRESS_SCORE_CRON], - // v1.10.0 — computed scores (WX-E). Nightly 04:55 Europe/Berlin - // Strain-score compute + store, after the recovery + stress passes so - // the nightly score writes stay ordered. - [STRAIN_SCORE_QUEUE, STRAIN_SCORE_CRON], - // v1.11.0 — nightly 05:05 Europe/Berlin period-narrative warm. The - // handler only fans out on a week (Mon) / month (1st) boundary; every - // other night is a cheap no-op. Budget-gated per user inside the runner. - [ - PERIOD_NARRATIVE_QUEUE, - PERIOD_NARRATIVE_CRON, - { - ...insightRetryOptions, - expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS, - }, - ], - // v1.15.20 — daily 05:15 Europe/Berlin proactive Coach nudge, after - // the 04:45–04:55 score crons so the recovery-score trigger reads - // settled rows. Deterministic triggers only — no AI call on this path. - [COACH_NUDGE_QUEUE, COACH_NUDGE_CRON], - // v1.22 — daily 05:20 Europe/Berlin Coach-reminder sweep, just after - // the nudge tick. Flips overdue reminders to `due` + mints plan-review - // reminders from passed CoachPlan.reviewDate. In-app surface only; no push. - [COACH_REMINDER_SWEEP_QUEUE, COACH_REMINDER_SWEEP_CRON], - // v1.22 — daily 05:25 Europe/Berlin n-of-1 experiment review, just - // after the reminder sweep mints the plan-review reminders this worker reads. - // Writes the grounded before/after read-back; no provider call, no push. - [COACH_PLAN_REVIEW_QUEUE, COACH_PLAN_REVIEW_CRON], - // v1.16.11 — medication low-stock pass at 09:00 Europe/Berlin: a - // supply alert is an errand prompt, so it fires at a time the user - // can act on it. Once daily; the per-medication stamp keeps it at - // one push per threshold crossing. - [MEDICATION_LOW_STOCK_QUEUE, MEDICATION_LOW_STOCK_CRON], - // Daily-briefing fallback slot. Ticks every 15 min; the per-user - // local-hour gate inside the tick selects each user's fixed morning slot, so - // one UTC cron serves every timezone. The `push_attempts` cap suppresses it - // when the sleep-arrival finalisation push already fired this morning. - [DAILY_BRIEFING_QUEUE, DAILY_BRIEFING_CRON], -]; - -/** - * De-duplication policy per status queue. - * - * Most queues here are keyless nightly cron ticks — no `singletonKey` on any - * send — so a policy would only constrain the empty key and is deliberately - * omitted. The LLM-bound warm queues (insight pre-generate, per-metric status - * generate, period narrative, Coach memory refresh) already pass an explicit - * `singletonSeconds` on every send, which populates `singleton_on` and is - * therefore constrained by pg-boss's `job_i4` index REGARDLESS of queue policy. - * Those already de-duplicate today; changing their policy would layer a second, - * differently-shaped constraint over a working one for no gain, so they are - * left alone. - * - * That leaves exactly two queues here whose keys would otherwise be inert. - */ -const queuePolicies: QueuePolicyTable = { - // v1.31.0 — the arrival spine's day-scoped keys - // (`arrival:::`) carry the SAME requirement as the - // morning refresh below, for the same reason: a user's local date cannot be - // expressed as a wall-clock `singletonSeconds` window, so the date lives in - // the key and the policy has to honour it. Under `standard` the key - // coalesces nothing at all, and a device posting a batch per minute would - // queue a job per batch instead of one per day. - // - // `exclusive` rather than `short` because the handler is self-converging: - // it claims a durable unique row, so a second run while the first is active - // is pure duplicated work, and the next ingest re-emits anyway. - // - // The `workout` kind is the deliberate exception — it is keyed per workout - // id and DOES pass `singletonSeconds`, so it is constrained by pg-boss's - // `job_i4` index regardless of this policy. Both shapes coexist on one - // queue safely: they populate different columns. - // The reaction line's key is `reaction-line:::` — the - // same day-scoped shape, and for the same reason: a wall-clock window cannot - // express a user's local date. `exclusive` is what makes the key real, and it - // is the queue-level restatement of the durable claim the worker re-checks - // (`generatedAt`) before it spends anything. - [REACTION_LINE_QUEUE]: { - policy: "exclusive", - reason: - "Day-scoped per-kind keys carry the user's local date; under standard they coalesce nothing and a retried spine job could stack a second provider call behind the first.", - }, - [DATA_ARRIVAL_QUEUE]: { - policy: "exclusive", - reason: - "Day-scoped arrival keys carry the user's local date; under standard they coalesce nothing and a chatty ingest queues a job per batch. The worker claims a durable unique row, so a concurrent second run is pure duplicated work.", - }, - // The sleep-arrival trigger has seven write seams that can each enqueue for - // the same user and the same local date. The key is - // `morning-refresh::`, and `exclusive` turns that into a - // real at-most-once-per-user-per-local-morning contract across queued, - // active, and retry-backoff states. - // - // `singletonSeconds` is NOT the tool here and the existing comment at the - // enqueue site says so: its window is a floor() over wall-clock, which does - // not line up with a user's local date, so it would both split one morning - // across two buckets and merge two mornings into one. The local date already - // lives in the key; the policy just has to honour it. - [MORNING_DIGEST_REFRESH_QUEUE]: { - policy: "exclusive", - reason: - "Seven sleep-write seams enqueue the same user+localDate key; exclusive makes the at-most-once-per-morning contract real. A wall-clock singletonSeconds window cannot express a local date.", - }, -}; - -/** - * Register every status / insight / computed-score queue. Returns the queue - * names created (for the boot-level aggregate assertion). - */ -export async function registerStatusQueues( - boss: PgBoss, -): Promise { - await createAndSchedule(boss, allQueues, schedules, queuePolicies); - await retireQueues(boss, RETIRED_STATUS_QUEUES); - - await createAndWork( - boss, - GENERAL_STATUS_QUEUE, - { localConcurrency: 1 }, - lockedPass( - GENERAL_STATUS_QUEUE, - () => WHOLE_PASS, - heavyLane(GENERAL_STATUS_QUEUE, handleGeneralStatusGenerate), - ), - ); - // v1.7.0 — nightly comprehensive-insight pre-generation. - // - // v1.16.8 — localConcurrency 2 (was 1). The queue carries BOTH the - // scheduled 04:30 cohort walk AND the visit-triggered per-user force - // warms; with a single slot a force warm enqueued during the nightly - // batch sat behind the entire cohort and the visiting user stared at - // a cold dashboard for the duration. Two slots let one force warm run - // alongside the batch while still bounding provider-level concurrency - // (the cohort walk is itself sequential per user, and the content-hash - // gate + per-user budget gate inside the handler cover the rare - // double-tick overlap the old single slot serialised away). - await createAndWork( - boss, - INSIGHT_PREGENERATE_QUEUE, - { localConcurrency: 2 }, - // The cohort walk is one pass at a time; a forced warm is one per - // account, so it never waits behind the walk. - lockedPass( - INSIGHT_PREGENERATE_QUEUE, - (job) => - job.data?.force && job.data.userId - ? `user:${job.data.userId}` - : WHOLE_PASS, - heavyLane( - INSIGHT_PREGENERATE_QUEUE, - handleInsightPregenerateJob, - (job) => !(job.data?.force && job.data.userId), - ), - ), - ); - // Event-driven morning digest refresh. One forced comprehensive - // regeneration per user per local morning, enqueued by the sleep-arrival - // trigger. Single-flight: the enqueue-side singletonKey already collapses a - // night's samples into one job, and the handler's marker re-check makes a - // rare double-fire a no-op, so one slot is enough and bounds the provider - // concurrency the same way the nightly path does. - await createAndWork( - boss, - MORNING_DIGEST_REFRESH_QUEUE, - { localConcurrency: 1 }, - async (jobs): Promise => { - return withBackgroundEvent("job.morning_digest_refresh", async (evt) => { - // The counts the batch already decides on: one refresh per job, and - // the finalised subset that reaches the morning push seam. - let refreshed = 0; - let finalised = 0; - for (const job of jobs) { - try { - const result = await runMorningDigestRefresh( - getWorkerPrisma(), - job.data, - ); - refreshed++; - evt.addMeta( - "morning_refresh", - `${result.status}:${result.comprehensive ?? "none"}`, - ); - // The day just finalised (last night's sleep folded in), so - // this is the natural, event-driven moment to fire the calm morning - // push carrying the FINAL digest. Fully fault-isolated inside the - // seam; the opt-in / morning-window / once-per-day gates all live - // there, so a user who never opted in or is outside the window is a - // clean no-op. The fallback cron covers the no-sleep-arrived case. - if (result.status === "finalised") { - finalised++; - const pushResult = await maybeDispatchDailyBriefing( - getWorkerPrisma(), - job.data.userId, - new Date(), - ); - evt.addMeta("daily_briefing_push", pushResult); - } - } catch (err) { - recordError(); - workerLog("error", "[morning-digest-refresh] pass failed", err); - // Rethrow so the queue's retry policy re-runs the refresh; the - // marker re-check keeps a retry idempotent. - throw err; - } - } - return jobDone({ refreshed, finalised }); - }); - }, - ); - // v1.8.3 — on-demand per-metric status generation enqueued by the - // read-only status route on a cold card. Low concurrency so a first - // visit that cold-misses several cards can't saturate the Prisma pool - // or fan out an unbounded number of concurrent provider calls. - await createAndWork( - boss, - INSIGHT_STATUS_GENERATE_QUEUE, - { localConcurrency: INSIGHT_STATUS_GENERATE_CONCURRENCY }, - handleInsightStatusGenerate, - ); - // v1.10.0 — computed scores (WX-C). Nightly Recovery-score compute + - // store. The cron tick carries an empty payload; the runner iterates every - // eligible user and upserts one `COMPUTED RECOVERY_SCORE` row per scored - // day (idempotent — a re-fire overwrites in place). Single-flight so two - // ticks never double-walk the cohort. - await createAndWork( - boss, - RECOVERY_SCORE_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - try { - const summary = await runRecoveryScore(getWorkerPrisma()); - workerLog( - "info", - `[recovery-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - return jobDone({ - considered: summary.considered, - stored: summary.stored, - insufficient: summary.insufficient, - errored: summary.errored, - }); - } catch (err) { - recordError(); - workerLog("error", "[recovery-score] pass failed", err); - throw err; - } - }, - ); - // v1.15.20 — proactive Coach nudge. Single-flight; the push-attempts - // ledger caps a user at one nudge per rolling week, so an overlapping - // tick would only waste reads. Deterministic triggers, no AI call. - await createAndWork( - boss, - COACH_NUDGE_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - return withBackgroundEvent("job.coach_nudge", async (evt) => { - try { - const summary = await runCoachNudgeTick( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.coach_nudge", - result: { - candidates_scanned: summary.candidatesScanned, - dispatched: summary.dispatched, - persisted: summary.persisted, - skipped_opted_out: summary.skippedOptedOut, - skipped_no_provider: summary.skippedNoProvider, - skipped_unavailable: summary.skippedUnavailable, - skipped_recent_nudge: summary.skippedRecentNudge, - skipped_recent_engagement: summary.skippedRecentEngagement, - skipped_no_trigger: summary.skippedNoTrigger, - skipped_no_channel: summary.skippedNoChannel, - failed: summary.failed, - }, - }); - // A per-user failure never fails this pass — it rides out as the - // `failed` count, exactly as it does in the wide event. - return jobDone({ - candidates_scanned: summary.candidatesScanned, - dispatched: summary.dispatched, - persisted: summary.persisted, - failed: summary.failed, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }, - ); - // v1.16.11 — medication low-stock pass. Single-flight; the - // per-medication stamp makes a re-fire idempotent (already-notified - // crossings skip), so an overlapping tick would only waste reads. - await createAndWork( - boss, - MEDICATION_LOW_STOCK_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - return withBackgroundEvent("job.medication_low_stock", async (evt) => { - try { - const summary = await runMedicationLowStockTick( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.medication_low_stock", - result: { - users_scanned: summary.usersScanned, - skipped_threshold_off: summary.skippedThresholdOff, - medications_evaluated: summary.medicationsEvaluated, - notified: summary.notified, - rearmed: summary.rearmed, - skipped_already_notified: summary.skippedAlreadyNotified, - skipped_above_threshold: summary.skippedAboveThreshold, - skipped_no_runway: summary.skippedNoRunway, - skipped_no_channel: summary.skippedNoChannel, - failed: summary.failed, - }, - }); - // A per-medication failure never fails this pass — it rides out as - // the `failed` count. - return jobDone({ - users_scanned: summary.usersScanned, - medications_evaluated: summary.medicationsEvaluated, - notified: summary.notified, - rearmed: summary.rearmed, - failed: summary.failed, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }, - ); - // v1.31.0 — the data-arrival spine. Single-flight: the worker's whole job is - // to claim a durable unique row, so two slots would only race each other for - // a claim exactly one of them can win. It makes no provider call — a - // module-graph test pins that — so one slot costs nothing in throughput. - await createAndWork( - boss, - DATA_ARRIVAL_QUEUE, - { localConcurrency: 1 }, - handleDataArrival, - ); - // v1.31.0 — the arrival reaction line. Single-flight: this is the spine's - // ONE provider call, and a second concurrent slot would only race the same - // durable `generatedAt` claim exactly one of them can win. - await createAndWork( - boss, - REACTION_LINE_QUEUE, - { localConcurrency: 1 }, - handleReactionLine, - ); - // v1.31.0 — the per-workout Activity Insight. Serial: this IS the provider - // path the spine deliberately does not walk, so it stays off the request pool - // and cannot fan out concurrent completions when several workouts land in one - // sync. - await createAndWork( - boss, - WORKOUT_INSIGHT_GENERATE_QUEUE, - { localConcurrency: WORKOUT_INSIGHT_GENERATE_CONCURRENCY }, - handleWorkoutInsightGenerate, - ); - // Daily-briefing fallback slot. Single-flight; the `push_attempts` - // frequency cap makes an overlapping tick a no-op (a second same-day attempt - // is suppressed), and the tick only loads a digest for opted-in users at - // their exact local fallback hour. No provider call — it reads the cached - // digest and dispatches the deterministic line + cached lead. - await createAndWork( - boss, - DAILY_BRIEFING_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - return withBackgroundEvent("job.daily_briefing", async (evt) => { - try { - const summary = await runDailyBriefingTick( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.daily_briefing", - result: { - candidates_scanned: summary.candidatesScanned, - in_slot: summary.inSlot, - sent: summary.sent, - suppressed_frequency: summary.suppressedFrequency, - no_digest: summary.noDigest, - opted_out: summary.optedOut, - no_channel: summary.noChannel, - outside_window: summary.outsideWindow, - failed: summary.failed, - }, - }); - // Most ticks land outside every user's slot; that is the tick doing - // its job, so a zero `sent` is a success with zero counts. - return jobDone({ - candidates_scanned: summary.candidatesScanned, - in_slot: summary.inSlot, - sent: summary.sent, - failed: summary.failed, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }, - ); - // v1.10.0 — computed scores (WX-E). Nightly Stress-score (HRV-derived - // proxy) compute + store. Single-flight so two ticks never double-walk - // the cohort. The runner iterates every eligible user and upserts one - // `COMPUTED STRESS_SCORE` row per scored day (idempotent — a re-fire - // overwrites in place). - await createAndWork( - boss, - STRESS_SCORE_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - try { - const summary = await runStressScore(getWorkerPrisma()); - workerLog( - "info", - `[stress-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - return jobDone({ - considered: summary.considered, - stored: summary.stored, - insufficient: summary.insufficient, - errored: summary.errored, - }); - } catch (err) { - recordError(); - workerLog("error", "[stress-score] pass failed", err); - throw err; - } - }, - ); - // v1.10.0 — computed scores (WX-E). Nightly Strain-score (Banister TRIMP - // cardio-load) compute + store. Single-flight; upserts one `COMPUTED - // STRAIN_SCORE` row per scored day (idempotent). - await createAndWork( - boss, - STRAIN_SCORE_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - try { - const summary = await runStrainScore(getWorkerPrisma()); - workerLog( - "info", - `[strain-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - return jobDone({ - considered: summary.considered, - stored: summary.stored, - insufficient: summary.insufficient, - errored: summary.errored, - }); - } catch (err) { - recordError(); - workerLog("error", "[strain-score] pass failed", err); - throw err; - } - }, - ); - // v1.11.0 — period-narrative warm. A scheduled tick (no `userId`) runs the - // boundary-gated nightly fan-out; a `userId` payload runs a single-user warm - // enqueued by the read-only GET on a cold/stale read. Single-flight so two - // ticks never double-walk the cohort; the per-user budget gate covers the - // fan-out and the enqueue `singletonKey` covers the single-user path. - await createAndWork( - boss, - PERIOD_NARRATIVE_QUEUE, - { localConcurrency: 1 }, - lockedPass( - PERIOD_NARRATIVE_QUEUE, - (job) => (job.data?.userId ? `user:${job.data.userId}` : WHOLE_PASS), - heavyLane( - PERIOD_NARRATIVE_QUEUE, - async (jobs): Promise => { - // Two modes on one queue, so the outcome counts them separately: a - // non-boundary night is a scheduled pass that generated nothing, which - // is a success with zero counts, not an idle queue. - let singleUser = 0; - let scheduled = 0; - let generated = 0; - let failed = 0; - let stoppedEarly = false; - for (const job of jobs) { - try { - if (job.data?.userId) { - await warmOneNarrative(job.data); - singleUser++; - } else { - const summary = await runPeriodNarrativeWarm( - getWorkerPrisma(), - { - shouldStop: jobBudget(jobs), - }, - ); - scheduled++; - generated += summary.generated; - failed += summary.failed; - stoppedEarly ||= summary.stoppedEarly; - workerLog( - "info", - `[period-narrative] periods=${summary.periods.join(",") || "none"} total=${summary.total} generated=${summary.generated} cached=${summary.cached} skipped=${summary.skipped} insufficient=${summary.insufficient} failed=${summary.failed} budget=${summary.budgetBlocked}`, - ); - } - } catch (err) { - recordError(); - await reportWorkerError(PERIOD_NARRATIVE_QUEUE, err, { - mode: job.data?.userId ? "single-user" : "scheduled", - }); - throw err; - } - } - return jobDone({ - single_user: singleUser, - scheduled, - generated, - failed, - stopped_early: stoppedEarly, - }); - }, - (job) => !job.data?.userId, - ), - ), - ); - // v1.11.1 — combined Coach memory refresh: rolling conversation summary + - // durable fact extraction for one long conversation. localConcurrency 1 so a - // burst of long-conversation turns can't fan out concurrent provider calls; - // each step is budget-gated inside runStatusCompletion and fault-isolated. - await createAndWork( - boss, - COACH_MEMORY_REFRESH_QUEUE, - { localConcurrency: 1 }, - async (jobs): Promise => { - // An incomplete payload is skipped rather than failed, so it has to be - // countable — otherwise a queue full of them reads as a clean run. - let refreshed = 0; - let skippedIncomplete = 0; - for (const job of jobs) { - if (!job.data?.conversationId || !job.data?.userId) { - skippedIncomplete++; - continue; - } - try { - await runCoachMemoryRefresh(job.data); - refreshed++; - } catch (err) { - recordError(); - workerLog("error", "[coach-memory-refresh] failed", err); - throw err; - } - } - return jobDone({ - refreshed, - skipped_incomplete: skippedIncomplete, - }); - }, - ); - - // v1.22 — daily Coach-reminder sweep. Single-flight; surfacing an - // overdue reminder into the user's Coach thread and minting plan-review - // reminders are both idempotent across re-fires (an already-surfaced - // reminder is not re-written; a plan whose reviewDate was cleared is not - // re-minted). No provider call, no push — the message body is a - // deterministic localized template around the user's own note. - await createAndWork( - boss, - COACH_REMINDER_SWEEP_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - return withBackgroundEvent("job.coach_reminder_sweep", async (evt) => { - try { - const summary = await runCoachReminderSweep( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.coach_reminder_sweep", - result: { - reminders_due: summary.remindersDue, - plan_reviews_minted: summary.planReviewsMinted, - context_surfaced: summary.contextSurfaced, - nag_dismissed: summary.nagDismissed, - errored: summary.errored, - }, - }); - return jobDone({ - reminders_due: summary.remindersDue, - plan_reviews_minted: summary.planReviewsMinted, - context_surfaced: summary.contextSurfaced, - nag_dismissed: summary.nagDismissed, - errored: summary.errored, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }, - ); - - // v1.22 — n-of-1 experiment review worker: reads the plan-review - // reminders the sweep minted and writes the grounded before/after read-back - // into the plan's encrypted outcome column, flipping it to `reviewed`. - // Idempotent (a reviewed plan is not re-processed). No provider call, no push. - await createAndWork( - boss, - COACH_PLAN_REVIEW_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - return withBackgroundEvent("job.coach_plan_review", async (evt) => { - try { - const summary = await runCoachPlanReviewTick( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.coach_plan_review", - result: { - reviewed: summary.reviewed, - insufficient: summary.insufficient, - errored: summary.errored, - }, - }); - return jobDone({ - reviewed: summary.reviewed, - insufficient: summary.insufficient, - errored: summary.errored, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }, - ); - - return allQueues; -} diff --git a/src/lib/jobs/reminder/rollup-handlers.ts b/src/lib/jobs/reminder/rollup-handlers.ts new file mode 100644 index 000000000..74c765d02 --- /dev/null +++ b/src/lib/jobs/reminder/rollup-handlers.ts @@ -0,0 +1,582 @@ +/** + * Handlers of the rollup and consolidation queues: the measurement and mood + * rollup folds, the consolidation and repair passes, the supplement nutrient + * recompute, and the nightly drain that runs the cumulative, daily-mean and + * dense-retention passes on one tick. The registry binds them + * (`registry/rollup.ts`) and loads this module on the first job. + */ +import type { Job, PgBoss } from "pg-boss"; + +import { + DRAIN_CONTINUATION_DELAY_SECONDS, + nextDrainContinuation, +} from "@/lib/jobs/drain-continuation"; +import { + DRAIN_CUMULATIVE_QUEUE, + type DrainCumulativePayload, +} from "@/lib/jobs/drain-cumulative-queue"; +import { runDenseIntradayHourlyRebuildForUser } from "@/lib/jobs/dense-intraday-hourly-rebuild"; +import type { DenseIntradayHourlyRebuildPayload } from "@/lib/jobs/dense-intraday-hourly-rebuild"; +import { + DENSE_INTRADAY_RETENTION_ENABLED, + runDenseIntradayRetentionForUser, + type DenseIntradayRetentionPayload, +} from "@/lib/jobs/dense-intraday-retention"; +import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { + runMeanConsolidationForUser, + type MeanConsolidationPayload, +} from "@/lib/jobs/mean-consolidation"; +import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { + runStepConsolidationForUser, + type StepConsolidationPayload, +} from "@/lib/jobs/step-consolidation"; +import { + runStepConsolidationRepairForUser, + type StepConsolidationRepairPayload, +} from "@/lib/jobs/step-consolidation-repair"; +import { recordError } from "@/lib/jobs/worker-status"; +import { + consolidateDailyMean, + MEAN_CONSOLIDATION_CUTOFF_HOURS, +} from "@/lib/measurements/consolidate-daily-mean"; +import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention"; +import { + drainPerSampleCumulative, + DRAIN_CUMULATIVE_CUTOFF_HOURS, +} from "@/lib/measurements/drain-per-sample-cumulative"; +import { + runSupplementNutrientRecompute, + type SupplementNutrientRecomputePayload, +} from "@/lib/medications/ingredients/recompute-job"; +import { + runCumulativePrRederivationForUser, + type CumulativePrRederivePayload, +} from "@/lib/personal-records/cumulative-pr-rederivation"; +import { + recomputeUserRollups, + type RollupFullBackfillPayload, + type RollupRecomputePayload, +} from "@/lib/rollups/measurement-rollups"; +import { + recomputeUserMoodRollups, + type MoodRollupFullBackfillPayload, + type MoodRollupRecomputePayload, +} from "@/lib/rollups/mood-rollups"; + +import { cronIsTheRetry } from "./registrar-shared"; +import { getWorkerPrisma, workerLog } from "./shared"; + +/** + * Folds the WEEK / MONTH / YEAR buckets the write-path hooks enqueue; the DAY + * bucket is recomputed by the hook itself. No per-bucket isolation on + * purpose: a recompute that throws fails the job, so reaching the end means + * every bucket in the batch was folded. + */ +export async function handleRollupRecompute( + jobs: Job[], +): Promise { + for (const job of jobs) { + const payload = job.data; + await recomputeUserRollups(payload.userId, { + types: [payload.type], + granularities: [payload.granularity], + from: new Date(payload.from), + to: new Date(payload.to), + }); + } + return jobDone({ buckets_recomputed: jobs.length }); +} + +/** + * The full fold of one uncovered account across the default window and every + * granularity. Each granularity commits on its own, so a run stopped on its + * budget throws and the retry folds the rest. + */ +export async function handleRollupFullBackfill( + jobs: Job[], +): Promise { + const shouldStop = jobBudget(jobs); + let rowsUpsertedTotal = 0; + for (const job of jobs) { + const { userId } = job.data; + const { rowsUpserted, durationMs, stopped } = await recomputeUserRollups( + userId, + { shouldStop }, + ); + if (stopped) { + throw new Error( + `rollup-full-backfill for user ${userId} stopped on its job budget; the retry folds the rest`, + ); + } + rowsUpsertedTotal += rowsUpserted; + workerLog( + "info", + `[rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, + ); + } + return jobDone({ + users: jobs.length, + rows_upserted: rowsUpsertedTotal, + }); +} + +/** Collapses legacy granular step rows into one daily total per day. */ +export async function handleStepConsolidation( + jobs: Job[], +): Promise { + let daysTotal = 0; + let legacyRowsTotal = 0; + for (const job of jobs) { + const { userId } = job.data; + const { daysConsolidated, legacyRowsSoftDeleted } = + await runStepConsolidationForUser(userId); + daysTotal += daysConsolidated; + legacyRowsTotal += legacyRowsSoftDeleted; + workerLog( + "info", + `[step-consolidation] user=${userId} days=${daysConsolidated} legacyRowsSoftDeleted=${legacyRowsSoftDeleted}`, + ); + } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + legacy_rows_soft_deleted: legacyRowsTotal, + }); +} + +/** + * Resurrects the provider daily-total step rows the earlier consolidation + * swept and removes the shadow manual totals it minted. Per-day failures are + * the repair's own and ride out as a count. + */ +export async function handleStepConsolidationRepair( + jobs: Job[], +): Promise { + let rowsResurrected = 0; + let wedgeSkipped = 0; + let manualMintsRemoved = 0; + let daysRecomputed = 0; + let failures = 0; + for (const job of jobs) { + const { userId } = job.data; + const summary = await runStepConsolidationRepairForUser(userId); + rowsResurrected += summary.rowsResurrected; + wedgeSkipped += summary.wedgeSkipped; + manualMintsRemoved += summary.manualMintsRemoved; + daysRecomputed += summary.daysRecomputed; + failures += summary.failures; + workerLog( + "info", + `[step-consolidation-repair] user=${userId} resurrected=${summary.rowsResurrected} wedgeSkipped=${summary.wedgeSkipped} manualMintsRemoved=${summary.manualMintsRemoved} daysRecomputed=${summary.daysRecomputed} failures=${summary.failures}`, + ); + } + return jobDone({ + users: jobs.length, + rows_resurrected: rowsResurrected, + wedge_skipped: wedgeSkipped, + manual_mints_removed: manualMintsRemoved, + days_recomputed: daysRecomputed, + failures, + }); +} + +/** One account's supplement doses recomputed into the day's micronutrients. */ +export async function handleSupplementNutrientRecompute( + jobs: Job[], +): Promise { + let rowsWritten = 0; + for (const job of jobs) { + const result = await runSupplementNutrientRecompute(job.data); + rowsWritten += result.rowsWritten; + } + return jobDone({ users: jobs.length, rows_written: rowsWritten }); +} + +/** + * Deletes an inflated cumulative-type personal record and re-runs detection + * silently, so the honestly derived best takes its place. + */ +export async function handleCumulativePrRederive( + jobs: Job[], +): Promise { + let rowsDeleted = 0; + let rowsReinserted = 0; + for (const job of jobs) { + const { userId } = job.data; + const summary = await runCumulativePrRederivationForUser(userId); + rowsDeleted += summary.rowsDeleted; + rowsReinserted += summary.rowsReinserted; + workerLog( + "info", + `[cumulative-pr-rederive] user=${userId} rowsDeleted=${summary.rowsDeleted} rowsReinserted=${summary.rowsReinserted}`, + ); + } + return jobDone({ + users: jobs.length, + rows_deleted: rowsDeleted, + rows_reinserted: rowsReinserted, + }); +} + +/** + * Collapses each completed day of per-sample high-frequency mean rows to its + * mean. Out of budget is a completed run: the folded days are committed and + * the nightly tick carries on from the first day this run did not reach. + */ +export async function handleMeanConsolidation( + jobs: Job[], +): Promise { + const shouldStop = jobBudget(jobs); + let daysTotal = 0; + let perSampleRowsTotal = 0; + let stoppedEarly = false; + for (const job of jobs) { + const { userId } = job.data; + const summary = await runMeanConsolidationForUser(userId, shouldStop); + const { daysConsolidated, perSampleRowsSoftDeleted } = summary; + stoppedEarly ||= summary.stoppedEarly; + daysTotal += daysConsolidated; + perSampleRowsTotal += perSampleRowsSoftDeleted; + workerLog( + "info", + `[mean-consolidation] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted}`, + ); + } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + per_sample_rows_soft_deleted: perSampleRowsTotal, + stopped_early: stoppedEarly, + }); +} + +/** + * Folds one account's dense-tier (HRV / HR / SpO2) samples older than the + * retention window to hourly means, keeping the in-window intra-day shape the + * Stress engine reads. Out of budget completes the job; the nightly tick + * continues from the first day this run did not reach. + */ +export async function handleDenseIntradayRetention( + jobs: Job[], +): Promise { + const shouldStop = jobBudget(jobs); + let daysTotal = 0; + let perSampleRowsTotal = 0; + let derivedRestingRowsTotal = 0; + let stoppedEarly = false; + for (const job of jobs) { + const { userId } = job.data; + try { + const summary = await runDenseIntradayRetentionForUser( + userId, + shouldStop, + ); + const { + daysConsolidated, + perSampleRowsSoftDeleted, + derivedRestingRowsUpserted, + } = summary; + stoppedEarly ||= summary.stoppedEarly; + daysTotal += daysConsolidated; + perSampleRowsTotal += perSampleRowsSoftDeleted; + derivedRestingRowsTotal += derivedRestingRowsUpserted; + workerLog( + "info", + `[dense-intraday-retention] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted} derivedRestingRowsUpserted=${derivedRestingRowsUpserted}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[dense-intraday-retention] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + per_sample_rows_soft_deleted: perSampleRowsTotal, + derived_resting_rows_upserted: derivedRestingRowsTotal, + stopped_early: stoppedEarly, + }); +} + +/** + * Reconstructs hourly means from the tombstones behind a pre-hourly daily row + * and retires the daily row. Out of budget fails the job so pg-boss retries + * (the boot send allows three), and the retry starts at the first day not yet + * rebuilt; the queue is `exclusive`, so it cannot hand itself on while active. + */ +export async function handleDenseIntradayHourlyRebuild( + jobs: Job[], +): Promise { + const shouldStop = jobBudget(jobs); + let stoppedEarly = false; + let daysRebuiltTotal = 0; + let hourlyRowsTotal = 0; + let dailyRowsRetiredTotal = 0; + let daysSkippedTotal = 0; + for (const job of jobs) { + const { userId } = job.data; + try { + const summary = await runDenseIntradayHourlyRebuildForUser( + userId, + shouldStop, + ); + const { + daysRebuilt, + hourlyRowsUpserted, + dailyRowsRetired, + daysSkippedNoTombstones, + } = summary; + stoppedEarly ||= summary.stoppedEarly; + daysRebuiltTotal += daysRebuilt; + hourlyRowsTotal += hourlyRowsUpserted; + dailyRowsRetiredTotal += dailyRowsRetired; + daysSkippedTotal += daysSkippedNoTombstones; + workerLog( + "info", + `[dense-intraday-hourly-rebuild] user=${userId} daysRebuilt=${daysRebuilt} hourlyRowsUpserted=${hourlyRowsUpserted} dailyRowsRetired=${dailyRowsRetired} daysSkippedNoTombstones=${daysSkippedNoTombstones}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[dense-intraday-hourly-rebuild] user=${userId} failed`, + err, + ); + throw err; + } + } + const did = { + users: jobs.length, + stopped_early: stoppedEarly, + days_rebuilt: daysRebuiltTotal, + hourly_rows_upserted: hourlyRowsTotal, + daily_rows_retired: dailyRowsRetiredTotal, + days_skipped_no_tombstones: daysSkippedTotal, + }; + if (stoppedEarly) { + return jobFailed( + "stopped before every day was rebuilt; the retry resumes", + undefined, + did, + ); + } + return jobDone(did); +} + +/** Folds the WEEK / MONTH / YEAR mood buckets the mood-entry hooks enqueue. */ +export async function handleMoodRollupRecompute( + jobs: Job[], +): Promise { + for (const job of jobs) { + const payload = job.data; + await recomputeUserMoodRollups(payload.userId, { + granularities: [payload.granularity], + from: new Date(payload.from), + to: new Date(payload.to), + }); + } + return jobDone({ buckets_recomputed: jobs.length }); +} + +/** + * The full mood fold of one account. Delete-then-refold, so the backfill + * cleans up any row minted under an older keying; the brief empty window + * reads correctly through the live fallback. + */ +export async function handleMoodRollupFullBackfill( + jobs: Job[], +): Promise { + let rowsUpsertedTotal = 0; + for (const job of jobs) { + const { userId } = job.data; + try { + const { rowsUpserted, durationMs } = await recomputeUserMoodRollups( + userId, + { replace: true }, + ); + rowsUpsertedTotal += rowsUpserted; + workerLog( + "info", + `[mood-rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[mood-rollup-full-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + users: jobs.length, + rows_upserted: rowsUpsertedTotal, + }); +} + +/** + * The nightly drain: the cumulative (SUM) drain, the daily-mean drain and the + * dense intra-day retention drain, in turn, on one budget. They are + * independent passes, so one failing does not stop the others; the first + * failure is reported once the tick is over and fails the job. A tick that ran + * out of budget hands the rest to a bounded follow-up. + */ +export function drainCumulativeHandler( + boss: PgBoss, +): (jobs: Job[]) => Promise { + return async (jobs) => { + const shouldStop = jobBudget(jobs); + let stoppedEarly = false; + let firstFailure: { reason: string; cause: unknown } | undefined; + let passesFailed = 0; + let bucketsCollapsed = 0; + let perSampleRowsDeleted = 0; + let meanDaysConsolidated = 0; + let meanRowsSoftDeleted = 0; + let denseDaysConsolidated = 0; + let denseRowsSoftDeleted = 0; + let denseSkipped = false; + + for (const job of jobs) { + try { + const summary = await drainPerSampleCumulative(getWorkerPrisma(), { + dryRun: false, + cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS, + shouldStop, + log: (line) => workerLog("info", line), + }); + stoppedEarly ||= summary.stoppedEarly; + bucketsCollapsed += summary.totals.bucketsCollapsed; + perSampleRowsDeleted += summary.totals.perSampleRowsDeleted; + workerLog( + "info", + `[drain-cumulative] triggeredAt=${job.data.triggeredAt} usersScanned=${summary.totals.usersScanned} bucketsCollapsed=${summary.totals.bucketsCollapsed} perSampleRowsDeleted=${summary.totals.perSampleRowsDeleted} dailyRowsUpserted=${summary.totals.dailyRowsUpserted}`, + ); + } catch (err) { + recordError(); + await reportWorkerError("drain-cumulative", err); + passesFailed++; + firstFailure ??= { reason: "cumulative drain failed", cause: err }; + } + + // The daily-mean drain rides the same tick, so spot samples between + // reboots do not accumulate raw; the 36-hour grace keeps today's + // in-flight watch syncs raw. + try { + const meanSummary = await consolidateDailyMean(getWorkerPrisma(), { + dryRun: false, + cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS, + shouldStop, + log: (line) => workerLog("info", line), + }); + stoppedEarly ||= meanSummary.stoppedEarly; + meanDaysConsolidated += meanSummary.totals.daysConsolidated; + meanRowsSoftDeleted += meanSummary.totals.perSampleRowsSoftDeleted; + workerLog( + "info", + `[mean-consolidation] triggeredAt=${job.data.triggeredAt} usersScanned=${meanSummary.totals.usersScanned} daysConsolidated=${meanSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${meanSummary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${meanSummary.totals.dailyRowsUpserted} daysFailed=${meanSummary.totals.daysFailed}`, + ); + } catch (err) { + recordError(); + await reportWorkerError("mean-consolidation", err, { + tick: "nightly", + }); + passesFailed++; + firstFailure ??= { reason: "mean consolidation failed", cause: err }; + } + + // The dense intra-day retention drain: HRV and pulse only, keeping the + // last DENSE_INTRADAY_RETENTION_DAYS raw for the Stress engine. These + // types are never in the destructive daily-mean allowlist. + try { + if (!DENSE_INTRADAY_RETENTION_ENABLED) { + denseSkipped = true; + workerLog( + "info", + "[dense-intraday-retention] disabled via DENSE_INTRADAY_RETENTION_ENABLED — skipping the nightly walk (operator kill-switch)", + ); + } else { + const denseSummary = await runDenseIntradayRetention( + getWorkerPrisma(), + { + dryRun: false, + shouldStop, + log: (line) => workerLog("info", line), + }, + ); + stoppedEarly ||= denseSummary.stoppedEarly; + denseDaysConsolidated += denseSummary.totals.daysConsolidated; + denseRowsSoftDeleted += denseSummary.totals.perSampleRowsSoftDeleted; + workerLog( + "info", + `[dense-intraday-retention] triggeredAt=${job.data.triggeredAt} usersScanned=${denseSummary.totals.usersScanned} daysConsolidated=${denseSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${denseSummary.totals.perSampleRowsSoftDeleted} hourlyRowsUpserted=${denseSummary.totals.hourlyRowsUpserted} dailyRowsRetired=${denseSummary.totals.dailyRowsRetired}`, + ); + } + } catch (err) { + recordError(); + await reportWorkerError("dense-intraday-retention", err, { + tick: "nightly", + }); + passesFailed++; + firstFailure ??= { reason: "dense retention failed", cause: err }; + } + } + + const continuation = Math.max( + 0, + ...jobs.map((job) => job.data.continuation ?? 0), + ); + const next = nextDrainContinuation({ + stoppedEarly, + daysFolded: + bucketsCollapsed + meanDaysConsolidated + denseDaysConsolidated, + continuation, + }); + let continued = false; + if (next !== null) { + try { + const followUp: DrainCumulativePayload = { + triggeredAt: new Date().toISOString(), + continuation: next, + }; + continued = + (await boss.send(DRAIN_CUMULATIVE_QUEUE, followUp, { + ...cronIsTheRetry, + startAfter: DRAIN_CONTINUATION_DELAY_SECONDS, + })) !== null; + } catch (err) { + // The next nightly tick picks the work up regardless; a lost + // follow-up only delays it. + workerLog("error", "[drain-cumulative] follow-up send failed", err); + } + } + + const did = { + stopped_early: stoppedEarly, + continuation, + continued, + passes_failed: passesFailed, + buckets_collapsed: bucketsCollapsed, + per_sample_rows_deleted: perSampleRowsDeleted, + mean_days_consolidated: meanDaysConsolidated, + mean_rows_soft_deleted: meanRowsSoftDeleted, + dense_days_consolidated: denseDaysConsolidated, + dense_rows_soft_deleted: denseRowsSoftDeleted, + dense_skipped: denseSkipped, + }; + + if (firstFailure) { + return jobFailed(firstFailure.reason, firstFailure.cause, did); + } + return jobDone(did); + }; +} diff --git a/src/lib/jobs/reminder/status-handlers.ts b/src/lib/jobs/reminder/status-handlers.ts new file mode 100644 index 000000000..501bf2601 --- /dev/null +++ b/src/lib/jobs/reminder/status-handlers.ts @@ -0,0 +1,394 @@ +/** + * Handlers of the status, score and Coach queues that are not a module's own + * export: the morning digest refresh, the nightly computed scores, the Coach + * ticks, the low-stock pass, the daily-briefing fallback, the period-narrative + * warm and the Coach memory refresh. The registry binds them + * (`registry/status.ts`) and loads this module on the first job. + */ +import type { Job } from "pg-boss"; + +import type { CoachMemoryRefreshPayload } from "@/lib/ai/coach/coach-memory-shared"; +import { runCoachMemoryRefresh } from "@/lib/ai/coach/coach-memory-refresh-worker"; +import { maybeDispatchDailyBriefing } from "@/lib/daily/daily-briefing-push"; +import { runCoachNudgeTick } from "@/lib/jobs/coach-nudge"; +import { runCoachPlanReviewTick } from "@/lib/jobs/coach-plan-review"; +import { runCoachReminderSweep } from "@/lib/jobs/coach-reminder-sweep"; +import { runDailyBriefingTick } from "@/lib/jobs/daily-briefing"; +import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { runMedicationLowStockTick } from "@/lib/jobs/medication-low-stock"; +import { + runMorningDigestRefresh, + type MorningDigestRefreshPayload, +} from "@/lib/jobs/morning-digest-refresh"; +import { + runPeriodNarrativeWarm, + warmOneNarrative, + type PeriodNarrativePayload, +} from "@/lib/jobs/period-narrative-warm"; +import { PERIOD_NARRATIVE_QUEUE } from "@/lib/jobs/period-narrative-queue"; +import { runRecoveryScore } from "@/lib/jobs/recovery-score"; +import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { runStrainScore } from "@/lib/jobs/strain-score"; +import { runStressScore } from "@/lib/jobs/stress-score"; +import { recordError } from "@/lib/jobs/worker-status"; +import { withBackgroundEvent } from "@/lib/logging/background"; + +import { getWorkerPrisma, workerLog } from "./shared"; + +/** + * One forced comprehensive regeneration per user per local morning, enqueued + * by the sleep-arrival trigger. A finalised day is the event-driven moment for + * the morning push carrying the final digest; the push seam owns its own + * opt-in, window and once-per-day gates, and the fallback cron covers the + * mornings no sleep arrives. + */ +export async function handleMorningDigestRefresh( + jobs: Job[], +): Promise { + return withBackgroundEvent("job.morning_digest_refresh", async (evt) => { + // The counts the batch already decides on: one refresh per job, and + // the finalised subset that reaches the morning push seam. + let refreshed = 0; + let finalised = 0; + for (const job of jobs) { + try { + const result = await runMorningDigestRefresh( + getWorkerPrisma(), + job.data, + ); + refreshed++; + evt.addMeta( + "morning_refresh", + `${result.status}:${result.comprehensive ?? "none"}`, + ); + if (result.status === "finalised") { + finalised++; + const pushResult = await maybeDispatchDailyBriefing( + getWorkerPrisma(), + job.data.userId, + new Date(), + ); + evt.addMeta("daily_briefing_push", pushResult); + } + } catch (err) { + recordError(); + workerLog("error", "[morning-digest-refresh] pass failed", err); + // Rethrow so the queue's retry policy re-runs the refresh; the + // marker re-check keeps a retry idempotent. + throw err; + } + } + return jobDone({ refreshed, finalised }); + }); +} + +interface ScoreSummary { + considered: number; + stored: number; + insufficient: number; + errored: number; +} + +/** + * A nightly computed-score pass: the cron tick carries an empty payload and + * the runner walks every eligible user, upserting one `COMPUTED` row per + * scored day (a re-fire overwrites in place). + */ +function scorePass( + name: string, + run: () => Promise, +): () => Promise { + return async () => { + try { + const summary = await run(); + workerLog( + "info", + `[${name}] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, + ); + return jobDone({ + considered: summary.considered, + stored: summary.stored, + insufficient: summary.insufficient, + errored: summary.errored, + }); + } catch (err) { + recordError(); + workerLog("error", `[${name}] pass failed`, err); + throw err; + } + }; +} + +export const handleRecoveryScore = scorePass("recovery-score", () => + runRecoveryScore(getWorkerPrisma()), +); +export const handleStressScore = scorePass("stress-score", () => + runStressScore(getWorkerPrisma()), +); +export const handleStrainScore = scorePass("strain-score", () => + runStrainScore(getWorkerPrisma()), +); + +/** + * The proactive Coach nudge. Deterministic triggers, no AI call; a per-user + * failure never fails the pass, it rides out as the `failed` count. + */ +export async function handleCoachNudge(): Promise { + return withBackgroundEvent("job.coach_nudge", async (evt) => { + try { + const summary = await runCoachNudgeTick(getWorkerPrisma(), new Date()); + evt.setBackground({ + task_name: "job.coach_nudge", + result: { + candidates_scanned: summary.candidatesScanned, + dispatched: summary.dispatched, + persisted: summary.persisted, + skipped_opted_out: summary.skippedOptedOut, + skipped_no_provider: summary.skippedNoProvider, + skipped_unavailable: summary.skippedUnavailable, + skipped_recent_nudge: summary.skippedRecentNudge, + skipped_recent_engagement: summary.skippedRecentEngagement, + skipped_no_trigger: summary.skippedNoTrigger, + skipped_no_channel: summary.skippedNoChannel, + failed: summary.failed, + }, + }); + return jobDone({ + candidates_scanned: summary.candidatesScanned, + dispatched: summary.dispatched, + persisted: summary.persisted, + failed: summary.failed, + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); +} + +/** + * The medication low-stock pass. The per-medication stamp makes a re-fire + * idempotent; a per-medication failure rides out as the `failed` count. + */ +export async function handleMedicationLowStock(): Promise { + return withBackgroundEvent("job.medication_low_stock", async (evt) => { + try { + const summary = await runMedicationLowStockTick( + getWorkerPrisma(), + new Date(), + ); + evt.setBackground({ + task_name: "job.medication_low_stock", + result: { + users_scanned: summary.usersScanned, + skipped_threshold_off: summary.skippedThresholdOff, + medications_evaluated: summary.medicationsEvaluated, + notified: summary.notified, + rearmed: summary.rearmed, + skipped_already_notified: summary.skippedAlreadyNotified, + skipped_above_threshold: summary.skippedAboveThreshold, + skipped_no_runway: summary.skippedNoRunway, + skipped_no_channel: summary.skippedNoChannel, + failed: summary.failed, + }, + }); + return jobDone({ + users_scanned: summary.usersScanned, + medications_evaluated: summary.medicationsEvaluated, + notified: summary.notified, + rearmed: summary.rearmed, + failed: summary.failed, + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); +} + +/** + * The daily-briefing fallback slot. Most ticks land outside every user's + * slot; that is the tick doing its job, so a zero `sent` is a success. + */ +export async function handleDailyBriefing(): Promise { + return withBackgroundEvent("job.daily_briefing", async (evt) => { + try { + const summary = await runDailyBriefingTick(getWorkerPrisma(), new Date()); + evt.setBackground({ + task_name: "job.daily_briefing", + result: { + candidates_scanned: summary.candidatesScanned, + in_slot: summary.inSlot, + sent: summary.sent, + suppressed_frequency: summary.suppressedFrequency, + no_digest: summary.noDigest, + opted_out: summary.optedOut, + no_channel: summary.noChannel, + outside_window: summary.outsideWindow, + failed: summary.failed, + }, + }); + return jobDone({ + candidates_scanned: summary.candidatesScanned, + in_slot: summary.inSlot, + sent: summary.sent, + failed: summary.failed, + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); +} + +/** + * The period-narrative warm: a scheduled tick (no `userId`) runs the + * boundary-gated nightly fan-out, a `userId` payload warms one account for + * the read-only GET. Two modes on one queue, so the outcome counts them + * apart: a non-boundary night generates nothing and is still a success. + */ +export async function handlePeriodNarrative( + jobs: Job[], +): Promise { + let singleUser = 0; + let scheduled = 0; + let generated = 0; + let failed = 0; + let stoppedEarly = false; + for (const job of jobs) { + try { + if (job.data?.userId) { + await warmOneNarrative(job.data); + singleUser++; + } else { + const summary = await runPeriodNarrativeWarm(getWorkerPrisma(), { + shouldStop: jobBudget(jobs), + }); + scheduled++; + generated += summary.generated; + failed += summary.failed; + stoppedEarly ||= summary.stoppedEarly; + workerLog( + "info", + `[period-narrative] periods=${summary.periods.join(",") || "none"} total=${summary.total} generated=${summary.generated} cached=${summary.cached} skipped=${summary.skipped} insufficient=${summary.insufficient} failed=${summary.failed} budget=${summary.budgetBlocked}`, + ); + } + } catch (err) { + recordError(); + await reportWorkerError(PERIOD_NARRATIVE_QUEUE, err, { + mode: job.data?.userId ? "single-user" : "scheduled", + }); + throw err; + } + } + return jobDone({ + single_user: singleUser, + scheduled, + generated, + failed, + stopped_early: stoppedEarly, + }); +} + +/** + * The Coach memory refresh for one long conversation. An incomplete payload + * is skipped rather than failed, and counted, so a queue full of them does + * not read as a clean run. + */ +export async function handleCoachMemoryRefresh( + jobs: Job[], +): Promise { + let refreshed = 0; + let skippedIncomplete = 0; + for (const job of jobs) { + if (!job.data?.conversationId || !job.data?.userId) { + skippedIncomplete++; + continue; + } + try { + await runCoachMemoryRefresh(job.data); + refreshed++; + } catch (err) { + recordError(); + workerLog("error", "[coach-memory-refresh] failed", err); + throw err; + } + } + return jobDone({ + refreshed, + skipped_incomplete: skippedIncomplete, + }); +} + +/** + * The daily Coach-reminder sweep. Surfacing an overdue reminder and minting + * plan-review reminders are both idempotent across re-fires; no provider call, + * no push. + */ +export async function handleCoachReminderSweep(): Promise { + return withBackgroundEvent("job.coach_reminder_sweep", async (evt) => { + try { + const summary = await runCoachReminderSweep( + getWorkerPrisma(), + new Date(), + ); + evt.setBackground({ + task_name: "job.coach_reminder_sweep", + result: { + reminders_due: summary.remindersDue, + plan_reviews_minted: summary.planReviewsMinted, + context_surfaced: summary.contextSurfaced, + nag_dismissed: summary.nagDismissed, + errored: summary.errored, + }, + }); + return jobDone({ + reminders_due: summary.remindersDue, + plan_reviews_minted: summary.planReviewsMinted, + context_surfaced: summary.contextSurfaced, + nag_dismissed: summary.nagDismissed, + errored: summary.errored, + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); +} + +/** + * The n-of-1 experiment review: writes the grounded before/after read-back + * into the plan's encrypted outcome. A reviewed plan is not re-processed. + */ +export async function handleCoachPlanReview(): Promise { + return withBackgroundEvent("job.coach_plan_review", async (evt) => { + try { + const summary = await runCoachPlanReviewTick( + getWorkerPrisma(), + new Date(), + ); + evt.setBackground({ + task_name: "job.coach_plan_review", + result: { + reviewed: summary.reviewed, + insufficient: summary.insufficient, + errored: summary.errored, + }, + }); + return jobDone({ + reviewed: summary.reviewed, + insufficient: summary.insufficient, + errored: summary.errored, + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); +} diff --git a/src/lib/jobs/restore-drill-queue.ts b/src/lib/jobs/restore-drill-queue.ts new file mode 100644 index 000000000..397def8b5 --- /dev/null +++ b/src/lib/jobs/restore-drill-queue.ts @@ -0,0 +1,23 @@ +/** + * The queue facts of `restore-drill.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const RESTORE_DRILL_QUEUE = "data-restore-drill"; + +export const RESTORE_DRILL_CRON = "11 4 1 * *"; + +/** + * Four hours, the off-host upload's own window: the drill downloads and + * stream-parses every object that upload wrote in one night. + */ +export const RESTORE_DRILL_EXPIRE_SECONDS = 4 * 60 * 60; + +/** + * What the monthly schedule sends with: the expiry above and no retry + * (`retryLimit: 0`, the maintenance schedules' `cronIsTheRetry`). + */ +export const RESTORE_DRILL_SEND_OPTIONS = { + expireInSeconds: RESTORE_DRILL_EXPIRE_SECONDS, + retryLimit: 0, +} as const; diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index 1a6854bc1..a2d38c476 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -48,24 +48,7 @@ import { reportWorkerError } from "@/lib/jobs/report-worker-error"; import { jobBudget } from "@/lib/jobs/job-budget"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; - -export const RESTORE_DRILL_QUEUE = "data-restore-drill"; -export const RESTORE_DRILL_CRON = "11 4 1 * *"; - -/** - * Four hours, the off-host upload's own window: the drill downloads and - * stream-parses every object that upload wrote in one night. - */ -export const RESTORE_DRILL_EXPIRE_SECONDS = 4 * 60 * 60; - -/** - * What the monthly schedule sends with: the expiry above and no retry - * (`retryLimit: 0`, the maintenance schedules' `cronIsTheRetry`). - */ -export const RESTORE_DRILL_SEND_OPTIONS = { - expireInSeconds: RESTORE_DRILL_EXPIRE_SECONDS, - retryLimit: 0, -} as const; +import { RESTORE_DRILL_QUEUE } from "./restore-drill-queue"; /** * A drill run that finds the newest backup older than this is reported diff --git a/src/lib/jobs/sleep-timeline-backfill-queue.ts b/src/lib/jobs/sleep-timeline-backfill-queue.ts new file mode 100644 index 000000000..c120b9920 --- /dev/null +++ b/src/lib/jobs/sleep-timeline-backfill-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `sleep-timeline-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const SLEEP_TIMELINE_BACKFILL_QUEUE = "sleep-timeline-backfill"; + +/** + * Serial concurrency — a pass deletes a source's sleep rows then walks a + * re-sync that is rate-bounded by the upstream API cap. Concurrency-1 keeps it + * from crowding the request pool, matching the other backfill queues. + */ +export const SLEEP_TIMELINE_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/sleep-timeline-backfill.ts b/src/lib/jobs/sleep-timeline-backfill.ts index 619a632da..2adbfa5d5 100644 --- a/src/lib/jobs/sleep-timeline-backfill.ts +++ b/src/lib/jobs/sleep-timeline-backfill.ts @@ -40,15 +40,7 @@ import { syncUserSleep as syncWithingsSleep, } from "@/lib/withings/sync-sleep"; import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; - -export const SLEEP_TIMELINE_BACKFILL_QUEUE = "sleep-timeline-backfill"; - -/** - * Serial concurrency — a pass deletes a source's sleep rows then walks a - * re-sync that is rate-bounded by the upstream API cap. Concurrency-1 keeps it - * from crowding the request pool, matching the other backfill queues. - */ -export const SLEEP_TIMELINE_BACKFILL_CONCURRENCY = 1; +import { SLEEP_TIMELINE_BACKFILL_QUEUE } from "./sleep-timeline-backfill-queue"; export type SleepTimelineProvider = "WHOOP" | "WITHINGS"; diff --git a/src/lib/jobs/strain-score-queue.ts b/src/lib/jobs/strain-score-queue.ts new file mode 100644 index 000000000..0417a725e --- /dev/null +++ b/src/lib/jobs/strain-score-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `strain-score.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const STRAIN_SCORE_QUEUE = "strain-score-compute"; + +/** + * Daily at 04:55 Europe/Berlin — inside the existing maintenance window, + * after the recovery-score (04:45) + stress-score (04:50) passes so the + * nightly score writes stay ordered and don't pile on one boss poll. + */ +export const STRAIN_SCORE_CRON = "55 4 * * *"; diff --git a/src/lib/jobs/strain-score.ts b/src/lib/jobs/strain-score.ts index 2d019fa05..afe09ddab 100644 --- a/src/lib/jobs/strain-score.ts +++ b/src/lib/jobs/strain-score.ts @@ -21,15 +21,6 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { persistStrainScore } from "@/lib/insights/strain-score"; import { runScoreBatch } from "@/lib/insights/score-row"; -export const STRAIN_SCORE_QUEUE = "strain-score-compute"; - -/** - * Daily at 04:55 Europe/Berlin — inside the existing maintenance window, - * after the recovery-score (04:45) + stress-score (04:50) passes so the - * nightly score writes stay ordered and don't pile on one boss poll. - */ -export const STRAIN_SCORE_CRON = "55 4 * * *"; - /** Per-run user cap — same starvation guard the other score passes use. */ export const STRAIN_SCORE_BATCH_CAP = 500; diff --git a/src/lib/jobs/strava-backfill-queue.ts b/src/lib/jobs/strava-backfill-queue.ts new file mode 100644 index 000000000..db638639f --- /dev/null +++ b/src/lib/jobs/strava-backfill-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `strava-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const STRAVA_BACKFILL_QUEUE = "strava-backfill"; + +/** + * Serial concurrency — a backfill walks the account's activity history and is + * rate-bounded by Strava's 200-req / 15-min cap; concurrency-1 keeps it from + * crowding the request pool, matching `WHOOP_BACKFILL_CONCURRENCY`. + */ +export const STRAVA_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/strava-backfill.ts b/src/lib/jobs/strava-backfill.ts index 10bb6dd21..a18a85d73 100644 --- a/src/lib/jobs/strava-backfill.ts +++ b/src/lib/jobs/strava-backfill.ts @@ -27,15 +27,7 @@ import { withPageWalkLedger, } from "@/lib/integrations/page-walk"; import { syncStravaHistory } from "@/lib/strava/sync"; - -export const STRAVA_BACKFILL_QUEUE = "strava-backfill"; - -/** - * Serial concurrency — a backfill walks the account's activity history and is - * rate-bounded by Strava's 200-req / 15-min cap; concurrency-1 keeps it from - * crowding the request pool, matching `WHOOP_BACKFILL_CONCURRENCY`. - */ -export const STRAVA_BACKFILL_CONCURRENCY = 1; +import { STRAVA_BACKFILL_QUEUE } from "./strava-backfill-queue"; export interface StravaBackfillPayload { userId: string; diff --git a/src/lib/jobs/stress-score-queue.ts b/src/lib/jobs/stress-score-queue.ts new file mode 100644 index 000000000..70261c31a --- /dev/null +++ b/src/lib/jobs/stress-score-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `stress-score.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const STRESS_SCORE_QUEUE = "stress-score-compute"; + +/** + * Daily at 04:50 Europe/Berlin — inside the existing maintenance window, + * after the dense intra-day retention drain (03:50) and the recovery-score + * pass (04:45) so the HRV inputs it reads are settled. + */ +export const STRESS_SCORE_CRON = "50 4 * * *"; diff --git a/src/lib/jobs/stress-score.ts b/src/lib/jobs/stress-score.ts index 04b25cb1d..8a73866bf 100644 --- a/src/lib/jobs/stress-score.ts +++ b/src/lib/jobs/stress-score.ts @@ -22,15 +22,6 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { persistStressScore } from "@/lib/insights/stress-score"; import { runScoreBatch } from "@/lib/insights/score-row"; -export const STRESS_SCORE_QUEUE = "stress-score-compute"; - -/** - * Daily at 04:50 Europe/Berlin — inside the existing maintenance window, - * after the dense intra-day retention drain (03:50) and the recovery-score - * pass (04:45) so the HRV inputs it reads are settled. - */ -export const STRESS_SCORE_CRON = "50 4 * * *"; - /** Per-run user cap — same starvation guard the recovery-score pass uses. */ export const STRESS_SCORE_BATCH_CAP = 500; diff --git a/src/lib/jobs/tls-pin-monitor-queue.ts b/src/lib/jobs/tls-pin-monitor-queue.ts new file mode 100644 index 000000000..f08599e58 --- /dev/null +++ b/src/lib/jobs/tls-pin-monitor-queue.ts @@ -0,0 +1,15 @@ +/** + * The queue facts of `tls-pin-monitor.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +/** + * pg-boss queue name + cron. Every 6 hours at :07 (offset off the :00 / :05 + * / :08 hourly sync crons so the probe doesn't pile onto a busy boss poll). + * GTS leaves renew roughly every ~90 days; a 6-hour cadence surfaces a + * change well inside the ≥11-day re-pin window the runbook targets while + * costing one short outbound TLS handshake per tick. + */ +export const TLS_PIN_MONITOR_QUEUE = "tls-pin-monitor"; + +export const TLS_PIN_MONITOR_CRON = "7 */6 * * *"; diff --git a/src/lib/jobs/tls-pin-monitor.ts b/src/lib/jobs/tls-pin-monitor.ts index 3c1c5e394..16cc93c10 100644 --- a/src/lib/jobs/tls-pin-monitor.ts +++ b/src/lib/jobs/tls-pin-monitor.ts @@ -40,16 +40,6 @@ import { type LeafProbeResult, } from "@/lib/tls/leaf-spki"; -/** - * pg-boss queue name + cron. Every 6 hours at :07 (offset off the :00 / :05 - * / :08 hourly sync crons so the probe doesn't pile onto a busy boss poll). - * GTS leaves renew roughly every ~90 days; a 6-hour cadence surfaces a - * change well inside the ≥11-day re-pin window the runbook targets while - * costing one short outbound TLS handshake per tick. - */ -export const TLS_PIN_MONITOR_QUEUE = "tls-pin-monitor"; -export const TLS_PIN_MONITOR_CRON = "7 */6 * * *"; - export interface TlsPinMonitorSummary { /** "ok" served pin is in the known set; "changed" it is not; "skipped" no target / no baseline; "probe_failed" the TLS probe threw. */ outcome: "ok" | "changed" | "skipped" | "probe_failed"; diff --git a/src/lib/jobs/whoop-backfill-queue.ts b/src/lib/jobs/whoop-backfill-queue.ts new file mode 100644 index 000000000..21ea1793e --- /dev/null +++ b/src/lib/jobs/whoop-backfill-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `whoop-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const WHOOP_BACKFILL_QUEUE = "whoop-backfill"; + +/** + * Serial concurrency — a backfill walks years of history for one account and + * is rate-bounded by WHOOP's 100 req/min cap; concurrency-1 keeps it from + * crowding the request pool, matching `ROLLUP_FULL_BACKFILL_CONCURRENCY`. + */ +export const WHOOP_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/whoop-backfill.ts b/src/lib/jobs/whoop-backfill.ts index 83d87ad82..d61822d8a 100644 --- a/src/lib/jobs/whoop-backfill.ts +++ b/src/lib/jobs/whoop-backfill.ts @@ -26,15 +26,7 @@ import { withPageWalkLedger, } from "@/lib/integrations/page-walk"; import { syncUserWhoop } from "@/lib/whoop/sync"; - -export const WHOOP_BACKFILL_QUEUE = "whoop-backfill"; - -/** - * Serial concurrency — a backfill walks years of history for one account and - * is rate-bounded by WHOOP's 100 req/min cap; concurrency-1 keeps it from - * crowding the request pool, matching `ROLLUP_FULL_BACKFILL_CONCURRENCY`. - */ -export const WHOOP_BACKFILL_CONCURRENCY = 1; +import { WHOOP_BACKFILL_QUEUE } from "./whoop-backfill-queue"; export interface WhoopBackfillPayload { userId: string; diff --git a/src/lib/medications/intake-import-admission.ts b/src/lib/medications/intake-import-admission.ts index 591140da9..21a487b49 100644 --- a/src/lib/medications/intake-import-admission.ts +++ b/src/lib/medications/intake-import-admission.ts @@ -23,12 +23,12 @@ */ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { - MEDICATION_INTAKE_IMPORT_QUEUE, MEDICATION_INTAKE_IMPORT_SEND_OPTIONS, MEDICATION_INTAKE_IMPORT_STALE_AFTER_MS, type MedicationImportPayload, type MedicationImportProgress, } from "@/lib/jobs/medication-intake-import"; +import { MEDICATION_INTAKE_IMPORT_QUEUE } from "@/lib/jobs/medication-intake-import-queue"; import { auditLog } from "@/lib/auth/audit"; import { prisma, toJson } from "@/lib/db"; diff --git a/src/lib/support/report.ts b/src/lib/support/report.ts index d186ee20b..fbbfa8fa3 100644 --- a/src/lib/support/report.ts +++ b/src/lib/support/report.ts @@ -38,12 +38,10 @@ import { PROBLEMS_BUFFER_MAX, } from "@/lib/logging/in-memory-buffer"; import type { WideEvent } from "@/lib/logging/types"; -import { - offhostBackupConfigured, - OFFHOST_BACKUP_QUEUE, -} from "@/lib/jobs/offhost-backup"; +import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; +import { OFFHOST_BACKUP_QUEUE } from "@/lib/jobs/offhost-backup-queue"; import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; -import { RESTORE_DRILL_QUEUE } from "@/lib/jobs/restore-drill"; +import { RESTORE_DRILL_QUEUE } from "@/lib/jobs/restore-drill-queue"; import { JOB_FAILURE_WINDOW_HOURS, readFailingQueues, diff --git a/src/lib/whoop/webhook-handler.ts b/src/lib/whoop/webhook-handler.ts index a815cabba..a3c05e829 100644 --- a/src/lib/whoop/webhook-handler.ts +++ b/src/lib/whoop/webhook-handler.ts @@ -34,6 +34,11 @@ import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; import type { MeasurementType, Prisma } from "@/generated/prisma/client"; import { WHOOP_RECOVERY_FIELD_TAGS } from "./client"; +import { + WHOOP_RECOVERY_SYNC_QUEUE, + WHOOP_SLEEP_SYNC_QUEUE, + WHOOP_WORKOUT_SYNC_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; /** * Maximum age of a webhook timestamp before it is treated as a replay. Five @@ -48,9 +53,9 @@ const WHOOP_SIGNATURE_MAX_AGE_MS = 5 * 60 * 1000; * webhook (poll-only) so it is intentionally absent. */ const WHOOP_RESOURCE_QUEUE: Record = { - recovery: "whoop-recovery-sync", - sleep: "whoop-sleep-sync", - workout: "whoop-workout-sync", + recovery: WHOOP_RECOVERY_SYNC_QUEUE, + sleep: WHOOP_SLEEP_SYNC_QUEUE, + workout: WHOOP_WORKOUT_SYNC_QUEUE, }; /** diff --git a/src/lib/withings/webhook-handler.ts b/src/lib/withings/webhook-handler.ts index 0405653bf..f831c98ca 100644 --- a/src/lib/withings/webhook-handler.ts +++ b/src/lib/withings/webhook-handler.ts @@ -31,6 +31,10 @@ import { getClientIp, safeJson } from "@/lib/api-response"; import { annotate, getEvent } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { enqueueWithingsEcgSync } from "@/lib/jobs/withings-ecg-queue"; +import { + WITHINGS_ACTIVITY_QUEUE, + WITHINGS_SLEEP_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; /** * v1.4.25 — appli categories that have their own sync routine @@ -41,8 +45,6 @@ import { enqueueWithingsEcgSync } from "@/lib/jobs/withings-ecg-queue"; */ const WITHINGS_ACTIVITY_APPLI = 16; const WITHINGS_SLEEP_APPLI = 44; -const WITHINGS_ACTIVITY_QUEUE = "withings-activity-sync"; -const WITHINGS_SLEEP_QUEUE = "withings-sleep-sync"; /** * Constant-time comparison helper. Returns false unless both inputs have diff --git a/tests/integration/backup-restore-job.test.ts b/tests/integration/backup-restore-job.test.ts index 670143467..1902b918a 100644 --- a/tests/integration/backup-restore-job.test.ts +++ b/tests/integration/backup-restore-job.test.ts @@ -44,12 +44,12 @@ import { setGlobalBoss } from "@/lib/jobs/boss-instance"; import { runJob } from "@/lib/jobs/run-job"; import { BACKUP_RESTORE_MAX_ATTEMPTS, - BACKUP_RESTORE_QUEUE, handleBackupRestore, readBackupRestoreJob, runBackupRestoreJob, sweepInterruptedRestores, } from "@/lib/jobs/backup-restore"; +import { BACKUP_RESTORE_QUEUE } from "@/lib/jobs/backup-restore-queue"; import { backupPayloadSchema } from "@/lib/validations/backup"; import { cookieJar, headerJar } from "./mock-next-headers"; diff --git a/tests/integration/health-connect-import.test.ts b/tests/integration/health-connect-import.test.ts index e7dceb5bd..3c965acac 100644 --- a/tests/integration/health-connect-import.test.ts +++ b/tests/integration/health-connect-import.test.ts @@ -34,9 +34,9 @@ import { importHealthConnectExport } from "@/lib/import/health-connect/import"; import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention"; import { HEALTH_CONNECT_IMPORT_KIND, - HEALTH_CONNECT_IMPORT_QUEUE, runHealthConnectImport, } from "@/lib/jobs/health-connect-import-worker"; +import { HEALTH_CONNECT_IMPORT_QUEUE } from "@/lib/jobs/health-connect-import-queue"; import { _setWorkerPrismaForTests, reconcileOrphanImportJobs, diff --git a/tests/integration/queue-registration-parity.test.ts b/tests/integration/queue-registration-parity.test.ts index 40ff5d72b..544432dc0 100644 --- a/tests/integration/queue-registration-parity.test.ts +++ b/tests/integration/queue-registration-parity.test.ts @@ -71,21 +71,8 @@ function recordingBoss(real: PgBoss): PgBoss { } async function registerEveryQueue(target: PgBoss): Promise { - const [integration, status, rollup, reminders, maintenance] = - await Promise.all([ - import("@/lib/jobs/reminder/register-integration-sync"), - import("@/lib/jobs/reminder/register-status"), - import("@/lib/jobs/reminder/register-rollup"), - import("@/lib/jobs/reminder/register-reminders"), - import("@/lib/jobs/reminder/register-maintenance"), - ]); - returned.push( - ...(await integration.registerIntegrationSyncQueues(target)), - ...(await status.registerStatusQueues(target)), - ...(await rollup.registerRollupQueues(target)), - ...(await reminders.registerReminderQueues(target)), - ...(await maintenance.registerMaintenanceQueues(target)), - ); + const { registerAllJobGroups } = await import("@/lib/jobs/registry/index"); + returned.push(...(await registerAllJobGroups(target))); } beforeAll(async () => { @@ -142,4 +129,18 @@ describe("queue registration", () => { // golden. expect(returned.length).toBeGreaterThan(100); }); + + it("loads a function for every handler it declares", async () => { + // A handler module loads on its queue's first job; a broken import would + // otherwise surface only when that job arrives in production. + const { allJobDefinitions } = await import("@/lib/jobs/registry/index"); + const definitions = allJobDefinitions(); + expect(definitions.length).toBe(returned.length); + const unloaded: string[] = []; + for (const definition of definitions) { + const handler = await definition.load({ boss }); + if (typeof handler !== "function") unloaded.push(definition.queue); + } + expect(unloaded).toEqual([]); + }); }); diff --git a/tests/integration/routes/auth-me-documents-auto-ai-read.test.ts b/tests/integration/routes/auth-me-documents-auto-ai-read.test.ts index 6071eb8e2..fec91f067 100644 --- a/tests/integration/routes/auth-me-documents-auto-ai-read.test.ts +++ b/tests/integration/routes/auth-me-documents-auto-ai-read.test.ts @@ -27,10 +27,8 @@ import type { Prisma, User } from "@/generated/prisma/client"; import { GET, PATCH } from "@/app/api/auth/me/documents-auto-ai-read/route"; import { aiCapabilityForJob } from "@/lib/ai/capabilities/gate"; import { setGlobalBoss } from "@/lib/jobs/boss-instance"; -import { - DOCUMENT_SUMMARY_CATCHUP_QUEUE, - enqueueSummaryCatchUp, -} from "@/lib/jobs/document-summary-catchup"; +import { enqueueSummaryCatchUp } from "@/lib/jobs/document-summary-catchup"; +import { DOCUMENT_SUMMARY_CATCHUP_QUEUE } from "@/lib/jobs/document-summary-catchup-queue"; import { anonymous, callRoute } from "../../kit/call-route"; import { makeSession, makeUser, uniqueSuffix } from "../../kit/factories";