From 4807a70c1d6e233729019bf8d87e02aea52a48cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Fri, 23 Oct 2026 12:36:09 +0200 Subject: [PATCH] refactor(jobs): declare every queue in a typed registry and load handlers on first job The five hand-written registrars (about 5,500 lines, one 997-line function) are replaced by `jobs/registry/`: one `defineJob` entry per queue with its cron, schedule send options, policy, work options, lock, lane and a handler loader. `registerJobGroup` creates, schedules, retires and binds each group in the order the registrars used; the worker boot walks the groups and runs their boot discovery. Handlers load on their queue's first job. The facts a declaration needs at boot (names, crons, concurrency, send options) moved out of the heavy job modules into light `*-queue.ts` siblings, and their importers point there, so loading the registry evaluates 288 modules instead of the 1,164 the registrars pulled in. The formerly inline handlers moved verbatim into handler modules under `jobs/reminder/`. Fitbit's queues live in `fitbit/jobs/definitions.ts` and enter the registry through one spread, so removing the integration later is that folder and that line. `tests/integration/queue-registration-parity.test.ts` proves pg-boss holds exactly what the registrars left (queues, schedules, bindings, byte for byte) and loads every handler once. The queue tests and the runtime guard assert on the registry objects instead of reading registrar source. --- .../baselines/source-file-lines.json | 21 +- .../baselines/source-function-lines.json | 11 +- .../baselines/test-source-reads.json | 22 +- .../baselines/version-stamp-comments.json | 42 +- src/__tests__/guard-floor-guard.test.ts | 2 - src/__tests__/helpers/source-metrics.ts | 42 +- src/__tests__/job-queue-runtime-guard.test.ts | 305 +-- src/__tests__/moodlog-removal-guard.test.ts | 12 +- src/app/api/admin/backups/route.ts | 2 +- src/app/api/admin/encryption/rotate/route.ts | 2 +- .../admin/import-apple-health-export/route.ts | 2 +- src/app/api/fitbit/callback/route.ts | 2 +- src/app/api/google-health/callback/route.ts | 2 +- .../api/import/apple-health-export/route.ts | 2 +- .../api/import/health-connect-export/route.ts | 2 +- .../intake/import/__tests__/route.test.ts | 6 +- src/app/api/oura/callback/route.ts | 6 +- src/app/api/strava/callback/route.ts | 2 +- src/app/api/whoop/callback/route.ts | 2 +- src/lib/fitbit/jobs/definitions.ts | 98 + src/lib/fitbit/jobs/queues.ts | 30 + .../apple-health-import-legacy-bridge.test.ts | 2 +- .../apple-health-import-reconcile.test.ts | 51 +- .../apple-health-import-send-policy.test.ts | 47 +- .../coach-memory-refresh-queue.test.ts | 58 +- .../coach-message-cleanup-queue.test.ts | 68 +- .../cumulative-pr-rederive-queue.test.ts | 64 +- .../__tests__/cycle-reminder-queue.test.ts | 80 +- src/lib/jobs/__tests__/daily-briefing.test.ts | 96 +- .../jobs/__tests__/data-arrival-queue.test.ts | 86 +- .../__tests__/data-backup-send-policy.test.ts | 16 +- .../jobs/__tests__/document-ai-run.test.ts | 2 +- src/lib/jobs/__tests__/document-index.test.ts | 7 +- .../document-summary-catchup.test.ts | 2 +- .../__tests__/drain-cumulative-queue.test.ts | 173 +- src/lib/jobs/__tests__/fitbit-queues.test.ts | 152 +- src/lib/jobs/__tests__/geo-backfill.test.ts | 35 +- .../__tests__/geolite2-fetch-queue.test.ts | 70 +- .../__tests__/google-health-queues.test.ts | 128 +- .../__tests__/insight-pregenerate.test.ts | 51 +- .../__tests__/insight-status-generate.test.ts | 48 +- .../jobs/__tests__/intake-auto-skip.test.ts | 6 +- .../jobs/__tests__/intake-auto-take.test.ts | 6 +- .../__tests__/intake-slot-dedup-queue.test.ts | 161 +- .../integration-backfill-admission.test.ts | 188 +- .../__tests__/mcp-token-cleanup-queue.test.ts | 66 +- .../mean-consolidation-queue.test.ts | 62 +- .../measurement-reminder-queue.test.ts | 115 +- .../measurement-tombstone-cleanup.test.ts | 59 +- .../__tests__/mood-prognosis-refresh.test.ts | 6 +- .../jobs/__tests__/nightscout-queues.test.ts | 114 +- .../__tests__/period-narrative-warm.test.ts | 41 +- .../__tests__/queue-policy-tables.test.ts | 184 +- .../__tests__/read-side-effect-queues.test.ts | 69 +- .../__tests__/recovery-score-queue.test.ts | 54 +- src/lib/jobs/__tests__/restore-drill.test.ts | 5 +- .../__tests__/sleep-timeline-queues.test.ts | 82 +- .../__tests__/status-cron-candidates.test.ts | 47 +- .../step-consolidation-repair-queue.test.ts | 67 +- .../stress-strain-retention-queue.test.ts | 289 ++- .../jobs/__tests__/subsystem-surface.test.ts | 109 +- .../__tests__/tls-pin-monitor-queue.test.ts | 73 +- src/lib/jobs/__tests__/whoop-queues.test.ts | 173 +- .../__tests__/withings-ecg-worker.test.ts | 34 +- .../jobs/__tests__/withings-queues.test.ts | 105 +- .../__tests__/workout-insight-queue.test.ts | 51 +- .../jobs/achievement-unlock-sweep-queue.ts | 8 + src/lib/jobs/achievement-unlock-sweep.ts | 4 - src/lib/jobs/apple-health-import-queue.ts | 36 + src/lib/jobs/apple-health-import-worker.ts | 39 +- src/lib/jobs/backup-restore-queue.ts | 6 + src/lib/jobs/backup-restore.ts | 3 +- src/lib/jobs/coach-nudge-queue.ts | 10 + src/lib/jobs/coach-nudge.ts | 5 - src/lib/jobs/coach-reminder-sweep-queue.ts | 10 + src/lib/jobs/coach-reminder-sweep.ts | 5 - .../jobs/compaction-tombstone-purge-queue.ts | 6 + src/lib/jobs/compaction-tombstone-purge.ts | 3 +- src/lib/jobs/daily-briefing-queue.ts | 9 + src/lib/jobs/daily-briefing.ts | 4 - src/lib/jobs/dawarich-sync-queue.ts | 9 + src/lib/jobs/dawarich-sync.ts | 6 +- .../dense-intraday-hourly-rebuild-queue.ts | 14 + src/lib/jobs/dense-intraday-hourly-rebuild.ts | 11 +- .../jobs/dense-intraday-retention-queue.ts | 14 + src/lib/jobs/dense-intraday-retention.ts | 11 +- src/lib/jobs/document-ai-run-queue.ts | 9 + src/lib/jobs/document-ai-run-reaper-queue.ts | 9 + src/lib/jobs/document-ai-run-reaper.ts | 4 - src/lib/jobs/document-ai-run.ts | 6 +- .../document-content-index-backfill-queue.ts | 9 + .../jobs/document-content-index-backfill.ts | 6 +- src/lib/jobs/document-index-queue.ts | 9 + src/lib/jobs/document-index.ts | 6 +- .../jobs/document-summary-catchup-queue.ts | 9 + src/lib/jobs/document-summary-catchup.ts | 6 +- src/lib/jobs/document-summary-queue.ts | 9 + src/lib/jobs/document-summary-reaper-queue.ts | 9 + src/lib/jobs/document-summary-reaper.ts | 4 - src/lib/jobs/document-summary.ts | 6 +- .../jobs/document-thumbnail-backfill-queue.ts | 9 + src/lib/jobs/document-thumbnail-backfill.ts | 6 +- src/lib/jobs/document-thumbnail-queue.ts | 9 + src/lib/jobs/document-thumbnail.ts | 6 +- src/lib/jobs/drain-cumulative-queue.ts | 20 + src/lib/jobs/encryption-key-rotate-queue.ts | 8 + src/lib/jobs/encryption-key-rotate.ts | 3 - src/lib/jobs/fitbit-backfill.ts | 10 +- src/lib/jobs/fitbit-sleep-repair.ts | 10 +- src/lib/jobs/google-health-backfill-queue.ts | 13 + src/lib/jobs/google-health-backfill.ts | 10 +- .../jobs/google-health-sleep-repair-queue.ts | 13 + src/lib/jobs/google-health-sleep-repair.ts | 10 +- src/lib/jobs/health-connect-import-queue.ts | 9 + src/lib/jobs/health-connect-import-worker.ts | 5 - src/lib/jobs/insight-pregenerate-shared.ts | 7 + src/lib/jobs/insight-pregenerate.ts | 7 - src/lib/jobs/insight-status-generate-queue.ts | 13 + .../jobs/insight-status-generate-shared.ts | 10 +- src/lib/jobs/insight-status-generate.ts | 6 +- src/lib/jobs/intake-auto-skip-queue.ts | 13 + src/lib/jobs/intake-auto-skip.ts | 9 - src/lib/jobs/intake-auto-take-queue.ts | 12 + src/lib/jobs/intake-auto-take.ts | 8 - src/lib/jobs/integration-backfill-forward.ts | 27 + src/lib/jobs/integration-poll-queues.ts | 88 +- src/lib/jobs/integration-watchdog-queue.ts | 18 + src/lib/jobs/integration-watchdog.ts | 12 - src/lib/jobs/measurement-fold-repair-queue.ts | 6 + src/lib/jobs/measurement-fold-repair.ts | 9 +- .../jobs/medication-intake-import-queue.ts | 8 + src/lib/jobs/medication-intake-import.ts | 3 - src/lib/jobs/medication-low-stock-queue.ts | 14 + src/lib/jobs/medication-low-stock.ts | 9 - src/lib/jobs/mood-prognosis-refresh-queue.ts | 8 + src/lib/jobs/mood-prognosis-refresh.ts | 4 - src/lib/jobs/offhost-backup-queue.ts | 27 + src/lib/jobs/offhost-backup.ts | 23 +- src/lib/jobs/offhost-purge-queue.ts | 9 + src/lib/jobs/offhost-purge.ts | 5 +- src/lib/jobs/oura-backfill-queue.ts | 9 + src/lib/jobs/oura-backfill.ts | 6 +- src/lib/jobs/period-narrative-queue.ts | 24 + src/lib/jobs/period-narrative-shared.ts | 24 +- src/lib/jobs/period-narrative-warm.ts | 6 +- src/lib/jobs/queue-runtime.ts | 52 +- src/lib/jobs/recovery-score-queue.ts | 14 + src/lib/jobs/recovery-score.ts | 10 - .../__tests__/registry-contract.test.ts | 129 + .../registry/__tests__/registry-helpers.ts | 61 + src/lib/jobs/registry/define-job.ts | 109 + src/lib/jobs/registry/index.ts | 55 + src/lib/jobs/registry/integration-sync.ts | 391 +++ .../jobs/registry/maintenance-discovery.ts | 100 + src/lib/jobs/registry/maintenance.ts | 744 ++++++ src/lib/jobs/registry/register.ts | 146 ++ src/lib/jobs/registry/reminders.ts | 70 + src/lib/jobs/registry/rollup.ts | 281 +++ src/lib/jobs/registry/status.ts | 260 ++ src/lib/jobs/reminder-worker.ts | 109 +- .../jobs/reminder/climb-backfill-at-boot.ts | 34 + src/lib/jobs/reminder/document-handlers.ts | 233 ++ .../reminder/encryption-backfill-handlers.ts | 214 ++ src/lib/jobs/reminder/maintenance-handlers.ts | 186 ++ .../reminder/register-integration-sync.ts | 1051 -------- src/lib/jobs/reminder/register-maintenance.ts | 2197 ----------------- src/lib/jobs/reminder/register-reminders.ts | 180 -- src/lib/jobs/reminder/register-rollup.ts | 1156 --------- src/lib/jobs/reminder/register-status.ts | 878 ------- src/lib/jobs/reminder/rollup-handlers.ts | 582 +++++ src/lib/jobs/reminder/status-handlers.ts | 394 +++ src/lib/jobs/restore-drill-queue.ts | 23 + src/lib/jobs/restore-drill.ts | 19 +- src/lib/jobs/sleep-timeline-backfill-queue.ts | 13 + src/lib/jobs/sleep-timeline-backfill.ts | 10 +- src/lib/jobs/strain-score-queue.ts | 13 + src/lib/jobs/strain-score.ts | 9 - src/lib/jobs/strava-backfill-queue.ts | 13 + src/lib/jobs/strava-backfill.ts | 10 +- src/lib/jobs/stress-score-queue.ts | 13 + src/lib/jobs/stress-score.ts | 9 - src/lib/jobs/tls-pin-monitor-queue.ts | 15 + src/lib/jobs/tls-pin-monitor.ts | 10 - src/lib/jobs/whoop-backfill-queue.ts | 13 + src/lib/jobs/whoop-backfill.ts | 10 +- .../medications/intake-import-admission.ts | 2 +- src/lib/support/report.ts | 8 +- src/lib/whoop/webhook-handler.ts | 11 +- src/lib/withings/webhook-handler.ts | 6 +- tests/integration/backup-restore-job.test.ts | 2 +- .../integration/health-connect-import.test.ts | 2 +- .../queue-registration-parity.test.ts | 31 +- .../auth-me-documents-auto-ai-read.test.ts | 6 +- 193 files changed, 6830 insertions(+), 7950 deletions(-) create mode 100644 src/lib/fitbit/jobs/definitions.ts create mode 100644 src/lib/fitbit/jobs/queues.ts create mode 100644 src/lib/jobs/achievement-unlock-sweep-queue.ts create mode 100644 src/lib/jobs/apple-health-import-queue.ts create mode 100644 src/lib/jobs/backup-restore-queue.ts create mode 100644 src/lib/jobs/coach-nudge-queue.ts create mode 100644 src/lib/jobs/coach-reminder-sweep-queue.ts create mode 100644 src/lib/jobs/compaction-tombstone-purge-queue.ts create mode 100644 src/lib/jobs/daily-briefing-queue.ts create mode 100644 src/lib/jobs/dawarich-sync-queue.ts create mode 100644 src/lib/jobs/dense-intraday-hourly-rebuild-queue.ts create mode 100644 src/lib/jobs/dense-intraday-retention-queue.ts create mode 100644 src/lib/jobs/document-ai-run-queue.ts create mode 100644 src/lib/jobs/document-ai-run-reaper-queue.ts create mode 100644 src/lib/jobs/document-content-index-backfill-queue.ts create mode 100644 src/lib/jobs/document-index-queue.ts create mode 100644 src/lib/jobs/document-summary-catchup-queue.ts create mode 100644 src/lib/jobs/document-summary-queue.ts create mode 100644 src/lib/jobs/document-summary-reaper-queue.ts create mode 100644 src/lib/jobs/document-thumbnail-backfill-queue.ts create mode 100644 src/lib/jobs/document-thumbnail-queue.ts create mode 100644 src/lib/jobs/drain-cumulative-queue.ts create mode 100644 src/lib/jobs/encryption-key-rotate-queue.ts create mode 100644 src/lib/jobs/google-health-backfill-queue.ts create mode 100644 src/lib/jobs/google-health-sleep-repair-queue.ts create mode 100644 src/lib/jobs/health-connect-import-queue.ts create mode 100644 src/lib/jobs/insight-status-generate-queue.ts create mode 100644 src/lib/jobs/intake-auto-skip-queue.ts create mode 100644 src/lib/jobs/intake-auto-take-queue.ts create mode 100644 src/lib/jobs/integration-backfill-forward.ts create mode 100644 src/lib/jobs/integration-watchdog-queue.ts create mode 100644 src/lib/jobs/measurement-fold-repair-queue.ts create mode 100644 src/lib/jobs/medication-intake-import-queue.ts create mode 100644 src/lib/jobs/medication-low-stock-queue.ts create mode 100644 src/lib/jobs/mood-prognosis-refresh-queue.ts create mode 100644 src/lib/jobs/offhost-backup-queue.ts create mode 100644 src/lib/jobs/offhost-purge-queue.ts create mode 100644 src/lib/jobs/oura-backfill-queue.ts create mode 100644 src/lib/jobs/period-narrative-queue.ts create mode 100644 src/lib/jobs/recovery-score-queue.ts create mode 100644 src/lib/jobs/registry/__tests__/registry-contract.test.ts create mode 100644 src/lib/jobs/registry/__tests__/registry-helpers.ts create mode 100644 src/lib/jobs/registry/define-job.ts create mode 100644 src/lib/jobs/registry/index.ts create mode 100644 src/lib/jobs/registry/integration-sync.ts create mode 100644 src/lib/jobs/registry/maintenance-discovery.ts create mode 100644 src/lib/jobs/registry/maintenance.ts create mode 100644 src/lib/jobs/registry/register.ts create mode 100644 src/lib/jobs/registry/reminders.ts create mode 100644 src/lib/jobs/registry/rollup.ts create mode 100644 src/lib/jobs/registry/status.ts create mode 100644 src/lib/jobs/reminder/climb-backfill-at-boot.ts create mode 100644 src/lib/jobs/reminder/document-handlers.ts create mode 100644 src/lib/jobs/reminder/encryption-backfill-handlers.ts create mode 100644 src/lib/jobs/reminder/maintenance-handlers.ts delete mode 100644 src/lib/jobs/reminder/register-integration-sync.ts delete mode 100644 src/lib/jobs/reminder/register-maintenance.ts delete mode 100644 src/lib/jobs/reminder/register-reminders.ts delete mode 100644 src/lib/jobs/reminder/register-rollup.ts delete mode 100644 src/lib/jobs/reminder/register-status.ts create mode 100644 src/lib/jobs/reminder/rollup-handlers.ts create mode 100644 src/lib/jobs/reminder/status-handlers.ts create mode 100644 src/lib/jobs/restore-drill-queue.ts create mode 100644 src/lib/jobs/sleep-timeline-backfill-queue.ts create mode 100644 src/lib/jobs/strain-score-queue.ts create mode 100644 src/lib/jobs/strava-backfill-queue.ts create mode 100644 src/lib/jobs/stress-score-queue.ts create mode 100644 src/lib/jobs/tls-pin-monitor-queue.ts create mode 100644 src/lib/jobs/whoop-backfill-queue.ts diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index ae77487b9..0dba141af 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 121, - "total": 143770, + "count": 116, + "total": 137648, "entries": { "app/api/dashboard/summary/route.ts": 1032, "app/api/documents/inbound/route.ts": 811, @@ -89,18 +89,13 @@ "lib/insights/mood-status.ts": 814, "lib/insights/status-cache.ts": 812, "lib/integrations/status.ts": 1445, - "lib/jobs/apple-health-import-worker.ts": 801, - "lib/jobs/backup-restore.ts": 888, - "lib/jobs/coach-nudge.ts": 1089, - "lib/jobs/insight-pregenerate.ts": 1266, - "lib/jobs/medication-intake-import.ts": 822, - "lib/jobs/offhost-backup.ts": 1168, + "lib/jobs/backup-restore.ts": 887, + "lib/jobs/coach-nudge.ts": 1084, + "lib/jobs/insight-pregenerate.ts": 1259, + "lib/jobs/medication-intake-import.ts": 819, + "lib/jobs/offhost-backup.ts": 1147, "lib/jobs/reaction-line.ts": 824, "lib/jobs/reminder/medication-reminder-check.ts": 818, - "lib/jobs/reminder/register-integration-sync.ts": 1051, - "lib/jobs/reminder/register-maintenance.ts": 2197, - "lib/jobs/reminder/register-rollup.ts": 1156, - "lib/jobs/reminder/register-status.ts": 878, "lib/managed-profiles/handover.ts": 986, "lib/mcp/record-search.ts": 1467, "lib/mcp/rich-reads.ts": 1467, @@ -117,7 +112,7 @@ "lib/rollups/measurement-rollups.ts": 1496, "lib/search/search-records.ts": 996, "lib/sharing/grants.ts": 1046, - "lib/support/report.ts": 905, + "lib/support/report.ts": 903, "lib/telegram-webhook-handlers.ts": 1341, "lib/validations/inbound-documents.ts": 1130, "lib/whoop/client.ts": 917, diff --git a/src/__tests__/baselines/source-function-lines.json b/src/__tests__/baselines/source-function-lines.json index dd571a095..7cc32eb6d 100644 --- a/src/__tests__/baselines/source-function-lines.json +++ b/src/__tests__/baselines/source-function-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", - "count": 410, - "total": 147795, + "count": 403, + "total": 144626, "entries": { "app/achievements/page.tsx :: AchievementsPage": 206, "app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314, @@ -368,13 +368,6 @@ "lib/jobs/reaction-line.ts :: runReactionLine": 337, "lib/jobs/reminder/medication-reminder-check.ts :: handleReminderCheck": 619, "lib/jobs/reminder/medication-reminder-check.ts :: handleReminderCheck > withBackgroundEvent(arg)": 614, - "lib/jobs/reminder/register-integration-sync.ts :: enqueueIntegrationSyncBootDiscovery": 224, - "lib/jobs/reminder/register-integration-sync.ts :: registerIntegrationSyncQueues": 320, - "lib/jobs/reminder/register-maintenance.ts :: enqueueMaintenanceBootDiscovery": 232, - "lib/jobs/reminder/register-maintenance.ts :: registerMaintenanceQueues": 997, - "lib/jobs/reminder/register-rollup.ts :: enqueueRollupBootDiscovery": 238, - "lib/jobs/reminder/register-rollup.ts :: registerRollupQueues": 616, - "lib/jobs/reminder/register-status.ts :: registerStatusQueues": 542, "lib/jobs/workout-insight-generate.ts :: runWorkoutInsightGenerate": 324, "lib/labs/biomarker-merge.ts :: runMerge": 254, "lib/labs/ocr-extract.ts :: runOcrExtraction": 207, diff --git a/src/__tests__/baselines/test-source-reads.json b/src/__tests__/baselines/test-source-reads.json index 2aff0e4d8..309eef2f2 100644 --- a/src/__tests__/baselines/test-source-reads.json +++ b/src/__tests__/baselines/test-source-reads.json @@ -1,7 +1,7 @@ { "$comment": "tests that read a source file by path, per test file under src/. Maintained by src/__tests__/test-realism-ratchet.test.ts.", - "count": 367, - "total": 367, + "count": 349, + "total": 349, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 1, "__tests__/admin-data-wipe-completeness.test.ts": 1, @@ -314,44 +314,26 @@ "lib/jobs/__tests__/ai-job-capability-inventory.test.ts": 1, "lib/jobs/__tests__/apple-health-import-ecg.test.ts": 1, "lib/jobs/__tests__/apple-health-import-export-date.test.ts": 1, - "lib/jobs/__tests__/apple-health-import-reconcile.test.ts": 1, "lib/jobs/__tests__/apple-health-import-send-policy.test.ts": 1, - "lib/jobs/__tests__/coach-memory-refresh-queue.test.ts": 1, "lib/jobs/__tests__/coach-message-cleanup-queue.test.ts": 1, - "lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts": 1, "lib/jobs/__tests__/cycle-reminder-queue.test.ts": 1, - "lib/jobs/__tests__/daily-briefing.test.ts": 1, "lib/jobs/__tests__/data-arrival-provider-isolation.test.ts": 1, "lib/jobs/__tests__/data-arrival-queue.test.ts": 1, "lib/jobs/__tests__/data-backup-send-policy.test.ts": 1, - "lib/jobs/__tests__/drain-cumulative-queue.test.ts": 1, "lib/jobs/__tests__/fitbit-queues.test.ts": 1, "lib/jobs/__tests__/geo-backfill.test.ts": 1, "lib/jobs/__tests__/geolite2-fetch-queue.test.ts": 1, "lib/jobs/__tests__/google-health-queues.test.ts": 1, - "lib/jobs/__tests__/insight-pregenerate.test.ts": 1, - "lib/jobs/__tests__/insight-status-generate.test.ts": 1, - "lib/jobs/__tests__/intake-slot-dedup-queue.test.ts": 1, "lib/jobs/__tests__/integration-backfill-admission.test.ts": 1, "lib/jobs/__tests__/job-failures-retention.test.ts": 1, "lib/jobs/__tests__/job-outcome-source-inventory.test.ts": 1, "lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts": 1, - "lib/jobs/__tests__/mean-consolidation-queue.test.ts": 1, "lib/jobs/__tests__/measurement-reminder-queue.test.ts": 1, - "lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts": 1, - "lib/jobs/__tests__/nightscout-queues.test.ts": 1, - "lib/jobs/__tests__/period-narrative-warm.test.ts": 1, - "lib/jobs/__tests__/queue-policy-tables.test.ts": 1, - "lib/jobs/__tests__/read-side-effect-queues.test.ts": 1, "lib/jobs/__tests__/recovery-score-queue.test.ts": 1, - "lib/jobs/__tests__/sleep-timeline-queues.test.ts": 1, "lib/jobs/__tests__/status-cron-candidates.test.ts": 1, - "lib/jobs/__tests__/step-consolidation-repair-queue.test.ts": 1, "lib/jobs/__tests__/stress-strain-retention-queue.test.ts": 1, - "lib/jobs/__tests__/subsystem-surface.test.ts": 1, "lib/jobs/__tests__/tls-pin-monitor-queue.test.ts": 1, "lib/jobs/__tests__/whoop-queues.test.ts": 1, - "lib/jobs/__tests__/withings-ecg-worker.test.ts": 1, "lib/jobs/__tests__/withings-queues.test.ts": 1, "lib/jobs/__tests__/workout-insight-queue.test.ts": 1, "lib/jobs/reminder/__tests__/integration-cohort-outcomes.test.ts": 1, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index 65ad892a5..d98d3866f 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", - "count": 2954, - "total": 10499, + "count": 2924, + "total": 10176, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -2288,59 +2288,34 @@ "lib/integrations/metric-freshness.ts": 1, "lib/integrations/status.ts": 19, "lib/intl/__tests__/formatter-cache.test.ts": 1, - "lib/jobs/__tests__/apple-health-import-reconcile.test.ts": 2, - "lib/jobs/__tests__/apple-health-import-send-policy.test.ts": 1, - "lib/jobs/__tests__/coach-memory-refresh-queue.test.ts": 2, - "lib/jobs/__tests__/coach-message-cleanup-queue.test.ts": 1, + "lib/jobs/__tests__/apple-health-import-reconcile.test.ts": 1, "lib/jobs/__tests__/coach-nudge.test.ts": 3, "lib/jobs/__tests__/coach-reminder-sweep.test.ts": 1, - "lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts": 1, - "lib/jobs/__tests__/cycle-reminder-queue.test.ts": 2, "lib/jobs/__tests__/cycle-reminder.test.ts": 2, - "lib/jobs/__tests__/data-arrival-queue.test.ts": 1, - "lib/jobs/__tests__/drain-cumulative-queue.test.ts": 4, "lib/jobs/__tests__/feedback-aggregator.test.ts": 2, "lib/jobs/__tests__/fitbit-backfill.test.ts": 1, - "lib/jobs/__tests__/fitbit-queues.test.ts": 2, - "lib/jobs/__tests__/geo-backfill.test.ts": 3, - "lib/jobs/__tests__/google-health-queues.test.ts": 1, + "lib/jobs/__tests__/geo-backfill.test.ts": 1, "lib/jobs/__tests__/google-health-sleep-repair.test.ts": 1, "lib/jobs/__tests__/hero-line-json-envelope.test.ts": 1, "lib/jobs/__tests__/insight-pregenerate-retry-enqueue.test.ts": 1, - "lib/jobs/__tests__/insight-pregenerate.test.ts": 16, - "lib/jobs/__tests__/insight-status-generate.test.ts": 2, + "lib/jobs/__tests__/insight-pregenerate.test.ts": 14, "lib/jobs/__tests__/intake-auto-skip.test.ts": 3, "lib/jobs/__tests__/intake-auto-take.test.ts": 1, - "lib/jobs/__tests__/intake-slot-dedup-queue.test.ts": 6, "lib/jobs/__tests__/job-run-report.test.ts": 1, - "lib/jobs/__tests__/mean-consolidation-queue.test.ts": 2, - "lib/jobs/__tests__/measurement-reminder-queue.test.ts": 2, "lib/jobs/__tests__/measurement-reminder.test.ts": 6, - "lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts": 2, "lib/jobs/__tests__/medication-inventory-expire.test.ts": 1, "lib/jobs/__tests__/medication-low-stock.test.ts": 2, "lib/jobs/__tests__/medication-reminder-check.test.ts": 2, "lib/jobs/__tests__/mood-reminder.test.ts": 2, - "lib/jobs/__tests__/nightscout-queues.test.ts": 3, - "lib/jobs/__tests__/period-narrative-warm.test.ts": 1, "lib/jobs/__tests__/reaction-line-degradation.test.ts": 2, - "lib/jobs/__tests__/recovery-score-queue.test.ts": 2, "lib/jobs/__tests__/recovery-score.test.ts": 1, "lib/jobs/__tests__/reminder-phases.test.ts": 1, "lib/jobs/__tests__/reminder-satisfy.test.ts": 3, "lib/jobs/__tests__/report-worker-error.test.ts": 1, "lib/jobs/__tests__/run-job.test.ts": 1, "lib/jobs/__tests__/sleep-timeline-backfill.test.ts": 1, - "lib/jobs/__tests__/sleep-timeline-queues.test.ts": 2, - "lib/jobs/__tests__/status-cron-candidates.test.ts": 2, - "lib/jobs/__tests__/step-consolidation-repair-queue.test.ts": 1, - "lib/jobs/__tests__/stress-strain-retention-queue.test.ts": 2, - "lib/jobs/__tests__/tls-pin-monitor-queue.test.ts": 2, "lib/jobs/__tests__/whoop-backfill.test.ts": 1, - "lib/jobs/__tests__/whoop-queues.test.ts": 2, "lib/jobs/__tests__/whoop-resource-sync.test.ts": 1, - "lib/jobs/__tests__/withings-queues.test.ts": 2, - "lib/jobs/__tests__/workout-insight-queue.test.ts": 1, "lib/jobs/apple-health-import-worker.ts": 4, "lib/jobs/coach-message-cleanup.ts": 1, "lib/jobs/coach-nudge-ai.ts": 3, @@ -2396,7 +2371,7 @@ "lib/jobs/recovery-score.ts": 1, "lib/jobs/reminder-phases.ts": 1, "lib/jobs/reminder-satisfy.ts": 1, - "lib/jobs/reminder-worker.ts": 4, + "lib/jobs/reminder-worker.ts": 3, "lib/jobs/reminder/__tests__/cleanup-handlers-visibility.test.ts": 1, "lib/jobs/reminder/__tests__/queue-retention.test.ts": 1, "lib/jobs/reminder/cleanup-handlers.ts": 7, @@ -2410,11 +2385,6 @@ "lib/jobs/reminder/oura-sync.ts": 1, "lib/jobs/reminder/polar-sync.ts": 2, "lib/jobs/reminder/poll-cohort.ts": 2, - "lib/jobs/reminder/register-integration-sync.ts": 63, - "lib/jobs/reminder/register-maintenance.ts": 114, - "lib/jobs/reminder/register-reminders.ts": 15, - "lib/jobs/reminder/register-rollup.ts": 37, - "lib/jobs/reminder/register-status.ts": 38, "lib/jobs/reminder/registrar-shared.ts": 2, "lib/jobs/reminder/strava-sync.ts": 1, "lib/jobs/reminder/whoop-sync.ts": 3, diff --git a/src/__tests__/guard-floor-guard.test.ts b/src/__tests__/guard-floor-guard.test.ts index 7578cf6ce..687ec7d31 100644 --- a/src/__tests__/guard-floor-guard.test.ts +++ b/src/__tests__/guard-floor-guard.test.ts @@ -63,8 +63,6 @@ const NO_FLOOR_NEEDED: Readonly> = { "named walks, each must positively match the keyset cursor", "components/charts/__tests__/touch-action-guard.test.ts": "named chart wrappers, each must positively declare touch-pan-y", - "__tests__/moodlog-removal-guard.test.ts": - "absence checks over named paths, columns and keys; no matcher to drift", // Unit tests of runtime code whose name ends in "guard". They read no // source and sweep nothing. diff --git a/src/__tests__/helpers/source-metrics.ts b/src/__tests__/helpers/source-metrics.ts index f03d8b365..5fa9d3a2d 100644 --- a/src/__tests__/helpers/source-metrics.ts +++ b/src/__tests__/helpers/source-metrics.ts @@ -197,6 +197,8 @@ export interface ModuleImports { value: string[]; /** True when every statement is an `export … from` (or an import). */ reexportOnly: boolean; + /** The value imports evaluated when the module loads: `value` without `import("…")`. */ + staticValue: string[]; } /** @@ -240,6 +242,7 @@ export function moduleImports(sf: ts.SourceFile): ModuleImports { } other += 1; } + const staticValue = [...value]; const visit = (node: ts.Node): void => { if ( ts.isCallExpression(node) && @@ -252,7 +255,7 @@ export function moduleImports(sf: ts.SourceFile): ModuleImports { ts.forEachChild(node, visit); }; visit(sf); - return { value, reexportOnly: reexports > 0 && other === 0 }; + return { value, staticValue, reexportOnly: reexports > 0 && other === 0 }; } export interface ImportGraph { @@ -374,3 +377,40 @@ export function backEdgeCycles(edges: Map): string[][] { for (const v of [...edges.keys()].sort()) walk(v); return cycles; } + +/** + * Every module under `src/` that loading `entries` evaluates: static value + * imports and re-exports, transitively. A dynamic `import("…")` is not + * followed, since it loads only when it runs. Paths are relative to `src/`. + */ +export function staticImportClosure(entries: readonly string[]): Set { + const cache = ts.createModuleResolutionCache( + process.cwd(), + (x) => x, + COMPILER_OPTIONS, + ); + const seen = new Set(); + const stack = [...entries]; + while (stack.length > 0) { + const rel = stack.pop()!; + if (seen.has(rel)) continue; + seen.add(rel); + const abs = join(SRC, rel); + for (const spec of moduleImports(parse(rel, readFileSync(abs, "utf8"))) + .staticValue) { + const hit = ts.resolveModuleName( + spec, + abs, + COMPILER_OPTIONS, + ts.sys, + cache, + ).resolvedModule; + if (!hit || hit.isExternalLibraryImport) continue; + if (!hit.resolvedFileName.startsWith(`${SRC}/`)) continue; + const target = hit.resolvedFileName.slice(SRC.length + 1); + if (target.startsWith("generated/") || target.endsWith(".d.ts")) continue; + stack.push(target); + } + } + return seen; +} diff --git a/src/__tests__/job-queue-runtime-guard.test.ts b/src/__tests__/job-queue-runtime-guard.test.ts index 48da4c47d..90b14d8ec 100644 --- a/src/__tests__/job-queue-runtime-guard.test.ts +++ b/src/__tests__/job-queue-runtime-guard.test.ts @@ -11,23 +11,23 @@ * `src/lib/jobs/queue-runtime.ts` declares every queue `short`, `budgeted` or * `long`. This guard holds the tree to it: * - * - every queue bound with `createAndWork` is in the table, and every table - * entry is bound; - * - every `boss.send` and every schedule tuple names a queue in the table; + * - every queue the worker registry declares (`jobs/registry/`) is in the + * table, and every table entry is declared; + * - every `boss.send` and every scheduled queue names a queue in the table; * - a `long` queue has an expiry between the default and pg-boss's 24-hour - * ceiling, every send and schedule for it carries `expireInSeconds` (or an - * identifier the table names as its carrier), it stops on the job's budget - * where the table says (`jobBudget` / `jobDeadline`, or a declared - * one-shot), and it is exclusive per identity where the table says - * (`lockedPass` at the binding, or the named marker); + * ceiling, every send for it carries `expireInSeconds` (or an identifier + * the table names as its carrier) and its schedule's send options do, it + * stops on the job's budget where the table says (`jobBudget` / + * `jobDeadline`, or a declared one-shot), and it is exclusive per identity + * where the table says (the definition's `lock`, or the named marker); * - a `budgeted` queue stops on the budget and relies on the default expiry; * - a `short` queue's sends set no expiry of their own, which would mean the * job is longer than the table claims. * * Floors fail the guard if a matcher stops matching, and the "plants" cases * run the checks over a planted long queue that lacks each protection in turn - * (the mutation proof kept in the suite). Removing `lockedPass` from the - * pr-detection binding, or the expiry from its cron tuple, fails the real + * (the mutation proof kept in the suite). Removing the `lock` from the + * pr-detection definition, or the expiry from its schedule, fails the real * checks below. */ import { describe, expect, it } from "vitest"; @@ -39,19 +39,14 @@ import { DEFAULT_JOB_EXPIRE_SECONDS, QUEUE_RUNTIME, type QueueRuntime, - type SourceSite, } from "@/lib/jobs/queue-runtime"; import { SCHEDULED_AND_SENT_QUEUES } from "@/lib/jobs/reminder/registrar-shared"; import { HEAVY_QUEUES } from "@/lib/jobs/heavy-lane"; +import type { JobDefinition } from "@/lib/jobs/registry/define-job"; +import { allJobDefinitions } from "@/lib/jobs/registry/index"; const SRC = join(process.cwd(), "src"); -const REGISTRARS = [ - "lib/jobs/reminder/register-integration-sync.ts", - "lib/jobs/reminder/register-maintenance.ts", - "lib/jobs/reminder/register-reminders.ts", - "lib/jobs/reminder/register-rollup.ts", - "lib/jobs/reminder/register-status.ts", -]; +const DEFINITIONS = allJobDefinitions(); /** pg-boss refuses an expiry of 24 hours or more. */ const MAX_EXPIRE_SECONDS = 24 * 60 * 60; @@ -120,27 +115,6 @@ interface Site { text: string; } -/** Every `createAndWork(boss, QUEUE, …)` in the registrars. */ -function bindings( - texts: ReadonlyMap, - definitions: Definitions, -): Site[] { - const out: Site[] = []; - for (const file of REGISTRARS) { - const text = code(texts.get(file) ?? ""); - for (const m of text.matchAll( - /createAndWork\s*(?:<[^(]*?>)?\(\s*boss\s*,\s*([A-Z][A-Z0-9_]*)/g, - )) { - const open = text.indexOf("(", m.index); - const queue = resolve(definitions, file, m[1]); - if (queue) { - out.push({ file, queue, text: text.slice(open, closeOf(text, open)) }); - } - } - } - return out; -} - /** Every `boss.send(QUEUE, …)` / `getGlobalBoss()?.send(QUEUE, …)`. */ function sends( texts: ReadonlyMap, @@ -162,31 +136,6 @@ function sends( return out; } -/** Every `[QUEUE, CRON, options?]` tuple in a registrar's `schedules`. */ -function schedules( - texts: ReadonlyMap, - definitions: Definitions, -): Site[] { - const out: Site[] = []; - for (const file of REGISTRARS) { - const text = code(texts.get(file) ?? ""); - const m = /const\s+schedules\s*:\s*ScheduleEntry\[\]\s*=\s*\[/.exec(text); - if (!m) continue; - const open = m.index + m[0].length - 1; - const body = text.slice(open + 1, closeOf(text, open) - 1); - for (let i = 0; i < body.length; i += 1) { - if (body[i] !== "[") continue; - const end = closeOf(body, i); - const tuple = body.slice(i, end); - const ident = /^\[\s*([A-Z][A-Z0-9_]*)/.exec(tuple)?.[1]; - const queue = ident ? resolve(definitions, file, ident) : null; - if (queue) out.push({ file, queue, text: tuple }); - i = end - 1; - } - } - return out; -} - /** The text of `fn` (a function or a const) in `file`. */ function bodyOf( texts: ReadonlyMap, @@ -201,14 +150,6 @@ function bodyOf( return text.slice(m.index, end < 0 ? undefined : end + 2); } -function siteText( - texts: ReadonlyMap, - site: SourceSite, - binding: Site | undefined, -): string { - return site === "binding" ? (binding?.text ?? "") : bodyOf(texts, site); -} - const STOPS = /\bjob(?:Budget|Deadline)\s*\(/; /** Every finding for one queue's declaration against the tree. */ @@ -216,13 +157,12 @@ function check( queue: string, entry: QueueRuntime, texts: ReadonlyMap, - all: { bindings: Site[]; sends: Site[]; schedules: Site[] }, + all: { definitions: readonly JobDefinition[]; sends: Site[] }, ): string[] { const problems: string[] = []; - const binding = all.bindings.find((b) => b.queue === queue); - const carriers = [...all.sends, ...all.schedules].filter( - (s) => s.queue === queue, - ); + const definition = all.definitions.find((d) => d.queue === queue); + const sends = all.sends.filter((s) => s.queue === queue); + const scheduledExpiry = definition?.scheduleOptions?.expireInSeconds; if (entry.runtime === "long") { if ( !(entry.expireInSeconds > DEFAULT_JOB_EXPIRE_SECONDS) || @@ -230,85 +170,86 @@ function check( ) { problems.push(`${queue}: expiry ${entry.expireInSeconds}s out of range`); } - for (const site of carriers) { + for (const site of sends) { const carried = /\bexpireInSeconds\b/.test(site.text) || entry.expiryVia.some((v) => new RegExp(`\\b${v}\\b`).test(site.text)); if (!carried) { - problems.push( - `${queue}: ${site.file} sends or schedules it without an expiry`, - ); + problems.push(`${queue}: ${site.file} sends it without an expiry`); } } + if (definition?.cron && typeof scheduledExpiry !== "number") { + problems.push(`${queue}: its schedule carries no expiry`); + } if (typeof entry.stop === "object" && "oneShot" in entry.stop) { if (entry.stop.oneShot.length < 20) { problems.push(`${queue}: a one-shot needs its reason`); } - } else if (!STOPS.test(siteText(texts, entry.stop, binding))) { + } else if (!STOPS.test(bodyOf(texts, entry.stop))) { problems.push(`${queue}: does not stop on the job's budget`); } if (entry.exclusive === "lockedPass") { - if (!/\blockedPass\s*\(/.test(binding?.text ?? "")) { - problems.push(`${queue}: binding holds no lockedPass`); + if (!definition?.lock) { + problems.push(`${queue}: definition holds no lock`); } } else if ( - !siteText(texts, entry.exclusive.at, binding).includes( - entry.exclusive.marker, - ) + !bodyOf(texts, entry.exclusive.at).includes(entry.exclusive.marker) ) { problems.push(`${queue}: no "${entry.exclusive.marker}" where declared`); } } else { if (entry.runtime === "budgeted") { - if (!STOPS.test(siteText(texts, entry.budget, binding))) { + if (!STOPS.test(bodyOf(texts, entry.budget))) { problems.push(`${queue}: does not stop on the job's budget`); } } - for (const site of carriers) { + for (const site of sends) { if (/\bexpireInSeconds\b/.test(site.text)) { problems.push( `${queue}: ${site.file} sets an expiry on a queue declared ${entry.runtime}`, ); } } + if (scheduledExpiry !== undefined) { + problems.push( + `${queue}: its schedule sets an expiry on a queue declared ${entry.runtime}`, + ); + } } return problems; } function scan(texts: ReadonlyMap) { - const definitions = definitionsOf(texts); - return { - bindings: bindings(texts, definitions), - sends: sends(texts, definitions), - schedules: schedules(texts, definitions), - }; + return { sends: sends(texts, definitionsOf(texts)) }; } describe("every queue declares its runtime", () => { - const all = scan(source); - const bound = new Set(all.bindings.map((b) => b.queue)); + const all = { definitions: DEFINITIONS, ...scan(source) }; + const declared = new Set(DEFINITIONS.map((d) => d.queue)); - it("finds the bindings, sends and schedules it is meant to police", () => { - // Pinned below the counts on 2026-10-01 (112 / 64 / 74). - expect(all.bindings.length).toBeGreaterThanOrEqual(105); + it("finds the declarations, sends and schedules it is meant to police", () => { + // Pinned below the counts on 2026-10-23 (122 / 64 / 78). + expect(DEFINITIONS.length).toBeGreaterThanOrEqual(115); expect(all.sends.length).toBeGreaterThanOrEqual(50); - expect(all.schedules.length).toBeGreaterThanOrEqual(70); + expect(DEFINITIONS.filter((d) => d.cron).length).toBeGreaterThanOrEqual(70); expect( Object.values(QUEUE_RUNTIME).filter((e) => e.runtime === "long").length, ).toBeGreaterThanOrEqual(15); }); - it("every bound queue is in the table, and every entry is bound", () => { - expect([...bound].filter((q) => !(q in QUEUE_RUNTIME)).sort()).toEqual([]); + it("every declared queue is in the table, and every entry is declared", () => { + expect([...declared].filter((q) => !(q in QUEUE_RUNTIME)).sort()).toEqual( + [], + ); expect( Object.keys(QUEUE_RUNTIME) - .filter((q) => !bound.has(q)) + .filter((q) => !declared.has(q)) .sort(), ).toEqual([]); }); - it("every send and schedule names a queue in the table", () => { - const unknown = [...all.sends, ...all.schedules] + it("every send names a queue in the table", () => { + const unknown = all.sends .filter((s) => !(s.queue in QUEUE_RUNTIME)) .map((s) => `${s.file} → ${s.queue}`); expect(unknown).toEqual([]); @@ -322,51 +263,56 @@ describe("every queue declares its runtime", () => { }); it("plants: a long queue missing any protection is caught", () => { - const registrar = REGISTRARS[1]; - const planted = (binding: string, schedule: string) => + const file = "lib/jobs/planted.ts"; + const texts = (handler: string) => new Map([ ...source, - [ - registrar, - `const PLANT_QUEUE = "planted-long";\n` + - `const schedules: ScheduleEntry[] = [\n ${schedule},\n];\n` + - `async function register(boss) {\n await createAndWork(boss, PLANT_QUEUE, { localConcurrency: 1 }, ${binding});\n}\n`, - ], + [file, `export async function plant(jobs) {\n ${handler}\n}\n`], ]); const entry: QueueRuntime = { runtime: "long", expireInSeconds: 3600, expiryVia: [], - stop: "binding", + stop: { file, fn: "plant" }, exclusive: "lockedPass", why: "planted", }; - const findings = (binding: string, schedule: string) => { - const texts = planted(binding, schedule); - return check("planted-long", entry, texts, scan(texts)); + const planted = (overrides: Partial): JobDefinition => ({ + queue: "planted-long", + cron: "0 3 * * *", + scheduleOptions: { expireInSeconds: 3600 }, + work: { localConcurrency: 1 }, + lock: () => "pass", + load: async () => async () => ({ ok: true, did: {} }), + ...overrides, + }); + const findings = ( + handler: string, + overrides: Partial = {}, + ) => { + const t = texts(handler); + return check("planted-long", entry, t, { + definitions: [planted(overrides)], + ...scan(t), + }); }; - const safe = [ - "lockedPass(PLANT_QUEUE, () => WHOLE_PASS, async (jobs) => run(jobBudget(jobs)))", - "[PLANT_QUEUE, CRON, { expireInSeconds: 3600 }]", - ] as const; - expect(findings(...safe)).toEqual([]); - expect(findings(safe[0], "[PLANT_QUEUE, CRON]")).toEqual([ - `planted-long: ${registrar} sends or schedules it without an expiry`, + const budgeted = "return run(jobBudget(jobs));"; + expect(findings(budgeted)).toEqual([]); + expect(findings(budgeted, { scheduleOptions: { retryLimit: 0 } })).toEqual([ + "planted-long: its schedule carries no expiry", ]); - expect( - findings("lockedPass(PLANT_QUEUE, () => WHOLE_PASS, run)", safe[1]), - ).toEqual(["planted-long: does not stop on the job's budget"]); - expect(findings("async (jobs) => run(jobBudget(jobs))", safe[1])).toEqual([ - "planted-long: binding holds no lockedPass", + expect(findings("return run(jobs);")).toEqual([ + "planted-long: does not stop on the job's budget", + ]); + expect(findings(budgeted, { lock: undefined })).toEqual([ + "planted-long: definition holds no lock", ]); expect( check( "planted-long", { ...entry, expireInSeconds: 900 }, - ...((): [ReadonlyMap, ReturnType] => { - const texts = planted(...safe); - return [texts, scan(texts)]; - })(), + texts(budgeted), + { definitions: [planted({})], ...scan(texts(budgeted)) }, ), ).toEqual(["planted-long: expiry 900s out of range"]); }); @@ -374,12 +320,13 @@ describe("every queue declares its runtime", () => { /** * Queues a file sends to through a variable (`boss.send(queueName, …)`): every - * known queue name the file spells out, as a constant or a map value, counts - * as sent, because the matcher above cannot tell which one the variable holds. + * known queue the file names, as a literal, a map value or an imported + * `*_QUEUE` constant, counts as sent, because the matcher above cannot tell which one the variable holds. */ function dynamicallySent(texts: ReadonlyMap): Set { + const definitions = definitionsOf(texts); const out = new Set(); - for (const [, raw] of texts) { + for (const [file, raw] of texts) { const text = code(raw); if ( !/\b(?:boss|getGlobalBoss\(\)\??)\s*\.\s*send\s*(?:<[^(]*?>)?\(\s*[a-z]/.test( @@ -391,13 +338,20 @@ function dynamicallySent(texts: ReadonlyMap): Set { for (const m of text.matchAll(/"([a-z0-9-]+)"/g)) { if (m[1] in QUEUE_RUNTIME) out.add(m[1]); } + // A queue named through an imported constant counts the same. + for (const m of text.matchAll(/\b([A-Z][A-Z0-9_]*_QUEUE)\b/g)) { + const queue = resolve(definitions, file, m[1]); + if (queue && queue in QUEUE_RUNTIME) out.add(queue); + } } return out; } describe("a cron-fed queue polls slowly, a sent one does not", () => { const all = scan(source); - const scheduled = new Set(all.schedules.map((s) => s.queue)); + const scheduled = new Set( + DEFINITIONS.filter((d) => d.cron).map((d) => d.queue), + ); const sent = new Set([ ...all.sends.map((s) => s.queue), ...dynamicallySent(source), @@ -418,26 +372,14 @@ describe("a cron-fed queue polls slowly, a sent one does not", () => { }); describe("heavy jobs share the lane", () => { - const all = scan(source); - const laned = all.bindings.filter((b) => /\bheavyLane\s*\(/.test(b.text)); + const laned = DEFINITIONS.filter((d) => d.lane); - it("binds every heavy queue in the lane, and nothing else", () => { - expect(laned.map((b) => b.queue).sort()).toEqual( + it("puts every heavy queue in the lane, and nothing else", () => { + expect(laned.map((d) => d.queue).sort()).toEqual( Object.keys(HEAVY_QUEUES).sort(), ); }); - it("holds a lockedPass outside the lane, so an expiry retry finds the run going", () => { - // Outside, a retry pg-boss starts while the expired run still holds the - // lane finds the lock and settles; inside, the lane would defer it and - // run the job a second time once the first is done. - const inverted = laned - .filter((b) => /\blockedPass\s*\(/.test(b.text)) - .filter((b) => b.text.indexOf("lockedPass") > b.text.indexOf("heavyLane")) - .map((b) => b.queue); - expect(inverted).toEqual([]); - }); - it("declares every heavy queue long, so its expiry, stop and exclusivity are proven", () => { // A heavy job is the one most likely to run past pg-boss's default // expiry, and the retry that follows an expiry runs it a second time. @@ -464,41 +406,28 @@ describe("heavy jobs share the lane", () => { // A deferred job returns from active to created. Under `short` (one // queued job per key) and `stately` (one per state) a twin already // queued would refuse it, and the job would fail instead of waiting. - const policies = policiesOf(source); - expect(policies.size).toBeGreaterThanOrEqual(40); - const refused = Object.keys(HEAVY_QUEUES) - .filter((queue) => /^(?:short|stately)$/.test(policies.get(queue) ?? "")) - .sort(); - expect(refused).toEqual([]); - // The planted case: the same check over a table that declares one. - const planted = new Map(source); - const file = REGISTRARS[2]; - planted.set( - file, - `const DATA_BACKUP_QUEUE = "data-backup";\nconst queuePolicies: QueuePolicyTable = {\n [DATA_BACKUP_QUEUE]: { policy: "short", reason: "x" },\n};\n`, + const refusing = (definitions: readonly JobDefinition[]) => + definitions + .filter( + (d) => d.lane && /^(?:short|stately)$/.test(d.policy?.policy ?? ""), + ) + .map((d) => d.queue) + .sort(); + expect(DEFINITIONS.filter((d) => d.policy).length).toBeGreaterThanOrEqual( + 40, ); - expect(policiesOf(planted).get("data-backup")).toBe("short"); + expect(refusing(DEFINITIONS)).toEqual([]); + // The planted case: the same check over a heavy queue declared `short`. + expect( + refusing([ + { + queue: "data-backup", + policy: { policy: "short", reason: "x" }, + work: { localConcurrency: 1 }, + lane: true, + load: async () => async () => ({ ok: true, did: {} }), + }, + ]), + ).toEqual(["data-backup"]); }); }); - -/** Every `[QUEUE]: { policy: "…" }` in a registrar's `queuePolicies`. */ -function policiesOf(texts: ReadonlyMap): Map { - const definitions = definitionsOf(texts); - const out = new Map(); - for (const file of REGISTRARS) { - const text = code(texts.get(file) ?? ""); - const m = /const\s+queuePolicies\s*:\s*QueuePolicyTable\s*=\s*\{/.exec( - text, - ); - if (!m) continue; - const open = m.index + m[0].length - 1; - const body = text.slice(open, closeOf(text, open)); - for (const entry of body.matchAll( - /\[\s*([A-Z][A-Z0-9_]*)\s*\]\s*:\s*\{\s*policy\s*:\s*"([a-z]+)"/g, - )) { - const queue = resolve(definitions, file, entry[1]); - if (queue) out.set(queue, entry[2]); - } - } - return out; -} diff --git a/src/__tests__/moodlog-removal-guard.test.ts b/src/__tests__/moodlog-removal-guard.test.ts index b2b218b14..c012aed0f 100644 --- a/src/__tests__/moodlog-removal-guard.test.ts +++ b/src/__tests__/moodlog-removal-guard.test.ts @@ -18,11 +18,12 @@ */ import { readFileSync, existsSync } from "node:fs"; import path from "node:path"; + +import { allJobDefinitions } from "@/lib/jobs/registry/index"; import { describe, expect, it } from "vitest"; import { INTEGRATION_CADENCE } from "@/lib/integrations/sync-verdict"; import { INTEGRATION_DISPLAY_NAMES } from "@/components/settings/integrations/integration-fallback-row"; -import { readSource } from "@/__tests__/helpers/source-files"; const ROOT = path.resolve(__dirname, "../.."); @@ -76,12 +77,9 @@ describe("moodLog integration stays removed", () => { }); it("registers no moodlog queue, cron or handler binding", () => { - const registrar = readSource( - "src/lib/jobs/reminder/register-integration-sync.ts", - "export async function registerIntegrationSyncQueues", - ); - expect(registrar).not.toContain("moodlog"); - expect(registrar).not.toContain("MoodLog"); + const queues = allJobDefinitions().map((job) => job.queue); + expect(queues.length).toBeGreaterThan(100); + expect(queues.filter((queue) => /mood-?log/i.test(queue))).toEqual([]); }); it("carries no moodlog integration key on the status envelope", () => { diff --git a/src/app/api/admin/backups/route.ts b/src/app/api/admin/backups/route.ts index 0aac898e9..495777c14 100644 --- a/src/app/api/admin/backups/route.ts +++ b/src/app/api/admin/backups/route.ts @@ -32,10 +32,10 @@ import { import { readBackupPassActivity } from "@/lib/jobs/backup-pass-attempts"; import { summariseBackupSchedule } from "@/lib/jobs/backup-schedule-status"; import { - OFFHOST_BACKUP_QUEUE, offhostBackupConfigured, probeOffhostLifecycle, } from "@/lib/jobs/offhost-backup"; +import { OFFHOST_BACKUP_QUEUE } from "@/lib/jobs/offhost-backup-queue"; import { classifyOffhostBackup, OFFHOST_BACKUP_PERIOD_HOURS, diff --git a/src/app/api/admin/encryption/rotate/route.ts b/src/app/api/admin/encryption/rotate/route.ts index e3f8dc063..98ded9eef 100644 --- a/src/app/api/admin/encryption/rotate/route.ts +++ b/src/app/api/admin/encryption/rotate/route.ts @@ -31,10 +31,10 @@ import { annotate } from "@/lib/logging/context"; import { auditLog } from "@/lib/auth/audit"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { - ENCRYPTION_KEY_ROTATE_QUEUE, ENCRYPTION_KEY_ROTATE_SINGLETON, ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS, } from "@/lib/jobs/encryption-key-rotate"; +import { ENCRYPTION_KEY_ROTATE_QUEUE } from "@/lib/jobs/encryption-key-rotate-queue"; import { getActiveKeyId } from "@/lib/crypto"; import { NextRequest } from "next/server"; diff --git a/src/app/api/admin/import-apple-health-export/route.ts b/src/app/api/admin/import-apple-health-export/route.ts index 798061698..ddddadf5d 100644 --- a/src/app/api/admin/import-apple-health-export/route.ts +++ b/src/app/api/admin/import-apple-health-export/route.ts @@ -21,12 +21,12 @@ import { auditLog } from "@/lib/auth/audit"; import { checkRateLimit } from "@/lib/rate-limit"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { - APPLE_HEALTH_IMPORT_V2_QUEUE, APPLE_HEALTH_IMPORT_KIND, APPLE_HEALTH_IMPORT_PARSER_REVISION, APPLE_HEALTH_IMPORT_SEND_OPTIONS, type AppleHealthImportPayload, } from "@/lib/jobs/apple-health-import-worker"; +import { APPLE_HEALTH_IMPORT_V2_QUEUE } from "@/lib/jobs/apple-health-import-queue"; import { streamMultipartToDisk } from "@/lib/multipart/stream-to-disk"; import { createImportJobUnlessBusy, diff --git a/src/app/api/fitbit/callback/route.ts b/src/app/api/fitbit/callback/route.ts index a8284ae92..0e089baff 100644 --- a/src/app/api/fitbit/callback/route.ts +++ b/src/app/api/fitbit/callback/route.ts @@ -15,7 +15,7 @@ import { } from "@/lib/fitbit/client"; import { getUserFitbitCredentials } from "@/lib/fitbit/credentials"; import { FITBIT_OAUTH_STATE_COOKIE } from "@/lib/fitbit/oauth-state"; -import { FITBIT_BACKFILL_QUEUE } from "@/lib/jobs/fitbit-backfill"; +import { FITBIT_BACKFILL_QUEUE } from "@/lib/fitbit/jobs/queues"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { markReconnected } from "@/lib/integrations/status"; import { Prisma } from "@/generated/prisma/client"; diff --git a/src/app/api/google-health/callback/route.ts b/src/app/api/google-health/callback/route.ts index 4c0ac3d98..21a3638e4 100644 --- a/src/app/api/google-health/callback/route.ts +++ b/src/app/api/google-health/callback/route.ts @@ -17,7 +17,7 @@ import { } from "@/lib/google-health/client"; import { getUserGoogleHealthCredentials } from "@/lib/google-health/credentials"; import { GOOGLE_HEALTH_OAUTH_STATE_COOKIE } from "@/lib/google-health/oauth-state"; -import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "@/lib/jobs/google-health-backfill"; +import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "@/lib/jobs/google-health-backfill-queue"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { markReconnected } from "@/lib/integrations/status"; import { Prisma } from "@/generated/prisma/client"; diff --git a/src/app/api/import/apple-health-export/route.ts b/src/app/api/import/apple-health-export/route.ts index 20eb69061..d5e35b179 100644 --- a/src/app/api/import/apple-health-export/route.ts +++ b/src/app/api/import/apple-health-export/route.ts @@ -30,12 +30,12 @@ import { emitSignal } from "@/lib/logging/signal"; import { checkRateLimit } from "@/lib/rate-limit"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { - APPLE_HEALTH_IMPORT_V2_QUEUE, APPLE_HEALTH_IMPORT_KIND, APPLE_HEALTH_IMPORT_PARSER_REVISION, APPLE_HEALTH_IMPORT_SEND_OPTIONS, type AppleHealthImportPayload, } from "@/lib/jobs/apple-health-import-worker"; +import { APPLE_HEALTH_IMPORT_V2_QUEUE } from "@/lib/jobs/apple-health-import-queue"; import { streamMultipartToDisk } from "@/lib/multipart/stream-to-disk"; import { createImportJobUnlessBusy, diff --git a/src/app/api/import/health-connect-export/route.ts b/src/app/api/import/health-connect-export/route.ts index d8eb2b6b8..594c7fe1c 100644 --- a/src/app/api/import/health-connect-export/route.ts +++ b/src/app/api/import/health-connect-export/route.ts @@ -28,10 +28,10 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { HEALTH_CONNECT_IMPORT_KIND, HEALTH_CONNECT_IMPORT_PARSER_REVISION, - HEALTH_CONNECT_IMPORT_QUEUE, HEALTH_CONNECT_IMPORT_SEND_OPTIONS, type HealthConnectImportPayload, } from "@/lib/jobs/health-connect-import-worker"; +import { HEALTH_CONNECT_IMPORT_QUEUE } from "@/lib/jobs/health-connect-import-queue"; import { streamMultipartToDisk } from "@/lib/multipart/stream-to-disk"; import { createImportJobUnlessBusy, diff --git a/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts b/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts index 9966968a8..e04be16be 100644 --- a/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts +++ b/src/app/api/medications/[id]/intake/import/__tests__/route.test.ts @@ -63,10 +63,8 @@ import { auditLog } from "@/lib/auth/audit"; import { getSession } from "@/lib/auth/session"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { checkRateLimit } from "@/lib/rate-limit"; -import { - MEDICATION_INTAKE_IMPORT_QUEUE, - MEDICATION_INTAKE_IMPORT_STALE_AFTER_MS, -} from "@/lib/jobs/medication-intake-import"; +import { MEDICATION_INTAKE_IMPORT_STALE_AFTER_MS } from "@/lib/jobs/medication-intake-import"; +import { MEDICATION_INTAKE_IMPORT_QUEUE } from "@/lib/jobs/medication-intake-import-queue"; const SESSION_OK = { session: { id: "sess-1", expiresAt: new Date(Date.now() + 3_600_000) }, diff --git a/src/app/api/oura/callback/route.ts b/src/app/api/oura/callback/route.ts index 26b2e677e..350cdaf5c 100644 --- a/src/app/api/oura/callback/route.ts +++ b/src/app/api/oura/callback/route.ts @@ -27,10 +27,8 @@ import { markReconnected } from "@/lib/integrations/status"; import { OURA_SYNC_QUEUE } from "@/lib/jobs/integration-poll-queues"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { integrationBackfillSourceOptions } from "@/lib/jobs/integration-backfill-admission"; -import { - OURA_BACKFILL_QUEUE, - type OuraBackfillPayload, -} from "@/lib/jobs/oura-backfill"; +import { type OuraBackfillPayload } from "@/lib/jobs/oura-backfill"; +import { OURA_BACKFILL_QUEUE } from "@/lib/jobs/oura-backfill-queue"; import { NextRequest, NextResponse } from "next/server"; /** diff --git a/src/app/api/strava/callback/route.ts b/src/app/api/strava/callback/route.ts index e94544c3c..0bf259bf9 100644 --- a/src/app/api/strava/callback/route.ts +++ b/src/app/api/strava/callback/route.ts @@ -18,7 +18,7 @@ import { } from "@/lib/oauth/signed-state"; import { runOAuthCallbackOnce } from "@/lib/oauth/callback-claim"; import { markReconnected } from "@/lib/integrations/status"; -import { STRAVA_BACKFILL_QUEUE } from "@/lib/jobs/strava-backfill"; +import { STRAVA_BACKFILL_QUEUE } from "@/lib/jobs/strava-backfill-queue"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { NextRequest, NextResponse } from "next/server"; diff --git a/src/app/api/whoop/callback/route.ts b/src/app/api/whoop/callback/route.ts index 9bf1cef17..036c54565 100644 --- a/src/app/api/whoop/callback/route.ts +++ b/src/app/api/whoop/callback/route.ts @@ -14,7 +14,7 @@ import { import { getUserWhoopCredentials } from "@/lib/whoop/credentials"; import { WHOOP_OAUTH_STATE_COOKIE } from "@/lib/whoop/oauth-state"; import { buildReturnSchemeRedirect } from "@/lib/whoop/return-scheme"; -import { WHOOP_BACKFILL_QUEUE } from "@/lib/jobs/whoop-backfill"; +import { WHOOP_BACKFILL_QUEUE } from "@/lib/jobs/whoop-backfill-queue"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { markReconnected } from "@/lib/integrations/status"; import { Prisma } from "@/generated/prisma/client"; diff --git a/src/lib/fitbit/jobs/definitions.ts b/src/lib/fitbit/jobs/definitions.ts new file mode 100644 index 000000000..a4308c82d --- /dev/null +++ b/src/lib/fitbit/jobs/definitions.ts @@ -0,0 +1,98 @@ +/** + * Every queue the Fitbit integration runs, as one block of the worker + * registry. + * + * Google turns the classic Fitbit Web API off on 2026-10-30 + * (`lib/fitbit/retirement.ts`). Until the integration is removed it keeps + * syncing; removing it then is this folder plus the one line in + * `jobs/registry/integration-sync.ts` that spreads these definitions in. + */ +import { bootStaggerSecondsFor } from "@/lib/jobs/integration-backfill-admission"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + defineJob, + type BootDiscovery, + type JobDefinition, +} from "@/lib/jobs/registry/define-job"; + +import { + FITBIT_BACKFILL_CONCURRENCY, + FITBIT_BACKFILL_QUEUE, + FITBIT_OAUTH_STATE_CLEANUP_CRON, + FITBIT_OAUTH_STATE_CLEANUP_QUEUE, + FITBIT_SLEEP_REPAIR_CONCURRENCY, + FITBIT_SLEEP_REPAIR_QUEUE, + FITBIT_SYNC_CRON, + FITBIT_SYNC_QUEUE, +} from "./queues"; + +export const FITBIT_JOBS: readonly JobDefinition[] = [ + // The cron walks every connection; serial so a backfill-heavy tick never + // crowds the request pool. + defineJob({ + queue: FITBIT_SYNC_QUEUE, + cron: FITBIT_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/fitbit-sync")).handleFitbitSync, + }), + // The source queue keeps the per-connection singleton and forwards + // admitted work to the shared drain. + defineJob({ + queue: FITBIT_BACKFILL_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-connection full-history backfill; boot-discovery driven and self-converging.", + }, + work: { localConcurrency: FITBIT_BACKFILL_CONCURRENCY }, + load: async ({ boss }) => + ( + await import("@/lib/jobs/integration-backfill-forward") + ).forwardToBackfillAdmission(boss, "fitbit-backfill"), + }), + // One-shot sleep duplicate repair: a bounded sleep re-read per connection, + // forwarded to the shared drain. + defineJob({ + queue: FITBIT_SLEEP_REPAIR_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-connection one-shot sleep re-read; discovery drops the connection once it is marked repaired.", + }, + work: { localConcurrency: FITBIT_SLEEP_REPAIR_CONCURRENCY }, + load: async ({ boss }) => + ( + await import("@/lib/jobs/integration-backfill-forward") + ).forwardToBackfillAdmission(boss, "fitbit-sleep-repair"), + }), + defineJob({ + queue: FITBIT_OAUTH_STATE_CLEANUP_QUEUE, + cron: FITBIT_OAUTH_STATE_CLEANUP_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/fitbit-sync")) + .handleFitbitOAuthStateCleanup, + }), +]; + +/** The Fitbit backfill and sleep repair, offered again at every boot. */ +export const FITBIT_BOOT_DISCOVERY: readonly BootDiscovery[] = [ + { + name: "fitbit-backfill", + run: async () => + ( + await import("@/lib/jobs/fitbit-backfill") + ).enqueueBootTimeFitbitBackfill(bootStaggerSecondsFor("fitbit-backfill")), + }, + { + name: "fitbit-sleep-repair", + run: async () => + ( + await import("@/lib/jobs/fitbit-sleep-repair") + ).enqueueBootTimeFitbitSleepRepair( + bootStaggerSecondsFor("fitbit-sleep-repair"), + ), + }, +]; diff --git a/src/lib/fitbit/jobs/queues.ts b/src/lib/fitbit/jobs/queues.ts new file mode 100644 index 000000000..1bd11c674 --- /dev/null +++ b/src/lib/fitbit/jobs/queues.ts @@ -0,0 +1,30 @@ +/** + * The queue facts of the Fitbit jobs: names and concurrency. They live apart + * from the handlers so the worker registry and every sender can read them + * without loading the Fitbit sync. + */ +export const FITBIT_SLEEP_REPAIR_QUEUE = "fitbit-sleep-repair"; + +/** + * Serial concurrency — a repair walks a year of sleep history for one account + * in 30-day requests against the tight per-user budget; concurrency-1 keeps it + * from crowding the request pool, matching `FITBIT_BACKFILL_CONCURRENCY`. + */ +export const FITBIT_SLEEP_REPAIR_CONCURRENCY = 1; + +export const FITBIT_BACKFILL_QUEUE = "fitbit-backfill"; + +/** + * Serial concurrency — a backfill walks years of history for one account and is + * rate-bounded by Google's per-app quota; concurrency-1 keeps it from crowding + * the request pool, matching `WHOOP_BACKFILL_CONCURRENCY`. + */ +export const FITBIT_BACKFILL_CONCURRENCY = 1; + +/** Hourly Fitbit poll (no webhook). */ +export const FITBIT_SYNC_QUEUE = "fitbit-sync"; +export const FITBIT_SYNC_CRON = "8 * * * *"; + +/** Daily sweep of the Fitbit OAuth state ledger. */ +export const FITBIT_OAUTH_STATE_CLEANUP_QUEUE = "fitbit-oauth-state-cleanup"; +export const FITBIT_OAUTH_STATE_CLEANUP_CRON = "24 3 * * *"; diff --git a/src/lib/jobs/__tests__/apple-health-import-legacy-bridge.test.ts b/src/lib/jobs/__tests__/apple-health-import-legacy-bridge.test.ts index 1b50253ee..30191031c 100644 --- a/src/lib/jobs/__tests__/apple-health-import-legacy-bridge.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-legacy-bridge.test.ts @@ -25,10 +25,10 @@ vi.mock("@/lib/jobs/boss-instance", () => ({ import { APPLE_HEALTH_IMPORT_PARSER_REVISION, APPLE_HEALTH_IMPORT_SEND_OPTIONS, - APPLE_HEALTH_IMPORT_V2_QUEUE, migrateLegacyAppleHealthImport, type AppleHealthImportPayload, } from "../apple-health-import-worker"; +import { APPLE_HEALTH_IMPORT_V2_QUEUE } from "@/lib/jobs/apple-health-import-queue"; const payload: AppleHealthImportPayload = { userId: "user-1", diff --git a/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts b/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts index 6cfaf5e4c..270fe9565 100644 --- a/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-reconcile.test.ts @@ -1,5 +1,5 @@ /** - * v1.32.1 (issue #588) — periodic orphan-`ImportJob` reconcile. + * Periodic orphan-`ImportJob` reconcile (issue #588). * * Before this change `reconcileOrphanImportJobs()` only ran once, at * worker boot. A worker that crashed mid-extraction (OOM, restart) and @@ -11,10 +11,9 @@ * that one boot-time pass. These tests cover both the periodic-tick * wiring (queue provisioned, cron scheduled, handler bound) and the * reconcile logic itself (a genuinely-abandoned row flips to `failed`; - * a row another worker is still actively running does not). + * a row another worker is still actively running does not). The wiring is + * read off the worker registry. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ @@ -45,19 +44,22 @@ vi.mock("@/lib/jobs/boss-instance", () => ({ import { APPLE_HEALTH_IMPORT_PARSER_REVISION, - APPLE_HEALTH_IMPORT_V2_QUEUE, - IMPORT_JOB_RECONCILE_CRON, - IMPORT_JOB_RECONCILE_QUEUE, handleImportJobReconcileTick, reconcileOrphanImportJobs, stagedImportFilesInUse, } from "../apple-health-import-worker"; +import { + APPLE_HEALTH_IMPORT_V2_QUEUE, + IMPORT_JOB_RECONCILE_CRON, + IMPORT_JOB_RECONCILE_QUEUE, +} from "@/lib/jobs/apple-health-import-queue"; import { sweepStaleImportStaging } from "@/lib/import/apple-health-staging"; - -const maintenanceSource = readFileSync( - join(process.cwd(), "src/lib/jobs/reminder/register-maintenance.ts"), - "utf8", -); +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; beforeEach(() => { vi.resetAllMocks(); @@ -66,30 +68,21 @@ beforeEach(() => { }); describe("periodic reconcile — wiring", () => { - it("provisions the queue, schedules a 15-minute cron, and binds the handler", () => { + it("provisions the queue, schedules a 15-minute cron, and binds the handler", async () => { expect(IMPORT_JOB_RECONCILE_QUEUE).toBe("apple-health-import-reconcile"); expect(IMPORT_JOB_RECONCILE_CRON).toBe("*/15 * * * *"); - const allQueues = maintenanceSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bIMPORT_JOB_RECONCILE_QUEUE\b/); - - const schedules = maintenanceSource.match( - /const schedules[\s\S]*?=\s*\[([\s\S]*?)\];/, - ); - expect(schedules).not.toBeNull(); - // The third element pins the retry policy. It is load-bearing now that + expect(groupOf(IMPORT_JOB_RECONCILE_QUEUE).name).toBe("maintenance"); + const job = jobOf(IMPORT_JOB_RECONCILE_QUEUE); + expect(job.cron).toBe(IMPORT_JOB_RECONCILE_CRON); + // The schedule options pin the retry policy. It is load-bearing now that // the handler reports a failed outcome instead of swallowing: without // `cronIsTheRetry` a database outage would run the same doomed sweep // three times per tick, and the next tick is already the retry. - expect(schedules![1]).toMatch( - /\[IMPORT_JOB_RECONCILE_QUEUE,\s*IMPORT_JOB_RECONCILE_CRON,\s*cronIsTheRetry\]/, - ); + expect(job.scheduleOptions).toBe(cronIsTheRetry); - expect(maintenanceSource).toMatch( - /createAndWork\(\s*boss,\s*IMPORT_JOB_RECONCILE_QUEUE[\s\S]{0,120}handleImportJobReconcileTick/, + expect(await loadedHandler(IMPORT_JOB_RECONCILE_QUEUE)).toBe( + handleImportJobReconcileTick, ); }); }); diff --git a/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts b/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts index 8a3ac767b..1d645df5c 100644 --- a/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts +++ b/src/lib/jobs/__tests__/apple-health-import-send-policy.test.ts @@ -1,6 +1,5 @@ /** - * v1.28.33 (issue #486) — send-policy pins for the Apple Health import - * queue. + * Send-policy pins for the Apple Health import queue (issue #486). * * The queue defaults (retryLimit 2, expireInSeconds 900) redelivered a * long-running import after its staged `/tmp` upload was already @@ -21,20 +20,27 @@ vi.mock("@/lib/db", () => ({ import { APPLE_HEALTH_IMPORT_PARSER_REVISION, - APPLE_HEALTH_IMPORT_LEGACY_QUEUE, APPLE_HEALTH_IMPORT_SEND_OPTIONS, - APPLE_HEALTH_IMPORT_V2_QUEUE, } from "../apple-health-import-worker"; +import { + APPLE_HEALTH_IMPORT_LEGACY_QUEUE, + APPLE_HEALTH_IMPORT_V2_QUEUE, +} from "@/lib/jobs/apple-health-import-queue"; +import { + handleAppleHealthImportJobs, + handleLegacyAppleHealthImportJobs, +} from "@/lib/jobs/reminder/maintenance-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; const ROUTE_FILES = [ "src/app/api/import/apple-health-export/route.ts", "src/app/api/admin/import-apple-health-export/route.ts", ]; -const maintenanceSource = readFileSync( - join(process.cwd(), "src/lib/jobs/reminder/register-maintenance.ts"), - "utf8", -); const workerSource = readFileSync( join(process.cwd(), "src/lib/jobs/apple-health-import-worker.ts"), "utf8", @@ -80,21 +86,18 @@ describe("apple health import — parser revision boundary", () => { expect(APPLE_HEALTH_IMPORT_LEGACY_QUEUE).toBe("apple-health-import"); }); - it("drains current-parser jobs separately while bridging legacy backlog", () => { - const allQueues = maintenanceSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, + it("drains current-parser jobs separately while bridging legacy backlog", async () => { + expect(groupOf(APPLE_HEALTH_IMPORT_V2_QUEUE).name).toBe("maintenance"); + expect(groupOf(APPLE_HEALTH_IMPORT_LEGACY_QUEUE).name).toBe("maintenance"); + // Only the current-parser queue runs in the heavy lane; the legacy + // bridge re-sends its backlog onto it. + expect(jobOf(APPLE_HEALTH_IMPORT_V2_QUEUE).lane).toBe(true); + expect(jobOf(APPLE_HEALTH_IMPORT_LEGACY_QUEUE).lane).toBeUndefined(); + expect(await loadedHandler(APPLE_HEALTH_IMPORT_V2_QUEUE)).toBe( + handleAppleHealthImportJobs, ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bAPPLE_HEALTH_IMPORT_V2_QUEUE\b/); - expect(allQueues![1]).toMatch(/\bAPPLE_HEALTH_IMPORT_LEGACY_QUEUE\b/); - // The v2 window is the wider of the two: its binding wraps the batch in - // a loop that keeps the first failed outcome, so the handler call sits - // further from the queue name than the legacy bridge's does. - expect(maintenanceSource).toMatch( - /createAndWork[\s\S]{0,200}APPLE_HEALTH_IMPORT_V2_QUEUE[\s\S]{0,600}handleAppleHealthImport/, - ); - expect(maintenanceSource).toMatch( - /createAndWork[\s\S]{0,200}APPLE_HEALTH_IMPORT_LEGACY_QUEUE[\s\S]{0,400}migrateLegacyAppleHealthImport/, + expect(await loadedHandler(APPLE_HEALTH_IMPORT_LEGACY_QUEUE)).toBe( + handleLegacyAppleHealthImportJobs, ); }); diff --git a/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts b/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts index bb6147eab..e9daf5438 100644 --- a/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts +++ b/src/lib/jobs/__tests__/coach-memory-refresh-queue.test.ts @@ -1,49 +1,35 @@ /** - * v1.11.1 — queue-registration guard for the combined Coach memory-refresh - * job (rolling conversation summary + durable fact extraction). + * Queue-registration guard for the combined Coach memory-refresh job + * (rolling conversation summary + durable fact extraction). * - * Same source-text-grep approach as the period-narrative / stress-strain - * guards: assert the queue is imported, registered in `allQueues`, and wired - * to a `createAndWork` handler that runs the refresh pipeline — without booting - * pg-boss + Prisma. An unregistered queue silently never runs (the recurring - * past bug this guards against; v1.4.37 W10 caught exactly this class). + * Reads the worker registry: the queue is declared in the status group and + * its loader resolves to the handler that runs the refresh pipeline. An + * unregistered queue silently never runs (the dead-queue class). */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -// v1.18.1 — the coach-memory-refresh wiring moved out of the 2143-LOC -// reminder-worker boot file into the status registrar. The dead-queue guard -// follows the wiring there. -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-status.ts"); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { COACH_MEMORY_REFRESH_QUEUE } from "@/lib/ai/coach/coach-memory-shared"; +import { handleCoachMemoryRefresh } from "@/lib/jobs/reminder/status-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -function allQueuesBlock(): string { - const m = source.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(m).not.toBeNull(); - return m![1]; -} - -describe("reminder-worker — coach-memory-refresh wiring", () => { - it("imports the queue symbol + worker from the coach-memory modules", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/ai\/coach\/coach-memory-shared["']/, - ); - expect(source).toMatch(/\bCOACH_MEMORY_REFRESH_QUEUE\b/); - expect(source).toMatch( - /from\s*["']@\/lib\/ai\/coach\/coach-memory-refresh-worker["']/, - ); - expect(source).toMatch(/\brunCoachMemoryRefresh\b/); +describe("coach-memory-refresh registration", () => { + it("declares the coach-memory-refresh queue in the status group", () => { + expect(groupOf(COACH_MEMORY_REFRESH_QUEUE).name).toBe("status"); }); - it("registers the coach-memory-refresh queue in allQueues", () => { - expect(allQueuesBlock()).toMatch(/\bCOACH_MEMORY_REFRESH_QUEUE\b/); + it("runs one refresh at a time and has no cron", () => { + const job = jobOf(COACH_MEMORY_REFRESH_QUEUE); + expect(job.work).toEqual({ localConcurrency: 1 }); + expect(job.cron).toBeUndefined(); }); - it("registers a createAndWork handler that runs the refresh pipeline", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}COACH_MEMORY_REFRESH_QUEUE[\s\S]{0,600}runCoachMemoryRefresh/, + it("binds the handler that runs the refresh pipeline", async () => { + expect(await loadedHandler(COACH_MEMORY_REFRESH_QUEUE)).toBe( + handleCoachMemoryRefresh, ); }); }); diff --git a/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts b/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts index 0567b4e40..1999316f6 100644 --- a/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts +++ b/src/lib/jobs/__tests__/coach-message-cleanup-queue.test.ts @@ -1,58 +1,50 @@ /** - * v1.18.7 — coach-message-cleanup queue registration guard. + * Coach-message-cleanup queue registration guard. * - * Same source-text-grep approach as the other queue-wiring guards: assert the - * queue is registered in `allQueues`, scheduled, and wired to a `createAndWork` - * handler — without booting pg-boss + Prisma. An unregistered queue silently - * never drains (the recurring v1.4.37 dead-queue bug), which would leave the - * encrypted Coach history growing unbounded with the retention job dark. + * Reads the worker registry: the queue is declared in the maintenance group, + * scheduled with the cron as its retry, and bound to the cleanup handler. An + * unregistered queue silently never drains (the dead-queue class), which + * would leave the encrypted Coach history growing unbounded with the + * retention job dark. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-maintenance.ts", +import { handleCoachMessageCleanup } from "@/lib/jobs/reminder/cleanup-handlers"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; + +const QUEUE = "coach-message-cleanup"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "cleanup-handlers.ts"), + "utf8", ); -const workerSource = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync( - join(__dirname, "..", "reminder", "cleanup-handlers.ts"), - "utf8", - ); -describe("reminder-worker — coach-message-cleanup wiring", () => { - it("imports the cleanup handler from the cleanup-handlers module", () => { - expect(workerSource).toMatch(/\bhandleCoachMessageCleanup\b/); - expect(workerSource).toMatch(/\bCoachMessageCleanupPayload\b/); +describe("coach-message-cleanup registration", () => { + it("declares the queue in the maintenance group", () => { + expect(groupOf(QUEUE).name).toBe("maintenance"); }); - it("registers the coach-message-cleanup queue in the allQueues loop", () => { - const allQueuesMatch = workerSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bCOACH_MESSAGE_CLEANUP_QUEUE\b/); + it("schedules the nightly cron with the cron as the retry", () => { + const job = jobOf(QUEUE); + expect(job.cron).toBe("50 3 * * *"); + expect(job.scheduleOptions).toBe(cronIsTheRetry); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("schedules the coach-message-cleanup cron", () => { - expect(workerSource).toMatch( - /\[COACH_MESSAGE_CLEANUP_QUEUE,\s*COACH_MESSAGE_CLEANUP_CRON,\s*cronIsTheRetry\]/, - ); - }); - - it("registers a createAndWork handler for the queue", () => { - expect(workerSource).toMatch( - /createAndWork[\s\S]{0,200}COACH_MESSAGE_CLEANUP_QUEUE[\s\S]{0,200}handleCoachMessageCleanup/, - ); + it("binds the cleanup handler", async () => { + expect(await loadedHandler(QUEUE)).toBe(handleCoachMessageCleanup); }); it("prunes coach messages inside the handler", () => { - expect(workerSource).toMatch( + expect(handlerSource).toMatch( /handleCoachMessageCleanup[\s\S]{0,400}cleanupOldCoachMessages/, ); }); diff --git a/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts b/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts index b93282312..96bb7da83 100644 --- a/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts +++ b/src/lib/jobs/__tests__/cumulative-pr-rederive-queue.test.ts @@ -1,48 +1,46 @@ /** - * v1.30.3 — cumulative-PersonalRecord rederivation queue - * registration guard. + * Cumulative-PersonalRecord rederivation queue registration guard. * - * Same source-text-grep approach as the step-consolidation-repair guard: - * assert the queue is registered in `allQueues`, a `createAndWork` handler is - * wired against it, and the boot-discovery enqueue helper fires — without - * booting pg-boss + Prisma. An unregistered queue silently never drains - * (the v1.4.37 dead-queue class). + * Reads the worker registry: the queue is declared in the rollup group, + * bound to the rederivation handler, and offered again by the rollup boot + * discovery. An unregistered queue silently never drains (the dead-queue + * class). */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-rollup.ts"); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { handleCumulativePrRederive } from "@/lib/jobs/reminder/rollup-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { + CUMULATIVE_PR_REDERIVE_CONCURRENCY, + CUMULATIVE_PR_REDERIVE_QUEUE, +} from "@/lib/personal-records/cumulative-pr-rederivation"; -describe("reminder-worker — cumulative-pr-rederive wiring", () => { - it("imports the queue symbols from the rederivation module", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/personal-records\/cumulative-pr-rederivation["']/, - ); - expect(source).toMatch(/\bCUMULATIVE_PR_REDERIVE_QUEUE\b/); - expect(source).toMatch(/\brunCumulativePrRederivationForUser\b/); - expect(source).toMatch(/\benqueueBootTimeCumulativePrRederivation\b/); +describe("cumulative-pr-rederive registration", () => { + it("declares the queue in the rollup group", () => { + expect(groupOf(CUMULATIVE_PR_REDERIVE_QUEUE).name).toBe("rollup"); }); - it("registers the queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bCUMULATIVE_PR_REDERIVE_QUEUE\b/); + it("is a per-user exclusive one-shot at the module's concurrency", () => { + const job = jobOf(CUMULATIVE_PR_REDERIVE_QUEUE); + expect(job.policy?.policy).toBe("exclusive"); + expect(job.work).toEqual({ + localConcurrency: CUMULATIVE_PR_REDERIVE_CONCURRENCY, + }); + expect(job.cron).toBeUndefined(); }); - it("registers a createAndWork handler against the queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}CUMULATIVE_PR_REDERIVE_QUEUE[\s\S]{0,400}runCumulativePrRederivationForUser/, + it("binds the rederivation handler", async () => { + expect(await loadedHandler(CUMULATIVE_PR_REDERIVE_QUEUE)).toBe( + handleCumulativePrRederive, ); }); - it("fires the boot-discovery enqueue helper", () => { - expect(source).toMatch( - /await enqueueBootTimeCumulativePrRederivation\([^)]*\)/, - ); + it("runs the boot discovery pass", () => { + expect(discoveryOf("rollup")).toContain("cumulative-pr-rederive"); }); }); diff --git a/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts b/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts index 8c9158af2..8be8a1395 100644 --- a/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts +++ b/src/lib/jobs/__tests__/cycle-reminder-queue.test.ts @@ -1,67 +1,55 @@ /** - * v1.15 — cycle-reminder queue registration guard. + * Cycle-reminder queue registration guard. * - * Same source-text-grep approach as the mood-reminder / drain / mean- - * consolidation registration guards: assert the cycle-reminder queue is - * declared, listed in `allQueues` (the v1.4.37 dead-queue lesson — an - * unregistered queue is never provisioned and the schedule silently - * no-ops), scheduled on its cron, and wired to a `createAndWork` handler that - * delegates to `runCycleReminderTick`. No pg-boss / Prisma boot. + * Reads the worker registry: the cycle-reminder queue is declared in the + * reminders group (an unregistered queue is never provisioned and the + * schedule silently no-ops), scheduled on its 15-minute cron, and bound to + * the handler that delegates to `runCycleReminderTick`. No pg-boss / Prisma + * boot. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the reminder-dispatch wiring moved out of the 2143-LOC -// reminder-worker boot file into the reminders registrar. The dead-queue guard -// follows the wiring there (the handler module is still concatenated for the -// handler-symbol assertions). -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-reminders.ts", +import { handleCycleReminderCheck } from "@/lib/jobs/reminder/mood-cycle-checks"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; + +const CYCLE_REMINDER_QUEUE = "cycle-reminder-check"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "mood-cycle-checks.ts"), + "utf8", ); -const source = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync( - join(__dirname, "..", "reminder", "mood-cycle-checks.ts"), - "utf8", - ); -describe("reminder-worker — cycle-reminder wiring", () => { +describe("cycle-reminder registration", () => { it("imports the cycle-reminder tick runner", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/cycle-reminder["']/); - expect(source).toMatch(/\brunCycleReminderTick\b/); + expect(handlerSource).toMatch( + /from\s*["']@\/lib\/jobs\/cycle-reminder["']/, + ); + expect(handlerSource).toMatch(/\brunCycleReminderTick\b/); }); - it("declares the queue + cron constants", () => { - expect(source).toMatch( - /CYCLE_REMINDER_QUEUE\s*=\s*["']cycle-reminder-check["']/, - ); - expect(source).toMatch( - /CYCLE_REMINDER_CRON\s*=\s*["']\*\/15 \* \* \* \*["']/, - ); + it("declares the cycle-reminder queue in the reminders group", () => { + expect(groupOf(CYCLE_REMINDER_QUEUE).name).toBe("reminders"); }); - it("registers the cycle-reminder queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bCYCLE_REMINDER_QUEUE\b/); + it("schedules the cycle-reminder cron, single-flight", () => { + const job = jobOf(CYCLE_REMINDER_QUEUE); + expect(job.cron).toBe("*/15 * * * *"); + expect(job.scheduleOptions).toBeUndefined(); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("schedules the cycle-reminder cron", () => { - expect(source).toMatch(/\[CYCLE_REMINDER_QUEUE,\s*CYCLE_REMINDER_CRON\]/); - }); - - it("registers a createAndWork handler that runs the cycle-reminder tick", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}CYCLE_REMINDER_QUEUE[\s\S]{0,200}handleCycleReminderCheck/, + it("binds the handler that runs the cycle-reminder tick", async () => { + expect(await loadedHandler(CYCLE_REMINDER_QUEUE)).toBe( + handleCycleReminderCheck, ); - expect(source).toMatch( + expect(handlerSource).toMatch( /handleCycleReminderCheck[\s\S]{0,400}runCycleReminderTick/, ); }); diff --git a/src/lib/jobs/__tests__/daily-briefing.test.ts b/src/lib/jobs/__tests__/daily-briefing.test.ts index 8c7a5ca43..67c01ed9e 100644 --- a/src/lib/jobs/__tests__/daily-briefing.test.ts +++ b/src/lib/jobs/__tests__/daily-briefing.test.ts @@ -1,19 +1,47 @@ /** * Daily-briefing fallback cron: tz slot-gating + result accounting, plus - * the dead-queue wiring guard (imported / in allQueues / scheduled / worked). + * the dead-queue wiring guard read off the worker registry (declared, + * scheduled, bound) and the morning push the finalised refresh fires. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, it, expect, vi } from "vitest"; + +const { runMorningDigestRefresh, maybeDispatchPush } = vi.hoisted(() => ({ + runMorningDigestRefresh: vi.fn(), + maybeDispatchPush: vi.fn(async () => "sent"), +})); +vi.mock("@/lib/jobs/morning-digest-refresh", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/morning-digest-refresh") + >()), + runMorningDigestRefresh, +})); +vi.mock("@/lib/daily/daily-briefing-push", async (importOriginal) => ({ + ...(await importOriginal()), + maybeDispatchDailyBriefing: maybeDispatchPush, +})); +vi.mock("@/lib/jobs/reminder/shared", async (importOriginal) => ({ + ...(await importOriginal()), + getWorkerPrisma: () => ({}), +})); import type { PrismaClient } from "@/generated/prisma/client"; +import { runDailyBriefingTick } from "@/lib/jobs/daily-briefing"; import { - runDailyBriefingTick, DAILY_BRIEFING_QUEUE, DAILY_BRIEFING_CRON, -} from "@/lib/jobs/daily-briefing"; +} from "@/lib/jobs/daily-briefing-queue"; import type { maybeDispatchDailyBriefing } from "@/lib/daily/daily-briefing-push"; +import type { MorningDigestRefreshPayload } from "@/lib/jobs/morning-digest-refresh-shared"; +import { + handleDailyBriefing, + handleMorningDigestRefresh, +} from "@/lib/jobs/reminder/status-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import type { Job } from "pg-boss"; // 06:00Z → 08:00 Berlin (the fallback hour) but 02:00 in New York (not the // slot), so a mixed-tz cohort exercises the per-user local-hour gate. @@ -76,38 +104,44 @@ describe("runDailyBriefingTick — slot gating", () => { }); }); -describe("reminder-worker — daily-briefing wiring", () => { - const registrar = readFileSync( - join(__dirname, "..", "reminder", "register-status.ts"), - "utf8", - ); +describe("daily-briefing registration", () => { + function refreshJob(): Job { + return { + id: "j1", + name: "morning-digest-refresh", + data: { userId: "u1", localDate: "2026-07-16" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }; + } - it("imports the queue symbols from the daily-briefing module", () => { - expect(registrar).toMatch(/from\s*["']@\/lib\/jobs\/daily-briefing["']/); - expect(registrar).toMatch(/\bDAILY_BRIEFING_QUEUE\b/); - expect(registrar).toMatch(/\bDAILY_BRIEFING_CRON\b/); - expect(registrar).toMatch(/\brunDailyBriefingTick\b/); + it("declares the queue in the status group and schedules the cron", () => { + expect(groupOf(DAILY_BRIEFING_QUEUE).name).toBe("status"); + const job = jobOf(DAILY_BRIEFING_QUEUE); + expect(job.cron).toBe(DAILY_BRIEFING_CRON); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("registers the queue in allQueues and schedules the cron", () => { - const allQueues = registrar.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bDAILY_BRIEFING_QUEUE\b/); - expect(registrar).toMatch( - /\[DAILY_BRIEFING_QUEUE,\s*DAILY_BRIEFING_CRON\]/, - ); + it("binds the handler that runs the tick", async () => { + expect(await loadedHandler(DAILY_BRIEFING_QUEUE)).toBe(handleDailyBriefing); }); - it("wires a createAndWork handler that runs the tick", () => { - expect(registrar).toMatch( - /createAndWork[\s\S]{0,200}DAILY_BRIEFING_QUEUE[\s\S]{0,400}runDailyBriefingTick/, + it("fires the finalisation-hook push after a finalised morning refresh", async () => { + runMorningDigestRefresh.mockResolvedValueOnce({ status: "finalised" }); + await handleMorningDigestRefresh([refreshJob()]); + expect(maybeDispatchPush).toHaveBeenCalledTimes(1); + expect(maybeDispatchPush).toHaveBeenCalledWith( + expect.anything(), + "u1", + expect.any(Date), ); - }); - it("fires the finalisation-hook push after a finalised morning refresh", () => { - expect(registrar).toMatch( - /result\.status\s*===\s*["']finalised["'][\s\S]{0,300}maybeDispatchDailyBriefing/, - ); + maybeDispatchPush.mockClear(); + runMorningDigestRefresh.mockResolvedValueOnce({ status: "already-final" }); + await handleMorningDigestRefresh([refreshJob()]); + expect(maybeDispatchPush).not.toHaveBeenCalled(); }); it("uses an every-15-min cron for the fallback slot", () => { diff --git a/src/lib/jobs/__tests__/data-arrival-queue.test.ts b/src/lib/jobs/__tests__/data-arrival-queue.test.ts index a86f8057f..b97123667 100644 --- a/src/lib/jobs/__tests__/data-arrival-queue.test.ts +++ b/src/lib/jobs/__tests__/data-arrival-queue.test.ts @@ -1,5 +1,5 @@ /** - * v1.4.37 dead-queue guard for the `data-arrival` queue. + * Dead-queue guard for the `data-arrival` queue. * * Four facts have to hold together or the spine is silently inert: the queue * name is provisioned, a handler drains it, the queue carries a policy that @@ -7,83 +7,65 @@ * is itself registered. Any one of them missing fails open in the worst way — * the emits succeed, the jobs vanish, and nothing anywhere reports a problem. * - * Source-text assertions over the registrar modules, matching the shape of the - * sibling `*-queue.test.ts` guards. + * Reads the worker registry for the queue facts; the retention handler's body + * is read as text. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -const statusRegistrar = readFileSync( - join(__dirname, "..", "reminder", "register-status.ts"), +import { DATA_ARRIVAL_QUEUE } from "@/lib/arrivals/emit-shared"; +import { handleDataArrival } from "@/lib/jobs/data-arrival"; +import { handleArrivalReactionCleanup } from "@/lib/jobs/reminder/cleanup-handlers"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; + +const ARRIVAL_REACTION_CLEANUP_QUEUE = "arrival-reaction-cleanup"; + +const cleanupHandlers = readFileSync( + join(__dirname, "..", "reminder", "cleanup-handlers.ts"), "utf8", ); -const maintenanceSource = - readFileSync( - join(__dirname, "..", "reminder", "register-maintenance.ts"), - "utf8", - ) + - readFileSync( - join(__dirname, "..", "reminder", "cleanup-handlers.ts"), - "utf8", - ); - describe("data-arrival queue wiring", () => { - it("provisions the queue in the allQueues list", () => { - const allQueues = statusRegistrar.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bDATA_ARRIVAL_QUEUE\b/); + it("provisions the queue in the status group", () => { + expect(groupOf(DATA_ARRIVAL_QUEUE).name).toBe("status"); }); - it("binds a createAndWork handler that drains it", () => { - expect(statusRegistrar).toMatch( - /createAndWork[\s\S]{0,200}DATA_ARRIVAL_QUEUE[\s\S]{0,200}handleDataArrival/, - ); + it("binds the handler that drains it", async () => { + expect(await loadedHandler(DATA_ARRIVAL_QUEUE)).toBe(handleDataArrival); }); it("carries the exclusive policy its singleton keys depend on", () => { // Under pg-boss's default `standard` policy NO index covers // `singleton_key`, so the day-scoped keys would coalesce nothing at all and - // a chatty ingest would queue one job per batch. This assertion is anchored - // INSIDE the extracted policy table so an unrelated mention of the queue - // name elsewhere in the file cannot satisfy it. - const table = statusRegistrar.match( - /const queuePolicies: QueuePolicyTable\s*=\s*\{([\s\S]*?)\n\};/, - ); - expect(table).not.toBeNull(); - expect(table![1]).toMatch( - /\[DATA_ARRIVAL_QUEUE\][\s\S]{0,200}policy:\s*"exclusive"/, - ); + // a chatty ingest would queue one job per batch. + expect(jobOf(DATA_ARRIVAL_QUEUE).policy?.policy).toBe("exclusive"); }); it("has no cron schedule — it is send-only, driven by ingest", () => { - // Tolerates the optional third schedule element, so a cron sneaking in - // with send options still trips this. - expect(statusRegistrar).not.toMatch( - /\[\s*DATA_ARRIVAL_QUEUE,\s*\w+_CRON\s*[,\]]/, - ); + const job = jobOf(DATA_ARRIVAL_QUEUE); + expect(job.cron).toBeUndefined(); + expect(job.scheduleOptions).toBeUndefined(); }); - it("registers the retention job that prunes the markers", () => { - const allQueues = maintenanceSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bARRIVAL_REACTION_CLEANUP_QUEUE\b/); - expect(maintenanceSource).toMatch( - /\[\s*ARRIVAL_REACTION_CLEANUP_QUEUE,\s*ARRIVAL_REACTION_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, - ); - expect(maintenanceSource).toMatch( - /createAndWork[\s\S]{0,200}ARRIVAL_REACTION_CLEANUP_QUEUE[\s\S]{0,200}handleArrivalReactionCleanup/, + it("registers the retention job that prunes the markers", async () => { + expect(groupOf(ARRIVAL_REACTION_CLEANUP_QUEUE).name).toBe("maintenance"); + const job = jobOf(ARRIVAL_REACTION_CLEANUP_QUEUE); + expect(job.cron).toBe("10 4 * * *"); + expect(job.scheduleOptions).toBe(cronIsTheRetry); + expect(await loadedHandler(ARRIVAL_REACTION_CLEANUP_QUEUE)).toBe( + handleArrivalReactionCleanup, ); }); it("the retention handler actually deletes arrival reactions by age", () => { - expect(maintenanceSource).toMatch( + expect(cleanupHandlers).toMatch( /handleArrivalReactionCleanup[\s\S]{0,600}arrivalReaction\.deleteMany/, ); }); diff --git a/src/lib/jobs/__tests__/data-backup-send-policy.test.ts b/src/lib/jobs/__tests__/data-backup-send-policy.test.ts index c7e470c91..a24ba53ce 100644 --- a/src/lib/jobs/__tests__/data-backup-send-policy.test.ts +++ b/src/lib/jobs/__tests__/data-backup-send-policy.test.ts @@ -29,7 +29,8 @@ import { import { OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_SEND_OPTIONS, -} from "../offhost-backup"; +} from "@/lib/jobs/offhost-backup-queue"; +import { jobOf } from "../registry/__tests__/registry-helpers"; const read = (file: string) => readFileSync(join(process.cwd(), file), "utf8"); @@ -59,13 +60,12 @@ describe("data-backup — pg-boss send policy", () => { }); it("the weekly and nightly schedules carry the policy", () => { - const source = read("src/lib/jobs/reminder/register-maintenance.ts"); - expect(source).toMatch( - /\[\s*DATA_BACKUP_QUEUE,\s*DATA_BACKUP_CRON,\s*DATA_BACKUP_SEND_OPTIONS,?\s*\]/, - ); - expect(source).toMatch( - /\[\s*OFFHOST_BACKUP_QUEUE,\s*OFFHOST_BACKUP_CRON,\s*OFFHOST_BACKUP_SEND_OPTIONS,?\s*\]/, - ); + const weekly = jobOf(DATA_BACKUP_QUEUE); + expect(weekly.cron).toBe("0 3 * * 0"); + expect(weekly.scheduleOptions).toBe(DATA_BACKUP_SEND_OPTIONS); + const nightly = jobOf(OFFHOST_BACKUP_QUEUE); + expect(nightly.cron).toBe("30 2 * * *"); + expect(nightly.scheduleOptions).toBe(OFFHOST_BACKUP_SEND_OPTIONS); }); it("the admin run-now route sends with the same policy", () => { diff --git a/src/lib/jobs/__tests__/document-ai-run.test.ts b/src/lib/jobs/__tests__/document-ai-run.test.ts index 5fc7878f1..3f8bf1757 100644 --- a/src/lib/jobs/__tests__/document-ai-run.test.ts +++ b/src/lib/jobs/__tests__/document-ai-run.test.ts @@ -112,11 +112,11 @@ vi.mock("@/lib/jobs/boss-instance", () => ({ import { AiUnavailableError } from "@/lib/ai/capabilities/refusal"; import { DOCUMENT_AI_RUN_EXPIRE_SECONDS, - DOCUMENT_AI_RUN_QUEUE, enqueueDocumentAiRun, handleDocumentAiRunJobs, runDocumentAiRun, } from "@/lib/jobs/document-ai-run"; +import { DOCUMENT_AI_RUN_QUEUE } from "@/lib/jobs/document-ai-run-queue"; const NOW = 1_800_000_000_000; const now = () => NOW; diff --git a/src/lib/jobs/__tests__/document-index.test.ts b/src/lib/jobs/__tests__/document-index.test.ts index 3d7652ae9..456005ac2 100644 --- a/src/lib/jobs/__tests__/document-index.test.ts +++ b/src/lib/jobs/__tests__/document-index.test.ts @@ -15,11 +15,8 @@ vi.mock("@/lib/documents/auto-stage-labs", () => ({ maybeAutoStageLabFacts: vi.fn().mockResolvedValue(undefined), })); -import { - DOCUMENT_INDEX_QUEUE, - enqueueDocumentIndex, - runDocumentIndex, -} from "../document-index"; +import { enqueueDocumentIndex, runDocumentIndex } from "../document-index"; +import { DOCUMENT_INDEX_QUEUE } from "../document-index-queue"; import { indexDocumentContent } from "@/lib/documents/index-document"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { maybeAutoStageLabFacts } from "@/lib/documents/auto-stage-labs"; diff --git a/src/lib/jobs/__tests__/document-summary-catchup.test.ts b/src/lib/jobs/__tests__/document-summary-catchup.test.ts index 96c7dd4da..111a6afb7 100644 --- a/src/lib/jobs/__tests__/document-summary-catchup.test.ts +++ b/src/lib/jobs/__tests__/document-summary-catchup.test.ts @@ -32,8 +32,8 @@ import { enqueueSummaryCatchUp, runSummaryCatchUpForUser, MAX_ENQUEUES_PER_RUN, - DOCUMENT_SUMMARY_CATCHUP_QUEUE, } from "../document-summary-catchup"; +import { DOCUMENT_SUMMARY_CATCHUP_QUEUE } from "../document-summary-catchup-queue"; import { prisma } from "@/lib/db"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { documentAutoReadEnabled } from "@/lib/documents/document-settings"; diff --git a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts index 412d147fd..65c0ecb63 100644 --- a/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts +++ b/src/lib/jobs/__tests__/drain-cumulative-queue.test.ts @@ -1,75 +1,136 @@ /** - * v1.4.37 — pg-boss queue registration for the nightly drain of - * per-sample APPLE_HEALTH cumulative rows. The drain helper exists - * since v1.4.30 but only ran via the admin endpoint / CLI; the - * scheduled wrapper closes the loop so the list view stops painting - * hundreds of step chunks per day. + * pg-boss queue registration for the nightly drain of per-sample + * APPLE_HEALTH cumulative rows. The drain helper once ran only via the admin + * endpoint / CLI; the scheduled wrapper closes the loop so the list view + * stops painting hundreds of step chunks per day. * - * Same source-text-grep approach as the Withings queue regression - * guard so we never have to boot pg-boss + Prisma to assert the - * scheduler wiring stays intact. + * Reads the worker registry (the queue, its cron and its binding) and runs + * the bound handler against mocked passes, so pg-boss and Prisma never boot. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; -import { describe, expect, it } from "vitest"; +const drainPerSampleCumulative = vi.hoisted(() => + vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + bucketsCollapsed: 0, + perSampleRowsDeleted: 0, + dailyRowsUpserted: 0, + }, + })), +); +vi.mock( + "@/lib/measurements/drain-per-sample-cumulative", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/drain-per-sample-cumulative") + >()), + drainPerSampleCumulative, + }), +); +vi.mock( + "@/lib/measurements/consolidate-daily-mean", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/consolidate-daily-mean") + >()), + consolidateDailyMean: vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + daysConsolidated: 0, + perSampleRowsSoftDeleted: 0, + dailyRowsUpserted: 0, + daysFailed: 0, + }, + })), + }), +); +vi.mock( + "@/lib/measurements/dense-intraday-retention", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/dense-intraday-retention") + >()), + runDenseIntradayRetention: vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + daysConsolidated: 0, + perSampleRowsSoftDeleted: 0, + hourlyRowsUpserted: 0, + dailyRowsRetired: 0, + }, + })), + }), +); +vi.mock("@/lib/jobs/reminder/shared", async (importOriginal) => ({ + ...(await importOriginal()), + getWorkerPrisma: () => ({}), +})); -// v1.18.1 — the drain-cumulative wiring moved out of the 2143-LOC -// reminder-worker boot file into the rollup registrar. The dead-queue guard -// follows the wiring there. -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-rollup.ts"); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { + DRAIN_CUMULATIVE_CRON, + DRAIN_CUMULATIVE_QUEUE, +} from "@/lib/jobs/drain-cumulative-queue"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + groupOf, + jobOf, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { DRAIN_CUMULATIVE_CUTOFF_HOURS } from "@/lib/measurements/drain-per-sample-cumulative"; -describe("reminder-worker — drainPerSampleCumulative nightly schedule", () => { +describe("drainPerSampleCumulative nightly schedule", () => { it("declares the drain queue at the documented Berlin cadence", () => { - expect(source).toMatch( - /DRAIN_CUMULATIVE_QUEUE\s*=\s*["']drain-per-sample-cumulative["']/, - ); + expect(DRAIN_CUMULATIVE_QUEUE).toBe("drain-per-sample-cumulative"); // 03:45 Europe/Berlin — between audit-log cleanup (03:15) and the // feedback aggregator (04:00). Keep the slot stable so an operator // staring at the cron table can find the job easily. - expect(source).toMatch(/DRAIN_CUMULATIVE_CRON\s*=\s*["']45 3 \* \* \*["']/); + expect(DRAIN_CUMULATIVE_CRON).toBe("45 3 * * *"); }); - it("passes the 36 hour grace window into the drain helper", () => { - // v1.4.38 — the literal `36` lives in the helper module so the - // worker, the admin route, and the CLI all read one source of - // truth. The worker still references the constant by name in the - // cutoffHours slot so the cron's behaviour is unchanged. - expect(source).toMatch(/cutoffHours:\s*DRAIN_CUMULATIVE_CUTOFF_HOURS/); - }); - - it("imports the cutoff constant and the helper from the drain module", () => { - // v1.4.38 — the import is now a multi-symbol form. Match the helper - // and the constant separately so a future contributor who adds a - // third symbol does not break the assertion. - expect(source).toMatch( - /from\s*["']@\/lib\/measurements\/drain-per-sample-cumulative["']/, + it("passes the 36 hour grace window into the drain helper", async () => { + // The literal `36` lives in the helper module so the worker, the admin + // route, and the CLI all read one source of truth. + expect(DRAIN_CUMULATIVE_CUTOFF_HOURS).toBe(36); + const { boss, sends } = recordingBoss(); + const handler = await jobOf(DRAIN_CUMULATIVE_QUEUE).load({ boss }); + await handler([ + { + id: "j1", + name: DRAIN_CUMULATIVE_QUEUE, + data: { triggeredAt: "2026-01-01T03:45:00.000Z" }, + expireInSeconds: 900, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + expect(drainPerSampleCumulative).toHaveBeenCalledTimes(1); + expect(drainPerSampleCumulative).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + dryRun: false, + cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS, + }), ); - expect(source).toMatch(/\bdrainPerSampleCumulative\b/); - expect(source).toMatch(/\bDRAIN_CUMULATIVE_CUTOFF_HOURS\b/); + // A tick that finished inside its budget hands nothing on. + expect(sends).toEqual([]); }); - it("registers a createAndWork handler against the drain queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}DRAIN_CUMULATIVE_QUEUE[\s\S]{0,1100}drainPerSampleCumulative/, - ); + it("schedules the drain cron with the cron as the retry", () => { + const job = jobOf(DRAIN_CUMULATIVE_QUEUE); + expect(job.cron).toBe(DRAIN_CUMULATIVE_CRON); + expect(job.scheduleOptions).toBe(cronIsTheRetry); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("schedules the drain cron via boss.schedule (allQueues + schedules)", () => { - expect(source).toMatch( - /\[DRAIN_CUMULATIVE_QUEUE,\s*DRAIN_CUMULATIVE_CRON,\s*cronIsTheRetry\]/, - ); - }); - - it("registers the drain queue in the allQueues createQueue loop", () => { - // pg-boss v12 requires explicit createQueue before scheduling. The - // allQueues array drives the boot-time `for (const q of allQueues)` - // loop; missing the entry silently no-ops the schedule below. - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bDRAIN_CUMULATIVE_QUEUE\b/); + it("declares the drain queue in the rollup group", () => { + // pg-boss v12 requires explicit createQueue before scheduling; a queue + // the registry does not declare is never created and its schedule + // silently no-ops. + expect(groupOf(DRAIN_CUMULATIVE_QUEUE).name).toBe("rollup"); }); }); diff --git a/src/lib/jobs/__tests__/fitbit-queues.test.ts b/src/lib/jobs/__tests__/fitbit-queues.test.ts index a9e9fbbc6..988c8fcf8 100644 --- a/src/lib/jobs/__tests__/fitbit-queues.test.ts +++ b/src/lib/jobs/__tests__/fitbit-queues.test.ts @@ -1,12 +1,12 @@ /** - * v1.12.0 — pg-boss queue registration for the Fitbit / Google Health sync layer. + * pg-boss queue registration for the Fitbit sync layer. * - * The reminder-worker imports heavy infrastructure (pg-boss, Prisma adapter, - * notification dispatchers, …) so we don't boot the whole worker here. Instead we - * read the source as text and assert the queue constants + cron schedules + - * handler wiring + allQueues + boot-enqueue are present — the fast guard that - * catches the v1.4.37 dead-queue class (a queue declared but never registered in - * `allQueues`, which silently never drains). + * Reads the worker registry instead of booting the worker: every Fitbit queue + * is declared in the integration-sync group with its cron and handler, the + * full-history queues forward to the shared admission lane, and the boot + * discovery offers the backfill and the sleep repair again. This is the fast + * guard for the dead-queue class (a queue declared but never registered, + * which silently never drains). * * Fitbit is poll-only at launch (no webhook — Pub/Sub deferred), so a single * hourly `fitbit-sync` queue drives the per-user `syncUserFitbit` driver across @@ -17,81 +17,101 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring + boot discovery moved out of the 2143-LOC -// reminder-worker boot file into domain registrars; the Fitbit queues live in -// the integration-sync registrar. The dead-queue guard follows the wiring there -// (the handler module is still concatenated for the import-routine assertions). -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-integration-sync.ts", -); -const source = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync(join(__dirname, "..", "reminder", "fitbit-sync.ts"), "utf8"); +import { + FITBIT_BACKFILL_QUEUE, + FITBIT_OAUTH_STATE_CLEANUP_CRON, + FITBIT_OAUTH_STATE_CLEANUP_QUEUE, + FITBIT_SLEEP_REPAIR_QUEUE, + FITBIT_SYNC_CRON, + FITBIT_SYNC_QUEUE, +} from "@/lib/fitbit/jobs/queues"; +import { INTEGRATION_BACKFILL_ADMISSION_QUEUE } from "@/lib/jobs/integration-backfill-admission"; +import { + handleFitbitOAuthStateCleanup, + handleFitbitSync, +} from "@/lib/jobs/reminder/fitbit-sync"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -const FITBIT_QUEUE_CONSTS = [ - "FITBIT_SYNC_QUEUE", - "FITBIT_BACKFILL_QUEUE", - "FITBIT_SLEEP_REPAIR_QUEUE", - "FITBIT_OAUTH_STATE_CLEANUP_QUEUE", +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "fitbit-sync.ts"), + "utf8", +); + +const FITBIT_QUEUES = [ + FITBIT_SYNC_QUEUE, + FITBIT_BACKFILL_QUEUE, + FITBIT_SLEEP_REPAIR_QUEUE, + FITBIT_OAUTH_STATE_CLEANUP_QUEUE, ] as const; -describe("reminder-worker — Fitbit sync queues", () => { +/** The admission sends the source queue's handler makes for one job. */ +async function forwardOneJob(queue: string) { + const { boss, sends } = recordingBoss(); + const handler = await jobOf(queue).load({ boss }); + await handler([ + { + id: "j1", + name: queue, + data: { userId: "u1" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + return sends; +} + +describe("Fitbit sync queues", () => { it("declares the Fitbit sync + oauth-state-cleanup queue constants", () => { - expect(source).toMatch(/FITBIT_SYNC_QUEUE\s*=\s*["']fitbit-sync["']/); - expect(source).toMatch( - /FITBIT_OAUTH_STATE_CLEANUP_QUEUE\s*=\s*["']fitbit-oauth-state-cleanup["']/, - ); + expect(FITBIT_SYNC_QUEUE).toBe("fitbit-sync"); + expect(FITBIT_OAUTH_STATE_CLEANUP_QUEUE).toBe("fitbit-oauth-state-cleanup"); }); - it("registers EVERY Fitbit queue in allQueues (v1.4.37 dead-queue guard)", () => { - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - const body = block![1]!; - for (const q of FITBIT_QUEUE_CONSTS) { - expect(body).toContain(q); + it("declares EVERY Fitbit queue in the integration-sync group", () => { + for (const q of FITBIT_QUEUES) { + expect(groupOf(q).name).toBe("integration_sync"); } }); it("schedules the hourly poll cron + the daily oauth-state sweep", () => { - expect(source).toMatch(/\[FITBIT_SYNC_QUEUE,\s*FITBIT_SYNC_CRON\]/); - expect(source).toMatch( - /\[\s*FITBIT_OAUTH_STATE_CLEANUP_QUEUE,\s*FITBIT_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, - ); + expect(jobOf(FITBIT_SYNC_QUEUE).cron).toBe(FITBIT_SYNC_CRON); + expect(jobOf(FITBIT_SYNC_QUEUE).scheduleOptions).toBeUndefined(); + const sweep = jobOf(FITBIT_OAUTH_STATE_CLEANUP_QUEUE); + expect(sweep.cron).toBe(FITBIT_OAUTH_STATE_CLEANUP_CRON); + expect(sweep.scheduleOptions).toBe(cronIsTheRetry); }); - it("registers a createAndWork handler for every Fitbit queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,160}FITBIT_SYNC_QUEUE[\s\S]{0,160}handleFitbitSync/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}FITBIT_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}FITBIT_SLEEP_REPAIR_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,160}FITBIT_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,160}handleFitbitOAuthStateCleanup/, + it("binds a handler for every Fitbit queue", async () => { + expect(await loadedHandler(FITBIT_SYNC_QUEUE)).toBe(handleFitbitSync); + expect(await loadedHandler(FITBIT_OAUTH_STATE_CLEANUP_QUEUE)).toBe( + handleFitbitOAuthStateCleanup, ); + for (const [queue, kind] of [ + [FITBIT_BACKFILL_QUEUE, "fitbit-backfill"], + [FITBIT_SLEEP_REPAIR_QUEUE, "fitbit-sleep-repair"], + ] as const) { + const sends = await forwardOneJob(queue); + expect(sends).toHaveLength(1); + expect(sends[0]!.queue).toBe(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(sends[0]!.data).toMatchObject({ kind, data: { userId: "u1" } }); + } }); - it("wires the self-converging Fitbit backfill boot discovery", () => { - expect(source).toMatch( - /enqueueBootTimeFitbitBackfill\(\s*bootStaggerSecondsFor\("fitbit-backfill"\)/, - ); + it("wires the self-converging backfill and the one-shot sleep-repair boot discovery", () => { + const passes = discoveryOf("integration_sync"); + expect(passes).toContain("fitbit-backfill"); + expect(passes).toContain("fitbit-sleep-repair"); }); - it("wires the one-shot Fitbit sleep-repair boot discovery", () => { - expect(source).toMatch( - /enqueueBootTimeFitbitSleepRepair\(\s*bootStaggerSecondsFor\("fitbit-sleep-repair"\)/, - ); - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/fitbit-sleep-repair["']/); - }); - - it("imports the Fitbit sync driver + backfill exports", () => { - expect(source).toMatch(/from\s*["']@\/lib\/fitbit\/sync["']/); - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/fitbit-backfill["']/); + it("imports the Fitbit sync driver into the handler module", () => { + expect(handlerSource).toMatch(/from\s*["']@\/lib\/fitbit\/sync["']/); }); }); diff --git a/src/lib/jobs/__tests__/geo-backfill.test.ts b/src/lib/jobs/__tests__/geo-backfill.test.ts index 8a6b4a835..036401d48 100644 --- a/src/lib/jobs/__tests__/geo-backfill.test.ts +++ b/src/lib/jobs/__tests__/geo-backfill.test.ts @@ -283,13 +283,13 @@ describe("runGeoBackfill", () => { }); /** - * v1.4.37 — the helper is now scheduled via pg-boss at the hourly - * :40 slot. The schedule constants live in the helper module so the - * reminder-worker boot pulls them by import; this test pins both - * shapes so a regression that drops the cadence or drifts the queue - * name lands here, not in production. + * The helper is scheduled via pg-boss at the hourly :40 slot. The + * schedule constants live in the helper module so the worker registry + * pulls them by import; this test pins both shapes so a regression that + * drops the cadence or drifts the queue name lands here, not in + * production. */ -describe("geo-backfill scheduling contract (v1.4.37)", () => { +describe("geo-backfill scheduling contract", () => { it("exports the pg-boss queue name", () => { expect(GEO_BACKFILL_QUEUE).toBe("geo-backfill"); }); @@ -298,20 +298,11 @@ describe("geo-backfill scheduling contract (v1.4.37)", () => { expect(GEO_BACKFILL_CRON).toBe("40 * * * *"); }); - it("reminder-worker imports both the runner and the schedule constants", async () => { - // Source-text probe avoids importing the worker module (which - // would drag pg-boss + a live Postgres connection). The contract - // is binary: the worker either registers the queue + cron or it - // does not; both presence checks pin the v1.4.37 scheduling - // shape end-to-end. + it("the worker registry schedules the queue and binds the ops handler", async () => { + // The handler module's import is read as text; the queue facts come from + // the registry, so neither pg-boss nor a live Postgres connection boots. const fs = await import("node:fs/promises"); const path = await import("node:path"); - // v1.18.1 — the geo-backfill wiring moved out of the 2143-LOC - // reminder-worker boot file into the maintenance registrar. - const workerSrc = await fs.readFile( - path.resolve(__dirname, "..", "reminder", "register-maintenance.ts"), - "utf-8", - ); const handlerSrc = await fs.readFile( path.resolve(__dirname, "..", "reminder", "ops-handlers.ts"), "utf-8", @@ -319,7 +310,11 @@ describe("geo-backfill scheduling contract (v1.4.37)", () => { expect(handlerSrc).toMatch( /import \{[^}]*runGeoBackfill[^}]*\} from "@\/lib\/jobs\/geo-backfill";/, ); - expect(workerSrc).toContain("GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON"); - expect(workerSrc).toContain("handleGeoBackfill"); + const { groupOf, jobOf, loadedHandler } = + await import("../registry/__tests__/registry-helpers"); + const { handleGeoBackfill } = await import("../reminder/ops-handlers"); + expect(groupOf(GEO_BACKFILL_QUEUE).name).toBe("maintenance"); + expect(jobOf(GEO_BACKFILL_QUEUE).cron).toBe(GEO_BACKFILL_CRON); + expect(await loadedHandler(GEO_BACKFILL_QUEUE)).toBe(handleGeoBackfill); }); }); diff --git a/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts b/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts index 7b65fe798..52362b370 100644 --- a/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts +++ b/src/lib/jobs/__tests__/geolite2-fetch-queue.test.ts @@ -1,10 +1,10 @@ /** * Runtime GeoLite2 fetch (issue #659) — queue-wiring guard + boot gating. * - * The source-text-grep half mirrors the other dead-queue guards: assert the - * queue is imported, in `allQueues`, scheduled, and bound to a `createAndWork` - * handler that runs the fetch — without booting pg-boss. An unregistered queue - * silently never drains (the recurring v1.4.37 dead-queue bug), which here + * The first half reads the worker registry: the queue is declared in the + * maintenance group, scheduled monthly, and bound to the handler that runs + * the fetch, and the maintenance boot discovery carries the fetch pass. An + * unregistered queue silently never drains (the dead-queue class), which here * would mean a keyed self-host never gets its databases fetched. * * The second half exercises the boot-discovery gate directly: it must send a @@ -15,54 +15,46 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-maintenance.ts", +import { + GEOLITE2_FETCH_CRON, + GEOLITE2_FETCH_QUEUE, +} from "@/lib/jobs/geolite2-fetch"; +import { handleGeolite2Fetch } from "@/lib/jobs/reminder/ops-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "ops-handlers.ts"), + "utf8", ); -const workerSource = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync(join(__dirname, "..", "reminder", "ops-handlers.ts"), "utf8"); -describe("reminder-worker — geolite2-fetch wiring", () => { - it("imports the queue symbols from the geolite2-fetch module", () => { - expect(workerSource).toMatch(/from\s*["']@\/lib\/jobs\/geolite2-fetch["']/); - expect(workerSource).toMatch(/\bGEOLITE2_FETCH_QUEUE\b/); - expect(workerSource).toMatch(/\bGEOLITE2_FETCH_CRON\b/); - expect(workerSource).toMatch(/\benqueueGeolite2FetchBootDiscovery\b/); +describe("geolite2-fetch registration", () => { + it("declares the queue in the maintenance group", () => { + expect(groupOf(GEOLITE2_FETCH_QUEUE).name).toBe("maintenance"); }); - it("registers the queue in the allQueues loop", () => { - const allQueuesMatch = workerSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bGEOLITE2_FETCH_QUEUE\b/); + it("schedules the monthly cron with the default retries", () => { + const job = jobOf(GEOLITE2_FETCH_QUEUE); + expect(job.cron).toBe(GEOLITE2_FETCH_CRON); + expect(job.scheduleOptions).toBeUndefined(); }); - it("schedules the monthly cron", () => { - expect(workerSource).toMatch( - /\[GEOLITE2_FETCH_QUEUE,\s*GEOLITE2_FETCH_CRON\]/, - ); - }); - - it("registers a createAndWork handler for the queue", () => { - expect(workerSource).toMatch( - /createAndWork[\s\S]{0,200}GEOLITE2_FETCH_QUEUE[\s\S]{0,200}handleGeolite2Fetch/, - ); + it("binds the fetch handler", async () => { + expect(await loadedHandler(GEOLITE2_FETCH_QUEUE)).toBe(handleGeolite2Fetch); }); it("runs the fetch inside the handler", () => { - expect(workerSource).toMatch( + expect(handlerSource).toMatch( /handleGeolite2Fetch[\s\S]{0,500}fetchGeoLite2Databases/, ); }); - it("fires the boot discovery from enqueueMaintenanceBootDiscovery", () => { - expect(workerSource).toMatch( - /enqueueMaintenanceBootDiscovery[\s\S]*enqueueGeolite2FetchBootDiscovery/, - ); + it("fires the boot discovery from the maintenance group", () => { + expect(discoveryOf("maintenance")).toContain("geolite2-fetch"); }); }); diff --git a/src/lib/jobs/__tests__/google-health-queues.test.ts b/src/lib/jobs/__tests__/google-health-queues.test.ts index 783593da4..b0c76de06 100644 --- a/src/lib/jobs/__tests__/google-health-queues.test.ts +++ b/src/lib/jobs/__tests__/google-health-queues.test.ts @@ -1,12 +1,11 @@ /** - * v1.26.0 — pg-boss queue registration for the Google Health sync layer. + * pg-boss queue registration for the Google Health sync layer. * - * The reminder-worker imports heavy infrastructure (pg-boss, Prisma adapter, - * notification dispatchers, …) so we don't boot the whole worker here. Instead - * we read the source as text and assert the queue constants + cron schedules + - * handler wiring + allQueues + boot-enqueue are present — the fast guard that - * catches the v1.4.37 dead-queue class (a queue declared but never registered - * in `allQueues`, which silently never drains). + * Reads the worker registry instead of booting the worker: every Google + * Health queue is declared in the integration-sync group with its cron and + * handler, the backfill forwards to the shared admission lane, and the boot + * discovery offers it again. This is the fast guard for the dead-queue class + * (a queue declared but never registered, which silently never drains). * * Google Health is poll-only at launch (no webhook — Pub/Sub deferred), so a * single hourly `google-health-sync` queue drives the per-user @@ -18,75 +17,90 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-integration-sync.ts", -); -const source = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync( - join(__dirname, "..", "reminder", "google-health-sync.ts"), - "utf8", - ); +import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "@/lib/jobs/google-health-backfill-queue"; +import { INTEGRATION_BACKFILL_ADMISSION_QUEUE } from "@/lib/jobs/integration-backfill-admission"; +import { + GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON, + GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, + GOOGLE_HEALTH_SYNC_CRON, + GOOGLE_HEALTH_SYNC_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; +import { + handleGoogleHealthOAuthStateCleanup, + handleGoogleHealthSync, +} from "@/lib/jobs/reminder/google-health-sync"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -const GOOGLE_HEALTH_QUEUE_CONSTS = [ - "GOOGLE_HEALTH_SYNC_QUEUE", - "GOOGLE_HEALTH_BACKFILL_QUEUE", - "GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE", +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "google-health-sync.ts"), + "utf8", +); + +const GOOGLE_HEALTH_QUEUES = [ + GOOGLE_HEALTH_SYNC_QUEUE, + GOOGLE_HEALTH_BACKFILL_QUEUE, + GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, ] as const; -describe("reminder-worker — Google Health sync queues", () => { +describe("Google Health sync queues", () => { it("declares the Google Health sync + oauth-state-cleanup queue constants", () => { - expect(source).toMatch( - /GOOGLE_HEALTH_SYNC_QUEUE\s*=\s*["']google-health-sync["']/, - ); - expect(source).toMatch( - /GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE\s*=\s*["']google-health-oauth-state-cleanup["']/, + expect(GOOGLE_HEALTH_SYNC_QUEUE).toBe("google-health-sync"); + expect(GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE).toBe( + "google-health-oauth-state-cleanup", ); }); - it("registers EVERY Google Health queue in allQueues (v1.4.37 dead-queue guard)", () => { - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - const body = block![1]!; - for (const q of GOOGLE_HEALTH_QUEUE_CONSTS) { - expect(body).toContain(q); + it("declares EVERY Google Health queue in the integration-sync group", () => { + for (const q of GOOGLE_HEALTH_QUEUES) { + expect(groupOf(q).name).toBe("integration_sync"); } }); it("schedules the hourly poll cron + the daily oauth-state sweep", () => { - expect(source).toMatch( - /\[GOOGLE_HEALTH_SYNC_QUEUE,\s*GOOGLE_HEALTH_SYNC_CRON\]/, - ); - expect(source).toMatch( - /\[\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE,\s*GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, - ); + expect(jobOf(GOOGLE_HEALTH_SYNC_QUEUE).cron).toBe(GOOGLE_HEALTH_SYNC_CRON); + expect(jobOf(GOOGLE_HEALTH_SYNC_QUEUE).scheduleOptions).toBeUndefined(); + const sweep = jobOf(GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE); + expect(sweep.cron).toBe(GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON); + expect(sweep.scheduleOptions).toBe(cronIsTheRetry); }); - it("registers a createAndWork handler for every Google Health queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}GOOGLE_HEALTH_SYNC_QUEUE[\s\S]{0,200}handleGoogleHealthSync/, + it("binds a handler for every Google Health queue", async () => { + expect(await loadedHandler(GOOGLE_HEALTH_SYNC_QUEUE)).toBe( + handleGoogleHealthSync, ); - expect(source).toMatch( - /createAndWork[\s\S]{0,320}GOOGLE_HEALTH_BACKFILL_QUEUE[\s\S]{0,320}enqueueIntegrationBackfillAdmission/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE[\s\S]{0,200}handleGoogleHealthOAuthStateCleanup/, + expect(await loadedHandler(GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE)).toBe( + handleGoogleHealthOAuthStateCleanup, ); + const { boss, sends } = recordingBoss(); + const backfill = await jobOf(GOOGLE_HEALTH_BACKFILL_QUEUE).load({ boss }); + await backfill([ + { + id: "j1", + name: GOOGLE_HEALTH_BACKFILL_QUEUE, + data: { userId: "u1" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + expect(sends).toHaveLength(1); + expect(sends[0]!.queue).toBe(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(sends[0]!.data).toMatchObject({ kind: "google-health-backfill" }); }); it("wires the self-converging Google Health backfill boot discovery", () => { - expect(source).toMatch( - /enqueueBootTimeGoogleHealthBackfill\(\s*bootStaggerSecondsFor\("google-health-backfill"\)/, - ); + expect(discoveryOf("integration_sync")).toContain("google-health-backfill"); }); - it("imports the Google Health sync driver + backfill exports", () => { - expect(source).toMatch(/from\s*["']@\/lib\/google-health\/sync["']/); - expect(source).toMatch( - /from\s*["']@\/lib\/jobs\/google-health-backfill["']/, - ); + it("imports the Google Health sync driver into the handler module", () => { + expect(handlerSource).toMatch(/from\s*["']@\/lib\/google-health\/sync["']/); }); }); diff --git a/src/lib/jobs/__tests__/insight-pregenerate.test.ts b/src/lib/jobs/__tests__/insight-pregenerate.test.ts index fd2cda222..f413497c1 100644 --- a/src/lib/jobs/__tests__/insight-pregenerate.test.ts +++ b/src/lib/jobs/__tests__/insight-pregenerate.test.ts @@ -1,5 +1,5 @@ /** - * v1.7.0 — unit tests for the nightly insight pre-generation cron. + * Unit tests for the nightly insight pre-generation cron. * * Covers: * - the discovery query selects stale-cache users who kept AI analysis @@ -14,12 +14,10 @@ * - the warm pass is single-locale + refill-only (v1.16.8); * - the forced single-user warm is idempotent (freshness re-check), * backs off after a failure, and is capped per day (v1.16.8); - * - the queue is registered in `allQueues` AND scheduled in - * reminder-worker.ts (the v1.4.37 W10 unregistered-queue catch). + * - the worker registry declares the queue, schedules its cron and binds + * its handler (the unregistered-queue catch). */ import { describe, it, expect, vi, beforeEach } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; const checkRateLimit = vi.fn(); const getAssistantFlags = vi.fn(); @@ -128,8 +126,18 @@ import { PREGENERATE_STALE_MS, FORCE_WARM_DAILY_LIMIT, INSIGHT_PREGENERATE_QUEUE, - INSIGHT_PREGENERATE_CRON, } from "../insight-pregenerate"; +import { + INSIGHT_PREGENERATE_CRON, + INSIGHT_PREGENERATE_EXPIRE_SECONDS, +} from "../insight-pregenerate-shared"; +import { handleInsightPregenerateJob } from "../reminder/insights-handlers"; +import { insightRetryOptions } from "../reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; import { AI_BUDGETS } from "@/lib/ai/ai-budgets"; function makePrisma( @@ -1402,29 +1410,22 @@ describe("forceWarmUser — on-demand single-user warm (v1.8.7.1)", () => { }); describe("queue registration", () => { - // v1.18.1 — the insight-pregenerate wiring moved out of the 2143-LOC - // reminder-worker boot file into the status registrar. The dead-queue guard - // follows the wiring there. - const workerSrc = fs.readFileSync( - path.resolve(__dirname, "../reminder/register-status.ts"), - "utf8", - ); - - it("registers the queue name in the allQueues createQueue loop", () => { - const match = workerSrc.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(match).not.toBeNull(); - expect(match![1]).toMatch(/\bINSIGHT_PREGENERATE_QUEUE\b/); + it("declares the queue in the status group", () => { + expect(groupOf(INSIGHT_PREGENERATE_QUEUE).name).toBe("status"); }); - it("schedules the cron in the schedules table (with retry policy)", () => { - expect(workerSrc).toMatch( - /\[\s*INSIGHT_PREGENERATE_QUEUE\s*,\s*INSIGHT_PREGENERATE_CRON\s*,\s*\{\s*\.\.\.insightRetryOptions\s*,\s*expireInSeconds:\s*INSIGHT_PREGENERATE_EXPIRE_SECONDS\s*,?\s*\}\s*,?\s*\]/, - ); + it("schedules the cron with the retry policy", () => { + const job = jobOf(INSIGHT_PREGENERATE_QUEUE); + expect(job.cron).toBe(INSIGHT_PREGENERATE_CRON); + expect(job.scheduleOptions).toEqual({ + ...insightRetryOptions, + expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS, + }); }); - it("registers a createAndWork handler for the queue", () => { - expect(workerSrc).toMatch( - /createAndWork[\s\S]{0,80}INSIGHT_PREGENERATE_QUEUE/, + it("binds the pre-generation handler to the queue", async () => { + expect(await loadedHandler(INSIGHT_PREGENERATE_QUEUE)).toBe( + handleInsightPregenerateJob, ); }); diff --git a/src/lib/jobs/__tests__/insight-status-generate.test.ts b/src/lib/jobs/__tests__/insight-status-generate.test.ts index da40217a7..cfd66efa4 100644 --- a/src/lib/jobs/__tests__/insight-status-generate.test.ts +++ b/src/lib/jobs/__tests__/insight-status-generate.test.ts @@ -1,18 +1,16 @@ /** - * v1.8.3 — unit tests for the on-demand per-metric status generation queue. + * Unit tests for the on-demand per-metric status generation queue. * * Covers: * - the dispatch runs the matching generator with force:true; * - an unknown metric is annotated and skipped, not thrown; * - the enqueue helper no-ops without a global boss instance and de-dupes * with a singletonKey when one is present; - * - the queue is registered in `allQueues` AND wired to a `createAndWork` - * handler in reminder-worker.ts (the v1.4.37 W10 unregistered-queue - * catch — a queue built but never registered silently never drains). + * - the worker registry declares the queue and binds its handler (the + * unregistered-queue catch — a queue built but never registered silently + * never drains). */ import { describe, it, expect, vi, beforeEach } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; const getGlobalBoss = vi.fn(); vi.mock("@/lib/jobs/boss-instance", () => ({ @@ -47,6 +45,12 @@ vi.mock("@/lib/ai/capabilities/gate", () => ({ aiCapabilityForJob: (...a: unknown[]) => aiCapabilityForJob(...a), aiCapabilityForRecord: vi.fn(), })); +// The worker's insights handlers (loaded to check the queue's binding) also +// import the nightly status batch; stub it so its card table never reads the +// stubbed generators above. +vi.mock("@/lib/insights/status-batch", () => ({ + generateStatusBatchForUser: vi.fn(), +})); const annotate = vi.fn(); vi.mock("@/lib/logging/context", async (importOriginal) => ({ ...(await importOriginal()), @@ -59,6 +63,13 @@ import { INSIGHT_STATUS_GENERATE_QUEUE, INSIGHT_STATUS_METRICS, } from "../insight-status-generate"; +import { INSIGHT_STATUS_GENERATE_CONCURRENCY } from "../insight-status-generate-queue"; +import { handleInsightStatusGenerate } from "../reminder/insights-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; beforeEach(() => { vi.clearAllMocks(); @@ -259,23 +270,18 @@ describe("enqueueStatusGeneration", () => { }); describe("queue registration", () => { - // v1.18.1 — the insight-status-generate wiring moved out of the 2143-LOC - // reminder-worker boot file into the status registrar. The dead-queue guard - // follows the wiring there. - const workerSrc = fs.readFileSync( - path.resolve(__dirname, "../reminder/register-status.ts"), - "utf8", - ); - - it("registers the queue name in the allQueues createQueue loop", () => { - const match = workerSrc.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(match).not.toBeNull(); - expect(match![1]).toMatch(/\bINSIGHT_STATUS_GENERATE_QUEUE\b/); + it("declares the queue in the status group, low-concurrency and cron-free", () => { + expect(groupOf(INSIGHT_STATUS_GENERATE_QUEUE).name).toBe("status"); + const job = jobOf(INSIGHT_STATUS_GENERATE_QUEUE); + expect(job.work).toEqual({ + localConcurrency: INSIGHT_STATUS_GENERATE_CONCURRENCY, + }); + expect(job.cron).toBeUndefined(); }); - it("registers a createAndWork handler for the queue", () => { - expect(workerSrc).toMatch( - /createAndWork[\s\S]{0,120}INSIGHT_STATUS_GENERATE_QUEUE/, + it("binds the status handler to the queue", async () => { + expect(await loadedHandler(INSIGHT_STATUS_GENERATE_QUEUE)).toBe( + handleInsightStatusGenerate, ); }); diff --git a/src/lib/jobs/__tests__/intake-auto-skip.test.ts b/src/lib/jobs/__tests__/intake-auto-skip.test.ts index 4073e96d1..05ad666a0 100644 --- a/src/lib/jobs/__tests__/intake-auto-skip.test.ts +++ b/src/lib/jobs/__tests__/intake-auto-skip.test.ts @@ -27,12 +27,14 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { caches, __resetAllCachesForTests } from "@/lib/cache/server-cache"; import { - INTAKE_AUTO_SKIP_CRON, INTAKE_AUTO_SKIP_GRACE_HOURS, - INTAKE_AUTO_SKIP_QUEUE, medicationAutoMissDelayMs, runIntakeAutoSkipPass, } from "../intake-auto-skip"; +import { + INTAKE_AUTO_SKIP_CRON, + INTAKE_AUTO_SKIP_QUEUE, +} from "../intake-auto-skip-queue"; interface FakeIntakeRow { id: string; diff --git a/src/lib/jobs/__tests__/intake-auto-take.test.ts b/src/lib/jobs/__tests__/intake-auto-take.test.ts index db93d7c97..bee2b010c 100644 --- a/src/lib/jobs/__tests__/intake-auto-take.test.ts +++ b/src/lib/jobs/__tests__/intake-auto-take.test.ts @@ -39,15 +39,17 @@ import { consumeForIntake } from "@/lib/medications/inventory/consumption"; import { refreshDoseSlot } from "@/lib/medications/dose-day-refresh"; import { queueMedicationIntakeSync } from "@/lib/notifications/medication-intake-sync"; import { - INTAKE_AUTO_TAKE_CRON, INTAKE_AUTO_TAKE_LOOKBACK_MS, - INTAKE_AUTO_TAKE_QUEUE, autoTakeFloor, decideSlot, dueAutoTakeSlots, isInPauseEra, runIntakeAutoTakePass, } from "../intake-auto-take"; +import { + INTAKE_AUTO_TAKE_CRON, + INTAKE_AUTO_TAKE_QUEUE, +} from "../intake-auto-take-queue"; import { autoTakeOwnedFrom } from "../intake-auto-skip"; import { autoTakeRollingAnchor } from "@/lib/medications/auto-take"; import { computeDisplayDue } from "@/lib/medications/scheduling/next-due"; diff --git a/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts b/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts index a99672474..3f5f2dc6b 100644 --- a/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts +++ b/src/lib/jobs/__tests__/intake-slot-dedup-queue.test.ts @@ -1,95 +1,102 @@ /** - * v1.8.2 — duplicate dose-slot dedup queue registration guard. + * Duplicate dose-slot dedup queue registration guard. * - * Same source-text-grep approach as the drain + step-consolidation + - * mean-consolidation regression guards: assert the queue is registered - * in `allQueues`, a `createAndWork` handler is wired against it, and the - * boot-discovery enqueue helper is called — without booting pg-boss + - * Prisma. An unregistered queue silently never drains (the past incident - * this guard exists to prevent). + * Reads the worker registry: the queue is declared in the maintenance group + * with its daily cron, bound to the dedup handler, and offered again by the + * maintenance boot discovery. The bound handler runs against a mocked dedup + * module, so pg-boss and Prisma never boot. An unregistered queue silently + * never drains (the past incident this guard exists to prevent). */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; -import { describe, expect, it } from "vitest"; - -// v1.18.1 — the intake-slot-dedup wiring + boot discovery moved out of the -// 2143-LOC reminder-worker boot file into the maintenance registrar. The -// dead-queue guard follows the wiring there. -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-maintenance.ts", +const { dedupeUserIntakeSlots, enqueueBootTimeIntakeSlotDedup } = vi.hoisted( + () => ({ + dedupeUserIntakeSlots: vi.fn(async () => ({ + slotsCollapsed: 0, + rowsSoftDeleted: 0, + rowsNormalised: 0, + daysRecomputed: 0, + })), + enqueueBootTimeIntakeSlotDedup: vi.fn(async () => ({ + enqueued: 0, + skipped: 0, + error: null, + })), + }), ); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +vi.mock("@/lib/medications/intake-slot-dedup", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/medications/intake-slot-dedup") + >()), + dedupeUserIntakeSlots, + enqueueBootTimeIntakeSlotDedup, +})); -describe("reminder-worker — intake-slot-dedup wiring", () => { - it("imports the queue symbols from the intake-slot-dedup module", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/medications\/intake-slot-dedup["']/, +import type { Job } from "pg-boss"; + +import { handleIntakeSlotDedup } from "@/lib/jobs/reminder/maintenance-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { + INTAKE_SLOT_DEDUP_CONCURRENCY, + INTAKE_SLOT_DEDUP_QUEUE, +} from "@/lib/medications/intake-slot-dedup"; + +function job(data: { userId?: string }): Job<{ userId?: string }> { + return { + id: "j1", + name: INTAKE_SLOT_DEDUP_QUEUE, + data, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("intake-slot-dedup registration", () => { + it("declares the intake-slot-dedup queue in the maintenance group", () => { + expect(groupOf(INTAKE_SLOT_DEDUP_QUEUE).name).toBe("maintenance"); + const definition = jobOf(INTAKE_SLOT_DEDUP_QUEUE); + expect(definition.policy?.policy).toBe("exclusive"); + expect(definition.work).toEqual({ + localConcurrency: INTAKE_SLOT_DEDUP_CONCURRENCY, + }); + }); + + it("binds the dedup handler, which runs the per-user dedup", async () => { + expect(await loadedHandler(INTAKE_SLOT_DEDUP_QUEUE)).toBe( + handleIntakeSlotDedup, ); - expect(source).toMatch(/\bINTAKE_SLOT_DEDUP_QUEUE\b/); - expect(source).toMatch(/\bdedupeUserIntakeSlots\b/); - expect(source).toMatch(/\benqueueBootTimeIntakeSlotDedup\b/); + await handleIntakeSlotDedup([job({ userId: "u1" })]); + expect(dedupeUserIntakeSlots).toHaveBeenCalledWith("u1"); + expect(enqueueBootTimeIntakeSlotDedup).not.toHaveBeenCalled(); }); - it("registers the intake-slot-dedup queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bINTAKE_SLOT_DEDUP_QUEUE\b/); + it("fires the boot-discovery pass", () => { + expect(discoveryOf("maintenance")).toContain("intake-slot-dedup"); }); - it("registers a createAndWork handler against the intake-slot-dedup queue", () => { - // v1.15.19 — span widened from {0,400}: the handler gained the daily - // cron-tick dispatch branch (no `userId` → discovery fan-out) between - // the queue name and the per-user dedup call. - expect(source).toMatch( - /createAndWork[\s\S]{0,200}INTAKE_SLOT_DEDUP_QUEUE[\s\S]{0,1400}dedupeUserIntakeSlots/, - ); - }); - - it("fires the boot-discovery enqueue helper", () => { - expect(source).toMatch(/await enqueueBootTimeIntakeSlotDedup\(\)/); - }); - - // v1.15.19 — the boot-only pass left duplicates created between deploys - // standing until the next worker restart. The queue now also carries a - // daily cron tick whose empty payload (no `userId`) dispatches the same - // discovery fan-out. + // The boot-only pass left duplicates created between deploys standing + // until the next worker restart. The queue also carries a daily cron tick + // whose empty payload (no `userId`) dispatches the same discovery fan-out. it("declares a daily cron inside the 03:xx maintenance window", () => { - const cronMatch = source.match( - /const INTAKE_SLOT_DEDUP_CRON\s*=\s*["']([^"']+)["']/, - ); - expect(cronMatch).not.toBeNull(); // Five-field cron, daily (day-of-month/month/day-of-week all `*`), // inside the 03:xx maintenance window. - expect(cronMatch![1]).toMatch(/^\d{1,2} 3 \* \* \*$/); + expect(jobOf(INTAKE_SLOT_DEDUP_QUEUE).cron).toMatch(/^\d{1,2} 3 \* \* \*$/); }); - it("registers the cron in the schedules table", () => { - // v1.15.20 — the schedules tuple grew an optional send-options slot - // (retry policy for the insight queues); match on the assignment, not - // the exact tuple type. - // v1.18.1 — `schedules` is now a module-level const in the registrar, so it - // closes at column 0 (`\n];`) rather than the monolith's function-body - // two-space indent. - const schedulesMatch = source.match( - /const schedules:[^=]*=\s*\[([\s\S]*?)\n\];/, - ); - expect(schedulesMatch).not.toBeNull(); - expect(schedulesMatch![1]).toMatch( - /\[INTAKE_SLOT_DEDUP_QUEUE,\s*INTAKE_SLOT_DEDUP_CRON\]/, - ); - }); - - it("dispatches a payload without userId to the discovery fan-out", () => { - // The handler must branch on the missing `userId` BEFORE the per-user - // dedup call, and the branch must run the discovery helper. - expect(source).toMatch( - /INTAKE_SLOT_DEDUP_QUEUE[\s\S]{0,800}if \(!userId\) \{[\s\S]{0,400}enqueueBootTimeIntakeSlotDedup\(\)/, - ); + it("dispatches a payload without userId to the discovery fan-out", async () => { + await handleIntakeSlotDedup([job({})]); + expect(enqueueBootTimeIntakeSlotDedup).toHaveBeenCalledTimes(1); + expect(dedupeUserIntakeSlots).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/jobs/__tests__/integration-backfill-admission.test.ts b/src/lib/jobs/__tests__/integration-backfill-admission.test.ts index a26bd6561..69ac042a1 100644 --- a/src/lib/jobs/__tests__/integration-backfill-admission.test.ts +++ b/src/lib/jobs/__tests__/integration-backfill-admission.test.ts @@ -1,7 +1,74 @@ +/** + * The shared full-history backfill admission: every provider backfill and + * repair forwards into one database-coordinated drain. The boot discovery, + * the drain's binding and the source queues' forwarding are read off the + * worker registry; the boot discovery runs against stubbed enqueue helpers + * that record the stagger each pass hands them. + */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { SendOptions } from "pg-boss"; + +const staggers = vi.hoisted(() => new Map()); +const recordStagger = vi.hoisted(() => (kind: string, stagger: number) => { + staggers.set(kind, stagger); + return { enqueued: 0, skipped: 0, error: null }; +}); +vi.mock("@/lib/jobs/whoop-backfill", async (importOriginal) => ({ + ...(await importOriginal()), + enqueueBootTimeWhoopBackfill: async (stagger: number) => + recordStagger("whoop-backfill", stagger), +})); +vi.mock("@/lib/jobs/fitbit-backfill", async (importOriginal) => ({ + ...(await importOriginal()), + enqueueBootTimeFitbitBackfill: async (stagger: number) => + recordStagger("fitbit-backfill", stagger), +})); +vi.mock("@/lib/jobs/google-health-backfill", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/google-health-backfill") + >()), + enqueueBootTimeGoogleHealthBackfill: async (stagger: number) => + recordStagger("google-health-backfill", stagger), +})); +vi.mock("@/lib/jobs/google-health-sleep-repair", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/google-health-sleep-repair") + >()), + enqueueBootTimeGoogleHealthSleepRepair: async (stagger: number) => + recordStagger("google-health-sleep-repair", stagger), +})); +vi.mock("@/lib/jobs/sleep-timeline-backfill", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/sleep-timeline-backfill") + >()), + enqueueBootTimeSleepTimelineBackfill: async (stagger: number) => + recordStagger("sleep-timeline-backfill", stagger), +})); +vi.mock("@/lib/jobs/lab-biomarker-backfill", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/lab-biomarker-backfill") + >()), + enqueueBootTimeLabBiomarkerBackfill: async (stagger: number) => + recordStagger("lab-biomarker-backfill", stagger), +})); +vi.mock("@/lib/jobs/strava-backfill", async (importOriginal) => ({ + ...(await importOriginal()), + enqueueBootTimeStravaBackfill: async (stagger: number) => + recordStagger("strava-backfill", stagger), +})); +vi.mock("@/lib/jobs/fitbit-sleep-repair", async (importOriginal) => ({ + ...(await importOriginal()), + enqueueBootTimeFitbitSleepRepair: async (stagger: number) => + recordStagger("fitbit-sleep-repair", stagger), +})); +vi.mock("@/lib/jobs/oura-backfill", async (importOriginal) => ({ + ...(await importOriginal()), + enqueueBootTimeOuraBackfill: async (stagger: number) => + recordStagger("oura-backfill", stagger), +})); + import { INTEGRATION_BACKFILL_ADMISSION_GROUP, INTEGRATION_BACKFILL_ADMISSION_QUEUE, @@ -12,12 +79,29 @@ import { bootStaggerSecondsFor, enqueueIntegrationBackfillAdmission, integrationBackfillSourceOptions, + type IntegrationBackfillKind, } from "../integration-backfill-admission"; - -const registrarSource = readFileSync( - join(__dirname, "../reminder/register-integration-sync.ts"), - "utf8", -); +import { + FITBIT_BACKFILL_QUEUE, + FITBIT_SLEEP_REPAIR_QUEUE, + FITBIT_SYNC_QUEUE, +} from "@/lib/fitbit/jobs/queues"; +import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "../google-health-backfill-queue"; +import { GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE } from "../google-health-sleep-repair-queue"; +import { + GOOGLE_HEALTH_SYNC_QUEUE, + STRAVA_SYNC_QUEUE, +} from "../integration-poll-queues"; +import { LAB_BIOMARKER_BACKFILL_QUEUE } from "../lab-biomarker-backfill"; +import { OURA_BACKFILL_QUEUE } from "../oura-backfill-queue"; +import { + groupNamed, + jobOf, + recordingBoss, +} from "../registry/__tests__/registry-helpers"; +import { SLEEP_TIMELINE_BACKFILL_QUEUE } from "../sleep-timeline-backfill-queue"; +import { STRAVA_BACKFILL_QUEUE } from "../strava-backfill-queue"; +import { WHOOP_BACKFILL_QUEUE } from "../whoop-backfill-queue"; const providerSources = [ "whoop-backfill.ts", @@ -32,7 +116,7 @@ const providerSources = [ ].map((file) => readFileSync(join(__dirname, "..", file), "utf8")); describe("shared full-history backfill admission", () => { - it("assigns every integration backfill a deterministic durable boot stagger", () => { + it("assigns every integration backfill a deterministic durable boot stagger", async () => { expect(INTEGRATION_BACKFILL_BOOT_ORDER).toEqual([ "whoop-backfill", "fitbit-backfill", @@ -54,23 +138,14 @@ describe("shared full-history backfill admission", () => { expect(source).toMatch(/startAfterSeconds:\s*number\s*=\s*0/); } - const discoveryCalls = { - "whoop-backfill": "enqueueBootTimeWhoopBackfill", - "fitbit-backfill": "enqueueBootTimeFitbitBackfill", - "google-health-backfill": "enqueueBootTimeGoogleHealthBackfill", - "google-health-sleep-repair": "enqueueBootTimeGoogleHealthSleepRepair", - "sleep-timeline-backfill": "enqueueBootTimeSleepTimelineBackfill", - "lab-biomarker-backfill": "enqueueBootTimeLabBiomarkerBackfill", - "strava-backfill": "enqueueBootTimeStravaBackfill", - "fitbit-sleep-repair": "enqueueBootTimeFitbitSleepRepair", - "oura-backfill": "enqueueBootTimeOuraBackfill", - } as const; + // Every kind has a boot discovery pass, and each pass hands its helper + // the kind's own stagger. + const passes = groupNamed("integration_sync").discovery ?? []; for (const kind of INTEGRATION_BACKFILL_BOOT_ORDER) { - expect(registrarSource).toMatch( - new RegExp( - `${discoveryCalls[kind]}\\(\\s*bootStaggerSecondsFor\\("${kind}"\\)`, - ), - ); + const pass = passes.find((p) => p.name === kind); + expect(pass, kind).toBeDefined(); + await pass!.run(); + expect(staggers.get(kind), kind).toBe(bootStaggerSecondsFor(kind)); } }); @@ -139,40 +214,51 @@ describe("shared full-history backfill admission", () => { ).not.toHaveProperty("startAfter"); }); - it("registers one shared globally bounded drain and leaves incremental workers outside it", () => { - expect(registrarSource).toMatch( - /INTEGRATION_BACKFILL_ADMISSION_QUEUE[\s\S]{0,220}localConcurrency:\s*INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY[\s\S]{0,120}groupConcurrency:\s*INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY/, - ); - expect(registrarSource).toMatch( - /\[INTEGRATION_BACKFILL_ADMISSION_QUEUE\]:\s*\{[\s\S]{0,100}policy:\s*"exclusive"/, - ); + it("registers one shared globally bounded drain and leaves incremental workers outside it", async () => { + const drain = jobOf(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(drain.work).toEqual({ + localConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, + groupConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, + }); + expect(drain.policy?.policy).toBe("exclusive"); - for (const queue of [ - "WHOOP_BACKFILL_QUEUE", - "FITBIT_BACKFILL_QUEUE", - "GOOGLE_HEALTH_BACKFILL_QUEUE", - "GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE", - "SLEEP_TIMELINE_BACKFILL_QUEUE", - "LAB_BIOMARKER_BACKFILL_QUEUE", - "STRAVA_BACKFILL_QUEUE", - "FITBIT_SLEEP_REPAIR_QUEUE", - "OURA_BACKFILL_QUEUE", - ]) { - expect(registrarSource).toMatch( - new RegExp( - `${queue}[\\s\\S]{0,500}enqueueIntegrationBackfillAdmission`, - ), - ); + const sources: ReadonlyArray<[string, IntegrationBackfillKind]> = [ + [WHOOP_BACKFILL_QUEUE, "whoop-backfill"], + [FITBIT_BACKFILL_QUEUE, "fitbit-backfill"], + [GOOGLE_HEALTH_BACKFILL_QUEUE, "google-health-backfill"], + [GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, "google-health-sleep-repair"], + [SLEEP_TIMELINE_BACKFILL_QUEUE, "sleep-timeline-backfill"], + [LAB_BIOMARKER_BACKFILL_QUEUE, "lab-biomarker-backfill"], + [STRAVA_BACKFILL_QUEUE, "strava-backfill"], + [FITBIT_SLEEP_REPAIR_QUEUE, "fitbit-sleep-repair"], + [OURA_BACKFILL_QUEUE, "oura-backfill"], + ]; + for (const [queue, kind] of sources) { + const { boss, sends } = recordingBoss(); + const handler = await jobOf(queue).load({ boss }); + await handler([ + { + id: "j1", + name: queue, + data: { userId: "u1", provider: "WHOOP" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + expect(sends, queue).toHaveLength(1); + expect(sends[0]!.queue).toBe(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(sends[0]!.data).toMatchObject({ kind }); } for (const queue of [ - "FITBIT_SYNC_QUEUE", - "GOOGLE_HEALTH_SYNC_QUEUE", - "STRAVA_SYNC_QUEUE", + FITBIT_SYNC_QUEUE, + GOOGLE_HEALTH_SYNC_QUEUE, + STRAVA_SYNC_QUEUE, ]) { - expect(registrarSource).toMatch( - new RegExp(`${queue}[\\s\\S]{0,100}localConcurrency:\\s*1`), - ); + expect(jobOf(queue).work, queue).toEqual({ localConcurrency: 1 }); + expect(jobOf(queue).lane, queue).toBeUndefined(); } }); }); diff --git a/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts b/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts index 7f42958a7..85babde37 100644 --- a/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts +++ b/src/lib/jobs/__tests__/mcp-token-cleanup-queue.test.ts @@ -1,57 +1,49 @@ /** - * MCP Phase 3 (M2) — mcp-token-cleanup queue registration guard. + * Mcp-token-cleanup queue registration guard. * - * Source-text-grep guard like the other queue-wiring checks: assert the queue - * is in `allQueues`, scheduled, and wired to a `createAndWork` handler. An - * unregistered queue silently never drains (the recurring v1.4.37 dead-queue - * bug), which would leave expired MCP connector access tokens growing unbounded. + * Reads the worker registry: the queue is declared in the maintenance group, + * scheduled with the cron as its retry, and bound to the cleanup handler. An + * unregistered queue silently never drains (the dead-queue class), which + * would leave expired MCP connector access tokens growing unbounded. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-maintenance.ts", +import { handleMcpTokenCleanup } from "@/lib/jobs/reminder/cleanup-handlers"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; + +const QUEUE = "mcp-token-cleanup"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "cleanup-handlers.ts"), + "utf8", ); -const workerSource = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync( - join(__dirname, "..", "reminder", "cleanup-handlers.ts"), - "utf8", - ); -describe("reminder-worker — mcp-token-cleanup wiring", () => { - it("imports the cleanup handler from the cleanup-handlers module", () => { - expect(workerSource).toMatch(/\bhandleMcpTokenCleanup\b/); - expect(workerSource).toMatch(/\bMcpTokenCleanupPayload\b/); +describe("mcp-token-cleanup registration", () => { + it("declares the queue in the maintenance group", () => { + expect(groupOf(QUEUE).name).toBe("maintenance"); }); - it("registers the mcp-token-cleanup queue in the allQueues loop", () => { - const allQueuesMatch = workerSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bMCP_TOKEN_CLEANUP_QUEUE\b/); + it("schedules the daily cron with the cron as the retry", () => { + const job = jobOf(QUEUE); + expect(job.cron).toBe("0 4 * * *"); + expect(job.scheduleOptions).toBe(cronIsTheRetry); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("schedules the mcp-token-cleanup cron", () => { - expect(workerSource).toMatch( - /\[MCP_TOKEN_CLEANUP_QUEUE,\s*MCP_TOKEN_CLEANUP_CRON,\s*cronIsTheRetry\]/, - ); - }); - - it("registers a createAndWork handler for the queue", () => { - expect(workerSource).toMatch( - /createAndWork[\s\S]{0,200}MCP_TOKEN_CLEANUP_QUEUE[\s\S]{0,200}handleMcpTokenCleanup/, - ); + it("binds the cleanup handler", async () => { + expect(await loadedHandler(QUEUE)).toBe(handleMcpTokenCleanup); }); it("prunes MCP tokens inside the handler", () => { - expect(workerSource).toMatch( + expect(handlerSource).toMatch( /handleMcpTokenCleanup[\s\S]{0,400}cleanupExpiredMcpTokens/, ); }); diff --git a/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts b/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts index 4dbfd6d11..a2076591a 100644 --- a/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts +++ b/src/lib/jobs/__tests__/mean-consolidation-queue.test.ts @@ -1,48 +1,44 @@ /** - * v1.7.0 — daily-mean consolidation queue registration guard. + * Daily-mean consolidation queue registration guard. * - * Same source-text-grep approach as the drain + step-consolidation - * regression guards: assert the queue is registered in `allQueues`, a - * `createAndWork` handler is wired against it, and the boot-discovery - * enqueue helper is called — without booting pg-boss + Prisma. An - * unregistered queue silently never drains. + * Reads the worker registry: the queue is declared in the rollup group, + * bound to the consolidation handler, and offered again by the rollup boot + * discovery. An unregistered queue silently never drains. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -// v1.18.1 — the mean-consolidation wiring + boot discovery moved out of the -// 2143-LOC reminder-worker boot file into the rollup registrar. The dead-queue -// guard follows the wiring there. -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-rollup.ts"); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { + MEAN_CONSOLIDATION_CONCURRENCY, + MEAN_CONSOLIDATION_QUEUE, +} from "@/lib/jobs/mean-consolidation"; +import { handleMeanConsolidation } from "@/lib/jobs/reminder/rollup-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -describe("reminder-worker — mean-consolidation wiring", () => { - it("imports the queue symbols from the mean-consolidation module", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/mean-consolidation["']/); - expect(source).toMatch(/\bMEAN_CONSOLIDATION_QUEUE\b/); - expect(source).toMatch(/\brunMeanConsolidationForUser\b/); - expect(source).toMatch(/\benqueueBootTimeMeanConsolidation\b/); +describe("mean-consolidation registration", () => { + it("declares the mean-consolidation queue in the rollup group", () => { + expect(groupOf(MEAN_CONSOLIDATION_QUEUE).name).toBe("rollup"); }); - it("registers the mean-consolidation queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bMEAN_CONSOLIDATION_QUEUE\b/); + it("is a per-user exclusive pass at the module's concurrency", () => { + const job = jobOf(MEAN_CONSOLIDATION_QUEUE); + expect(job.policy?.policy).toBe("exclusive"); + expect(job.work).toEqual({ + localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY, + }); }); - it("registers a createAndWork handler against the mean-consolidation queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}MEAN_CONSOLIDATION_QUEUE[\s\S]{0,400}runMeanConsolidationForUser/, + it("binds the consolidation handler", async () => { + expect(await loadedHandler(MEAN_CONSOLIDATION_QUEUE)).toBe( + handleMeanConsolidation, ); }); - it("fires the boot-discovery enqueue helper", () => { - // The boot wiring passes a stagger offset so the heavy backfills don't all - // drain in parallel onto one tenant; allow an optional argument here. - expect(source).toMatch(/await enqueueBootTimeMeanConsolidation\([^)]*\)/); + it("runs the boot discovery pass", () => { + expect(discoveryOf("rollup")).toContain("mean-consolidation"); }); }); diff --git a/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts b/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts index 564b9cb95..98f87af95 100644 --- a/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts +++ b/src/lib/jobs/__tests__/measurement-reminder-queue.test.ts @@ -1,90 +1,81 @@ /** - * v1.17.1 — measurement-reminder queue registration guard. + * Measurement-reminder queue registration guard. * - * Same source-text-grep approach as the cycle / mood / drain registration - * guards: assert the Vorsorge-reminder queue is declared, listed in - * `allQueues` (the v1.4.37 dead-queue lesson — an unregistered queue is - * never provisioned and the schedule silently no-ops), scheduled on its - * cron, and wired to a `createAndWork` handler that delegates to - * `runMeasurementReminderTick`. No pg-boss / Prisma boot. + * Reads the worker registry: the Vorsorge-reminder queue is declared in the + * reminders group (an unregistered queue is never provisioned and the + * schedule silently no-ops), scheduled on its cron, and bound to the handler + * that delegates to `runMeasurementReminderTick`; the eventful + * reminder-satisfy queue sits beside it. No pg-boss / Prisma boot. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the reminder-dispatch wiring (measurement reminder + -// reminder-satisfy) moved out of the 2143-LOC reminder-worker boot file into -// the reminders registrar. The dead-queue guard follows the wiring there (the -// handler module is still concatenated for the handler-symbol assertions). -const source = - readFileSync( - join(__dirname, "..", "reminder", "register-reminders.ts"), - "utf8", - ) + - readFileSync( - join(__dirname, "..", "reminder", "mood-cycle-checks.ts"), - "utf8", - ); +import { + REMINDER_SATISFY_CONCURRENCY, + REMINDER_SATISFY_QUEUE, +} from "@/lib/jobs/reminder-satisfy"; +import { + handleMeasurementReminderCheck, + handleReminderSatisfy, +} from "@/lib/jobs/reminder/mood-cycle-checks"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -describe("reminder-worker — measurement-reminder wiring", () => { +const MEASUREMENT_REMINDER_QUEUE = "measurement-reminder-check"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "mood-cycle-checks.ts"), + "utf8", +); + +describe("measurement-reminder registration", () => { it("imports the measurement-reminder tick runner", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/measurement-reminder["']/); - expect(source).toMatch(/\brunMeasurementReminderTick\b/); + expect(handlerSource).toMatch( + /from\s*["']@\/lib\/jobs\/measurement-reminder["']/, + ); + expect(handlerSource).toMatch(/\brunMeasurementReminderTick\b/); }); - it("declares the queue + cron constants", () => { - expect(source).toMatch( - /MEASUREMENT_REMINDER_QUEUE\s*=\s*["']measurement-reminder-check["']/, - ); - expect(source).toMatch( - /MEASUREMENT_REMINDER_CRON\s*=\s*["']\*\/15 \* \* \* \*["']/, - ); + it("declares the queue in the reminders group", () => { + expect(groupOf(MEASUREMENT_REMINDER_QUEUE).name).toBe("reminders"); }); - it("registers the queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bMEASUREMENT_REMINDER_QUEUE\b/); + it("schedules the measurement-reminder cron, single-flight", () => { + const job = jobOf(MEASUREMENT_REMINDER_QUEUE); + expect(job.cron).toBe("*/15 * * * *"); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("schedules the measurement-reminder cron", () => { - expect(source).toMatch( - /\[MEASUREMENT_REMINDER_QUEUE,\s*MEASUREMENT_REMINDER_CRON\]/, + it("binds the handler that runs the tick", async () => { + expect(await loadedHandler(MEASUREMENT_REMINDER_QUEUE)).toBe( + handleMeasurementReminderCheck, ); - }); - - it("registers a createAndWork handler that runs the tick", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}MEASUREMENT_REMINDER_QUEUE[\s\S]{0,200}handleMeasurementReminderCheck/, - ); - expect(source).toMatch( + expect(handlerSource).toMatch( /handleMeasurementReminderCheck[\s\S]{0,400}runMeasurementReminderTick/, ); }); }); -describe("reminder-worker — eventful reminder-satisfy wiring (v1.18.1)", () => { - it("imports the satisfy queue constants", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/reminder-satisfy["']/); - expect(source).toMatch(/\bREMINDER_SATISFY_QUEUE\b/); +describe("eventful reminder-satisfy registration", () => { + it("declares the satisfy queue in the reminders group, with no cron", () => { + expect(groupOf(REMINDER_SATISFY_QUEUE).name).toBe("reminders"); + const job = jobOf(REMINDER_SATISFY_QUEUE); + expect(job.cron).toBeUndefined(); + expect(job.work).toEqual({ + localConcurrency: REMINDER_SATISFY_CONCURRENCY, + }); }); - it("registers the satisfy queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, + it("binds the satisfy handler", async () => { + expect(await loadedHandler(REMINDER_SATISFY_QUEUE)).toBe( + handleReminderSatisfy, ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bREMINDER_SATISFY_QUEUE\b/); - }); - - it("binds a createAndWork handler for the satisfy queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}REMINDER_SATISFY_QUEUE[\s\S]{0,200}handleReminderSatisfy/, - ); - expect(source).toMatch( + expect(handlerSource).toMatch( /handleReminderSatisfy[\s\S]{0,800}runReminderSatisfyForUser/, ); }); diff --git a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts index f27b28739..9defa2f68 100644 --- a/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts +++ b/src/lib/jobs/__tests__/measurement-tombstone-cleanup.test.ts @@ -1,23 +1,26 @@ /** - * v1.7.0 — measurement-tombstone retention cleanup. + * Measurement-tombstone retention cleanup. * * Two guards: * 1. Behavioural: the helper hard-deletes only soft-deleted rows whose * `deletedAt` predates the retention horizon, leaving live rows and * recently-tombstoned rows untouched. - * 2. Source-grep wiring (same approach as the drain-cumulative guard): - * the queue is registered in `allQueues`, scheduled, and bound to a - * `createAndWork` handler — a missing `allQueues` entry silently no-ops - * the schedule under pg-boss v12. + * 2. Registry wiring: the queue is declared in the maintenance group, + * scheduled, and bound to its handler — a queue the registry does not + * declare is never created, and its schedule silently no-ops under + * pg-boss v12. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it, vi } from "vitest"; import { cleanupExpiredMeasurementTombstones } from "../measurement-tombstone-cleanup"; import { PURGE_BATCH_SIZE, PURGE_MAX_BATCHES } from "../purge-batch"; import { TOMBSTONE_RETENTION_DAYS } from "@/lib/auth/native-client"; +import { handleMeasurementTombstoneCleanup } from "../reminder/cleanup-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; const DAY_MS = 86_400_000; @@ -170,42 +173,24 @@ describe("cleanupExpiredMeasurementTombstones — accounts under restore", () => }); }); -describe("reminder-worker — measurement-tombstone-cleanup schedule", () => { - // v1.18.1 — the cleanup wiring moved out of the 2143-LOC reminder-worker - // boot file into the maintenance registrar. The dead-queue guard follows it. - const source = readFileSync( - join(__dirname, "..", "reminder", "register-maintenance.ts"), - "utf8", - ); +describe("measurement-tombstone-cleanup schedule", () => { + const QUEUE = "measurement-tombstone-cleanup"; it("declares the queue at the documented Berlin cadence", () => { - expect(source).toMatch( - /MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE\s*=\s*["']measurement-tombstone-cleanup["']/, - ); - expect(source).toMatch( - /MEASUREMENT_TOMBSTONE_CLEANUP_CRON\s*=\s*["']40 3 \* \* \*["']/, - ); + expect(jobOf(QUEUE).cron).toBe("40 3 * * *"); }); - it("registers the queue in the allQueues createQueue loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch( - /\bMEASUREMENT_TOMBSTONE_CLEANUP_QUEUE\b/, - ); + it("declares the queue in the maintenance group", () => { + expect(groupOf(QUEUE).name).toBe("maintenance"); }); - it("schedules the cron via boss.schedule (allQueues + schedules)", () => { - expect(source).toMatch( - /\[MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE,\s*MEASUREMENT_TOMBSTONE_CLEANUP_CRON\]/, - ); + it("keeps the default retries on the schedule", () => { + const job = jobOf(QUEUE); + expect(job.scheduleOptions).toBeUndefined(); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("binds a createAndWork handler to the queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE[\s\S]{0,200}handleMeasurementTombstoneCleanup/, - ); + it("binds the cleanup handler to the queue", async () => { + expect(await loadedHandler(QUEUE)).toBe(handleMeasurementTombstoneCleanup); }); }); diff --git a/src/lib/jobs/__tests__/mood-prognosis-refresh.test.ts b/src/lib/jobs/__tests__/mood-prognosis-refresh.test.ts index b3c77861d..8f6e7bd6a 100644 --- a/src/lib/jobs/__tests__/mood-prognosis-refresh.test.ts +++ b/src/lib/jobs/__tests__/mood-prognosis-refresh.test.ts @@ -46,11 +46,13 @@ const predictionFindManyMock = vi.fn(); const deleteManyMock = vi.fn(); import { - MOOD_PROGNOSIS_REFRESH_CRON, - MOOD_PROGNOSIS_REFRESH_QUEUE, handleMoodPrognosisRefresh, runMoodPrognosisRefresh, } from "@/lib/jobs/mood-prognosis-refresh"; +import { + MOOD_PROGNOSIS_REFRESH_CRON, + MOOD_PROGNOSIS_REFRESH_QUEUE, +} from "@/lib/jobs/mood-prognosis-refresh-queue"; import { MIN_ENTRIES_FOR_FORECAST } from "@/lib/analytics/mood-prognosis/thresholds"; const NOW = new Date("2026-08-08T02:35:00.000Z"); diff --git a/src/lib/jobs/__tests__/nightscout-queues.test.ts b/src/lib/jobs/__tests__/nightscout-queues.test.ts index 6c994d55a..fa2cf8173 100644 --- a/src/lib/jobs/__tests__/nightscout-queues.test.ts +++ b/src/lib/jobs/__tests__/nightscout-queues.test.ts @@ -1,96 +1,78 @@ /** - * v1.17.0 — pg-boss queue registration for the three new poll-only sync layers: - * Nightscout CGM, Polar, and Oura. + * pg-boss queue registration for the three poll-only sync layers: Nightscout + * CGM, Polar, and Oura. * - * Source-as-text guard (the reminder-worker pulls heavy infra we don't boot - * here): assert each queue constant + cron + handler wiring + allQueues - * membership are present — the fast guard against the v1.4.37 dead-queue class - * (a queue declared but never registered in `allQueues`, which silently never - * drains). All three syncs are poll-only (no webhook, no backfill queue): a - * single hourly tick walks every configured account. + * Reads the worker registry (the worker pulls heavy infra we don't boot + * here): each queue is declared in the integration-sync group with its cron + * and handler — the fast guard against the dead-queue class (a queue declared + * but never registered, which silently never drains). All three syncs are + * poll-only (no webhook): a single hourly tick walks every configured + * account. The queue names live in `integration-poll-queues.ts`, so an OAuth + * callback can enqueue a connect-time first sync without the handler graph. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring moved out of the 2143-LOC reminder-worker boot -// file into domain registrars; the Nightscout / Polar / Oura queues live in the -// integration-sync registrar. The dead-queue guard follows the wiring there -// (the handler module is still concatenated for the handler-symbol assertions). -// -// v1.32.28 — the four poll-queue NAMES moved again, into -// `integration-poll-queues.ts`, so an OAuth callback can enqueue a connect-time -// first sync without importing the registrar's handler graph. The guard follows -// them: the name declarations are asserted against that module, everything else -// (allQueues membership, cron schedule, handler binding) still against the -// registrar. -const source = - readFileSync(join(__dirname, "..", "integration-poll-queues.ts"), "utf8") + - readFileSync( - join(__dirname, "..", "reminder", "register-integration-sync.ts"), - "utf8", - ) + - readFileSync(join(__dirname, "..", "reminder", "nightscout-sync.ts"), "utf8"); +import { + NIGHTSCOUT_SYNC_CRON, + NIGHTSCOUT_SYNC_QUEUE, + OURA_SYNC_CRON, + OURA_SYNC_QUEUE, + POLAR_SYNC_CRON, + POLAR_SYNC_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; +import { handleNightscoutSync } from "@/lib/jobs/reminder/nightscout-sync"; +import { handleOuraSync } from "@/lib/jobs/reminder/oura-sync"; +import { handlePolarSync } from "@/lib/jobs/reminder/polar-sync"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -describe("reminder-worker — Nightscout sync queue", () => { +describe("Nightscout sync queue", () => { it("declares the nightscout-sync queue constant", () => { - expect(source).toMatch( - /NIGHTSCOUT_SYNC_QUEUE\s*=\s*["']nightscout-sync["']/, - ); + expect(NIGHTSCOUT_SYNC_QUEUE).toBe("nightscout-sync"); }); - it("registers the queue in allQueues (v1.4.37 dead-queue guard)", () => { - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - expect(block![1]!).toContain("NIGHTSCOUT_SYNC_QUEUE"); + it("declares the queue in the integration-sync group", () => { + expect(groupOf(NIGHTSCOUT_SYNC_QUEUE).name).toBe("integration_sync"); }); it("schedules the hourly poll cron", () => { - expect(source).toMatch(/\[NIGHTSCOUT_SYNC_QUEUE,\s*NIGHTSCOUT_SYNC_CRON\]/); + expect(jobOf(NIGHTSCOUT_SYNC_QUEUE).cron).toBe(NIGHTSCOUT_SYNC_CRON); }); - it("registers a createAndWork handler for the queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,160}NIGHTSCOUT_SYNC_QUEUE[\s\S]{0,160}handleNightscoutSync/, + it("binds the handler for the queue", async () => { + expect(await loadedHandler(NIGHTSCOUT_SYNC_QUEUE)).toBe( + handleNightscoutSync, ); }); }); -describe("reminder-worker — Polar / Oura sync queues", () => { - // The same v1.4.37 dead-queue class applies to the two OAuth-wearable polls - // shipped alongside Nightscout in v1.17.0 — assert their allQueues membership - // so a future refactor can't silently drop one and starve the sync. - const allQueuesBlock = () => { - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - return block![1]!; - }; - +describe("Polar / Oura sync queues", () => { + // The same dead-queue class applies to the two OAuth-wearable polls shipped + // alongside Nightscout — assert their registry membership so a future + // refactor can't silently drop one and starve the sync. it("declares the polar-sync and oura-sync queue constants", () => { - expect(source).toMatch(/POLAR_SYNC_QUEUE\s*=\s*["']polar-sync["']/); - expect(source).toMatch(/OURA_SYNC_QUEUE\s*=\s*["']oura-sync["']/); + expect(POLAR_SYNC_QUEUE).toBe("polar-sync"); + expect(OURA_SYNC_QUEUE).toBe("oura-sync"); }); - it("registers POLAR_SYNC_QUEUE in allQueues (v1.4.37 dead-queue guard)", () => { - expect(allQueuesBlock()).toContain("POLAR_SYNC_QUEUE"); + it("declares POLAR_SYNC_QUEUE in the integration-sync group", () => { + expect(groupOf(POLAR_SYNC_QUEUE).name).toBe("integration_sync"); }); - it("registers OURA_SYNC_QUEUE in allQueues (v1.4.37 dead-queue guard)", () => { - expect(allQueuesBlock()).toContain("OURA_SYNC_QUEUE"); + it("declares OURA_SYNC_QUEUE in the integration-sync group", () => { + expect(groupOf(OURA_SYNC_QUEUE).name).toBe("integration_sync"); }); it("schedules the hourly poll cron for both", () => { - expect(source).toMatch(/\[POLAR_SYNC_QUEUE,\s*POLAR_SYNC_CRON\]/); - expect(source).toMatch(/\[OURA_SYNC_QUEUE,\s*OURA_SYNC_CRON\]/); + expect(jobOf(POLAR_SYNC_QUEUE).cron).toBe(POLAR_SYNC_CRON); + expect(jobOf(OURA_SYNC_QUEUE).cron).toBe(OURA_SYNC_CRON); }); - it("registers a createAndWork handler for each queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,160}POLAR_SYNC_QUEUE[\s\S]{0,160}handlePolarSync/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,160}OURA_SYNC_QUEUE[\s\S]{0,160}handleOuraSync/, - ); + it("binds a handler for each queue", async () => { + expect(await loadedHandler(POLAR_SYNC_QUEUE)).toBe(handlePolarSync); + expect(await loadedHandler(OURA_SYNC_QUEUE)).toBe(handleOuraSync); }); }); diff --git a/src/lib/jobs/__tests__/period-narrative-warm.test.ts b/src/lib/jobs/__tests__/period-narrative-warm.test.ts index a41071ad5..8bdf1bb09 100644 --- a/src/lib/jobs/__tests__/period-narrative-warm.test.ts +++ b/src/lib/jobs/__tests__/period-narrative-warm.test.ts @@ -1,6 +1,4 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; const checkRateLimit = vi.fn(); const getAssistantFlags = vi.fn(); @@ -34,6 +32,14 @@ import { PERIOD_NARRATIVE_QUEUE, PERIOD_NARRATIVE_CRON, } from "../period-narrative-warm"; +import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "../period-narrative-queue"; +import { handlePeriodNarrative } from "../reminder/status-handlers"; +import { insightRetryOptions } from "../reminder/registrar-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; function makePrisma( users: Array<{ id: string; locale: string | null }>, @@ -234,29 +240,22 @@ describe("runPeriodNarrativeWarm", () => { }); describe("queue registration", () => { - // v1.18.1 — the period-narrative wiring moved out of the 2143-LOC - // reminder-worker boot file into the status registrar. The dead-queue guard - // follows the wiring there. - const workerSrc = fs.readFileSync( - path.resolve(__dirname, "../reminder/register-status.ts"), - "utf8", - ); - - it("registers the queue in the allQueues createQueue loop", () => { - const match = workerSrc.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(match).not.toBeNull(); - expect(match![1]).toMatch(/\bPERIOD_NARRATIVE_QUEUE\b/); + it("declares the queue in the status group", () => { + expect(groupOf(PERIOD_NARRATIVE_QUEUE).name).toBe("status"); }); - it("schedules the cron in the schedules table (with retry policy)", () => { - expect(workerSrc).toMatch( - /\[\s*PERIOD_NARRATIVE_QUEUE\s*,\s*PERIOD_NARRATIVE_CRON\s*,\s*\{\s*\.\.\.insightRetryOptions\s*,\s*expireInSeconds:\s*PERIOD_NARRATIVE_EXPIRE_SECONDS\s*,?\s*\}\s*,?\s*\]/, - ); + it("schedules the cron with the retry policy", () => { + const job = jobOf(PERIOD_NARRATIVE_QUEUE); + expect(job.cron).toBe(PERIOD_NARRATIVE_CRON); + expect(job.scheduleOptions).toEqual({ + ...insightRetryOptions, + expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS, + }); }); - it("registers a createAndWork handler for the queue", () => { - expect(workerSrc).toMatch( - /createAndWork[\s\S]{0,120}PERIOD_NARRATIVE_QUEUE/, + it("binds the warm handler to the queue", async () => { + expect(await loadedHandler(PERIOD_NARRATIVE_QUEUE)).toBe( + handlePeriodNarrative, ); }); diff --git a/src/lib/jobs/__tests__/queue-policy-tables.test.ts b/src/lib/jobs/__tests__/queue-policy-tables.test.ts index bc4b5ea04..7a728b6fb 100644 --- a/src/lib/jobs/__tests__/queue-policy-tables.test.ts +++ b/src/lib/jobs/__tests__/queue-policy-tables.test.ts @@ -1,160 +1,138 @@ /** - * Dead-policy guard, in the same spirit as the v1.4.37 dead-queue guards. + * Dead-policy guard, in the same spirit as the dead-queue guards. * - * A queue's `singletonKey` only de-duplicates if its registrar names a policy - * for it: pg-boss constrains `singleton_key` through partial unique indexes - * scoped by queue policy, and the default `standard` policy has none. So the - * enqueue site and the registrar entry are two halves of one guarantee that - * live in different files, and nothing in the type system couples them — - * deleting the registrar entry leaves the `singletonKey` at the call site - * looking entirely correct while it silently stops doing anything. + * A queue's `singletonKey` only de-duplicates if its registry declaration + * names a policy for it: pg-boss constrains `singleton_key` through partial + * unique indexes scoped by queue policy, and the default `standard` policy + * has none. So the enqueue site and the declaration are two halves of one + * guarantee that live in different files, and nothing in the type system + * couples them — dropping the declaration's policy leaves the `singletonKey` + * at the call site looking entirely correct while it silently stops doing + * anything. * - * These assertions are that coupling. Each queue below is listed with the - * policy its call site depends on; dropping or weakening an entry turns this - * suite red instead of quietly restoring the storm. + * These assertions are that coupling. Each queue below is listed by its wire + * name with the policy its call site depends on, and must be declared in the + * named registry group; dropping or weakening an entry turns this suite red + * instead of quietly restoring the storm. * * The behavioural proof that the policies mean what the names say lives in * `tests/integration/queue-singleton-policy.integration.test.ts`, which runs a * real pg-boss against a real Postgres. */ -import fs from "node:fs"; -import path from "node:path"; - import { describe, expect, it } from "vitest"; -const read = (file: string) => - fs.readFileSync(path.resolve(__dirname, "../reminder", file), "utf8"); +import type { JobGroupName } from "@/lib/jobs/registry/define-job"; +import { + groupNamed, + groupOf, + jobOf, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -/** - * Assert that `queue` appears in the registrar's `queuePolicies` table with - * `policy`. Anchored to the table so an unrelated mention of the constant - * elsewhere in the file cannot satisfy it. - */ -function expectPolicy(source: string, queue: string, policy: string) { - const table = source.match( - /const queuePolicies: QueuePolicyTable = \{([\s\S]*?)\n\};/, - ); - expect(table, "registrar has no queuePolicies table").not.toBeNull(); - const entry = new RegExp(`\\[${queue}\\]:\\s*\\{\\s*policy:\\s*"${policy}"`); +/** Assert that `queue` is declared in `group` with `policy`. */ +function expectPolicy(group: JobGroupName, queue: string, policy: string) { + expect(groupOf(queue).name).toBe(group); expect( - table![1], + jobOf(queue).policy?.policy, `${queue} must carry the "${policy}" policy — without it the singletonKey at its enqueue site is inert`, - ).toMatch(entry); + ).toBe(policy); } -describe("rollup registrar queue policies", () => { - const source = read("register-rollup.ts"); - +describe("rollup group queue policies", () => { // `short`, not `exclusive`: the handler re-reads the bucket when it starts, // so only QUEUED duplicates may be collapsed. Promoting either of these to // `exclusive` would strand a measurement written while a recompute ran. it.each([ - ["ROLLUP_RECOMPUTE_QUEUE", "short"], - ["MOOD_ROLLUP_RECOMPUTE_QUEUE", "short"], + ["rollup-recompute", "short"], + ["mood-rollup-recompute", "short"], ])("%s is %s", (queue, policy) => { - expectPolicy(source, queue, policy); + expectPolicy("rollup", queue, policy); }); // Per-user, self-converging backfills: a second concurrent run is pure // duplicated work and discovery re-offers the job while it is outstanding. it.each([ - ["ROLLUP_FULL_BACKFILL_QUEUE", "exclusive"], - ["MOOD_ROLLUP_FULL_BACKFILL_QUEUE", "exclusive"], - ["STEP_CONSOLIDATION_QUEUE", "exclusive"], - ["STEP_CONSOLIDATION_REPAIR_QUEUE", "exclusive"], - ["CUMULATIVE_PR_REDERIVE_QUEUE", "exclusive"], - ["MEAN_CONSOLIDATION_QUEUE", "exclusive"], - ["DENSE_INTRADAY_RETENTION_QUEUE", "exclusive"], - ["DENSE_INTRADAY_HOURLY_REBUILD_QUEUE", "exclusive"], + ["rollup-full-backfill", "exclusive"], + ["mood-rollup-full-backfill", "exclusive"], + ["step-consolidation", "exclusive"], + ["step-consolidation-repair", "exclusive"], + ["cumulative-pr-rederive", "exclusive"], + ["mean-consolidation", "exclusive"], + ["dense-intraday-retention", "exclusive"], + ["dense-intraday-hourly-rebuild", "exclusive"], ])("%s is %s", (queue, policy) => { - expectPolicy(source, queue, policy); + expectPolicy("rollup", queue, policy); }); }); -describe("status registrar queue policies", () => { - const source = read("register-status.ts"); - - it("MORNING_DIGEST_REFRESH_QUEUE is exclusive", () => { +describe("status group queue policies", () => { + it("morning-digest-refresh is exclusive", () => { // Seven sleep-write seams enqueue the same user+localDate key. Without // this the at-most-once-per-morning contract does not hold and the failure // path re-triggers forced comprehensive generations against the provider. - expectPolicy(source, "MORNING_DIGEST_REFRESH_QUEUE", "exclusive"); + expectPolicy("status", "morning-digest-refresh", "exclusive"); }); }); -describe("integration-sync registrar queue policies", () => { - const source = read("register-integration-sync.ts"); - +describe("integration-sync group queue policies", () => { it.each([ - ["WHOOP_BACKFILL_QUEUE", "exclusive"], - ["FITBIT_BACKFILL_QUEUE", "exclusive"], - ["GOOGLE_HEALTH_BACKFILL_QUEUE", "exclusive"], - ["STRAVA_BACKFILL_QUEUE", "exclusive"], - ["OURA_BACKFILL_QUEUE", "exclusive"], - ["GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE", "exclusive"], - ["FITBIT_SLEEP_REPAIR_QUEUE", "exclusive"], - ["SLEEP_TIMELINE_BACKFILL_QUEUE", "exclusive"], - ["LAB_BIOMARKER_BACKFILL_QUEUE", "exclusive"], + ["whoop-backfill", "exclusive"], + ["fitbit-backfill", "exclusive"], + ["google-health-backfill", "exclusive"], + ["strava-backfill", "exclusive"], + ["oura-backfill", "exclusive"], + ["google-health-sleep-repair", "exclusive"], + ["fitbit-sleep-repair", "exclusive"], + ["sleep-timeline-backfill", "exclusive"], + ["lab-biomarker-backfill", "exclusive"], ])("%s is %s", (queue, policy) => { - expectPolicy(source, queue, policy); + expectPolicy("integration_sync", queue, policy); }); }); -describe("maintenance registrar queue policies", () => { - const source = read("register-maintenance.ts"); - +describe("maintenance group queue policies", () => { it.each([ - ["INTAKE_SLOT_DEDUP_QUEUE", "exclusive"], - ["NOTE_ENCRYPTION_BACKFILL_QUEUE", "exclusive"], - ["MED_NOTES_ENCRYPTION_BACKFILL_QUEUE", "exclusive"], - ["FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE", "exclusive"], - ["DOCUMENT_THUMBNAIL_BACKFILL_QUEUE", "exclusive"], - ["CONTENT_INDEX_BACKFILL_QUEUE", "exclusive"], - ["ENCRYPTION_KEY_ROTATE_QUEUE", "exclusive"], + ["intake-slot-dedup", "exclusive"], + ["note-encryption-backfill", "exclusive"], + ["med-notes-encryption-backfill", "exclusive"], + ["free-text-encryption-backfill", "exclusive"], + ["document-thumbnail-backfill", "exclusive"], + ["document-content-index-backfill", "exclusive"], + ["encryption-key-rotate", "exclusive"], ])("%s is %s", (queue, policy) => { - expectPolicy(source, queue, policy); + expectPolicy("maintenance", queue, policy); }); // Per-document queues have no discovery pass to re-converge, so a re-process // requested after the running job read the old bytes must be admitted. it.each([ - ["DOCUMENT_INDEX_QUEUE", "short"], - ["DOCUMENT_THUMBNAIL_QUEUE", "short"], - ["DOCUMENT_SUMMARY_QUEUE", "short"], + ["document-index", "short"], + ["document-thumbnail", "short"], + ["document-summary", "short"], ])("%s is %s", (queue, policy) => { - expectPolicy(source, queue, policy); + expectPolicy("maintenance", queue, policy); }); - it("ENVIRONMENT_FETCH_QUEUE is deliberately left on standard", () => { + it("environment-fetch is deliberately left on standard", () => { // Its explicit-range backfill sends with no key at all, on purpose, so any // policy would collapse two different requested date ranges onto the shared - // empty key. This asserts the omission is intentional and documented, so a - // future reader does not "complete" the table and silently merge them. - const table = source.match( - /const queuePolicies: QueuePolicyTable = \{([\s\S]*?)\n\};/, - ); - expect(table![1]).not.toMatch(/\[ENVIRONMENT_FETCH_QUEUE\]/); - expect(source).toMatch(/ENVIRONMENT_FETCH_QUEUE is LEFT ALONE/); + // empty key. A future reader must not "complete" the declaration and + // silently merge them. + expect(groupOf("environment-fetch").name).toBe("maintenance"); + expect(jobOf("environment-fetch").policy).toBeUndefined(); }); }); describe("every policy decision carries a reason", () => { - it.each([ - "register-rollup.ts", - "register-status.ts", - "register-integration-sync.ts", - "register-maintenance.ts", - ])("%s", (file) => { - const table = read(file).match( - /const queuePolicies: QueuePolicyTable = \{([\s\S]*?)\n\};/, - ); - expect(table).not.toBeNull(); - - const policies = table![1].match(/policy:\s*"(?:short|exclusive)"/g) ?? []; - const reasons = table![1].match(/reason:\s*\n?\s*"/g) ?? []; - expect(policies.length).toBeGreaterThan(0); - // A policy without a stated reason is the thing this whole change exists - // to prevent: the next person re-deriving intent from the call sites. - expect(reasons.length).toBe(policies.length); - }); + it.each(["rollup", "status", "integration_sync", "maintenance"] as const)( + "%s", + (name) => { + const decided = groupNamed(name).jobs.filter((job) => job.policy); + expect(decided.length).toBeGreaterThan(0); + // A policy without a stated reason is the thing this whole change exists + // to prevent: the next person re-deriving intent from the call sites. + for (const job of decided) { + expect(job.policy!.reason.trim(), job.queue).not.toBe(""); + } + }, + ); }); diff --git a/src/lib/jobs/__tests__/read-side-effect-queues.test.ts b/src/lib/jobs/__tests__/read-side-effect-queues.test.ts index 71c952909..d956f6d95 100644 --- a/src/lib/jobs/__tests__/read-side-effect-queues.test.ts +++ b/src/lib/jobs/__tests__/read-side-effect-queues.test.ts @@ -2,72 +2,65 @@ * Dead-queue guard for the two queues that took the writes off the read paths. * * Moving a write off a GET only works if the new owner actually runs. A queue - * that is not in `allQueues` is never provisioned, a schedule without a - * `createAndWork` binding drains into nothing, and either failure is silent: + * the worker registry does not declare is never provisioned, a schedule + * without a bound handler drains into nothing, and either failure is silent: * the cron fires, the job vanishes, and the only symptom is a forecast that * stops moving or an unlock date that is never pinned. Both look exactly like * "nothing to do". * - * Source-text assertions over the registrar, matching the sibling - * `*-queue.test.ts` guards. + * Reads the registry declarations, matching the sibling `*-queue.test.ts` + * guards. */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; +import { handleAchievementUnlockSweep } from "@/lib/jobs/achievement-unlock-sweep"; +import { + ACHIEVEMENT_UNLOCK_SWEEP_CRON, + ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, +} from "@/lib/jobs/achievement-unlock-sweep-queue"; +import { + CYCLE_PREDICTION_REFRESH_CRON, + CYCLE_PREDICTION_REFRESH_QUEUE, + handleCyclePredictionRefresh, +} from "@/lib/jobs/cycle-prediction-refresh"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; import { SUBSYSTEM_SURFACE } from "@/lib/jobs/subsystem-surface"; -const registrar = readFileSync( - join(__dirname, "..", "reminder", "register-maintenance.ts"), - "utf8", -); - -const allQueues = registrar.match(/const allQueues\s*=\s*\[([\s\S]*?)\n\];/); - const QUEUES = [ { label: "cycle-prediction-refresh", - binding: "CYCLE_PREDICTION_REFRESH_QUEUE", - cron: "CYCLE_PREDICTION_REFRESH_CRON", - handler: "handleCyclePredictionRefresh", + queue: CYCLE_PREDICTION_REFRESH_QUEUE, + cron: CYCLE_PREDICTION_REFRESH_CRON, + handler: handleCyclePredictionRefresh, name: "cycle-prediction-refresh", }, { label: "achievement-unlock-sweep", - binding: "ACHIEVEMENT_UNLOCK_SWEEP_QUEUE", - cron: "ACHIEVEMENT_UNLOCK_SWEEP_CRON", - handler: "handleAchievementUnlockSweep", + queue: ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, + cron: ACHIEVEMENT_UNLOCK_SWEEP_CRON, + handler: handleAchievementUnlockSweep, name: "achievement-unlock-sweep", }, ]; describe("queues that own a write moved off a GET", () => { - it("extracted an allQueues list to assert against", () => { - // The regexes below all read from this capture; if it ever stops - // matching, every assertion in this file would pass on an empty string. - expect(allQueues).not.toBeNull(); - expect(allQueues![1].length).toBeGreaterThan(0); - }); - for (const queue of QUEUES) { describe(queue.label, () => { - it("is provisioned in the allQueues list", () => { - expect(allQueues![1]).toMatch(new RegExp(`\\b${queue.binding}\\b`)); + it("is declared in the maintenance group", () => { + expect(queue.queue).toBe(queue.name); + expect(groupOf(queue.queue).name).toBe("maintenance"); }); it("carries a cron schedule", () => { - expect(registrar).toMatch( - new RegExp(`\\[\\s*${queue.binding},\\s*${queue.cron}`), - ); + expect(jobOf(queue.queue).cron).toBe(queue.cron); }); - it("binds a createAndWork handler that drains it", () => { - expect(registrar).toMatch( - new RegExp( - `createAndWork[\\s\\S]{0,200}${queue.binding}[\\s\\S]{0,200}${queue.handler}`, - ), - ); + it("binds the handler that drains it", async () => { + expect(await loadedHandler(queue.queue)).toBe(queue.handler); }); it("declares a failure audience", () => { diff --git a/src/lib/jobs/__tests__/recovery-score-queue.test.ts b/src/lib/jobs/__tests__/recovery-score-queue.test.ts index 65628b9e9..d97c088ff 100644 --- a/src/lib/jobs/__tests__/recovery-score-queue.test.ts +++ b/src/lib/jobs/__tests__/recovery-score-queue.test.ts @@ -1,10 +1,10 @@ /** - * v1.10.0 — computed scores (WX-C). Recovery-score queue registration guard. + * Computed scores: recovery-score queue registration guard. * - * Same source-text-grep approach as the other queue-wiring guards: assert - * the queue is imported, registered in `allQueues`, scheduled, and wired to - * a `createAndWork` handler — without booting pg-boss + Prisma. An unregistered - * queue silently never drains (the recurring past bug this guards against). + * Reads the worker registry: the queue is declared in the status group, + * scheduled on its cron, and bound to the handler that runs the pass. An + * unregistered queue silently never drains (the recurring past bug this + * guards against). * * Also pins the iOS-write rejection of the server-owned COMPUTED source on * the batch ingest route: the `batchSourceEnum` allowlist must stay exactly @@ -16,11 +16,16 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring moved out of the 2143-LOC reminder-worker boot -// file into domain registrars; the recovery-score queue lives in the status -// registrar. The dead-queue guard follows the wiring there. -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-status.ts"); -const workerSource = readFileSync(REGISTRAR_PATH, "utf8"); +import { + RECOVERY_SCORE_CRON, + RECOVERY_SCORE_QUEUE, +} from "@/lib/jobs/recovery-score-queue"; +import { handleRecoveryScore } from "@/lib/jobs/reminder/status-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; const BATCH_ROUTE_PATH = join( __dirname, @@ -35,32 +40,19 @@ const BATCH_ROUTE_PATH = join( ); const batchSource = readFileSync(BATCH_ROUTE_PATH, "utf8"); -describe("reminder-worker — recovery-score wiring", () => { - it("imports the queue symbols from the recovery-score module", () => { - expect(workerSource).toMatch(/from\s*["']@\/lib\/jobs\/recovery-score["']/); - expect(workerSource).toMatch(/\bRECOVERY_SCORE_QUEUE\b/); - expect(workerSource).toMatch(/\bRECOVERY_SCORE_CRON\b/); - expect(workerSource).toMatch(/\brunRecoveryScore\b/); - }); - - it("registers the recovery-score queue in the allQueues loop", () => { - const allQueuesMatch = workerSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bRECOVERY_SCORE_QUEUE\b/); +describe("recovery-score registration", () => { + it("declares the recovery-score queue in the status group", () => { + expect(groupOf(RECOVERY_SCORE_QUEUE).name).toBe("status"); }); it("schedules the recovery-score cron", () => { - expect(workerSource).toMatch( - /\[RECOVERY_SCORE_QUEUE,\s*RECOVERY_SCORE_CRON\]/, - ); + const job = jobOf(RECOVERY_SCORE_QUEUE); + expect(job.cron).toBe(RECOVERY_SCORE_CRON); + expect(job.work).toEqual({ localConcurrency: 1 }); }); - it("registers a createAndWork handler that runs the recovery-score pass", () => { - expect(workerSource).toMatch( - /createAndWork[\s\S]{0,200}RECOVERY_SCORE_QUEUE[\s\S]{0,400}runRecoveryScore/, - ); + it("binds the handler that runs the recovery-score pass", async () => { + expect(await loadedHandler(RECOVERY_SCORE_QUEUE)).toBe(handleRecoveryScore); }); }); diff --git a/src/lib/jobs/__tests__/restore-drill.test.ts b/src/lib/jobs/__tests__/restore-drill.test.ts index 81eb035ef..07136af8d 100644 --- a/src/lib/jobs/__tests__/restore-drill.test.ts +++ b/src/lib/jobs/__tests__/restore-drill.test.ts @@ -2,13 +2,12 @@ import type { Readable } from "node:stream"; import { describe, it, expect, beforeEach, vi } from "vitest"; import { _resetCryptoCacheForTests, encrypt } from "@/lib/crypto"; import { encryptBackup } from "../offhost-backup"; +import { handleRestoreDrill, runRestoreDrill } from "../restore-drill"; import { - handleRestoreDrill, - runRestoreDrill, RESTORE_DRILL_CRON, RESTORE_DRILL_QUEUE, RESTORE_DRILL_SEND_OPTIONS, -} from "../restore-drill"; +} from "@/lib/jobs/restore-drill-queue"; import { QUEUE_RUNTIME } from "../queue-runtime"; vi.mock("../report-worker-error", () => ({ diff --git a/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts b/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts index cc803ecc5..f689739ec 100644 --- a/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts +++ b/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts @@ -1,50 +1,60 @@ /** - * v1.17.1 — pg-boss queue registration for the one-shot sleep-timeline backfill. + * pg-boss queue registration for the one-shot sleep-timeline backfill. * - * The reminder-worker imports heavy infrastructure, so we read the source as - * text and assert the queue constant is declared, registered in `allQueues`, - * wired to a `createAndWork` handler, and enqueued at boot — the fast guard that - * catches the v1.4.37 dead-queue class (a queue declared but never registered, - * which silently never drains). + * Reads the worker registry: the source queue is declared in the + * integration-sync group, forwards each job to the shared admission queue, + * and is offered again by the boot discovery. This is the fast guard for the + * dead-queue class (a queue declared but never registered, which silently + * never drains). */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring + boot discovery moved out of the 2143-LOC -// reminder-worker boot file into domain registrars; the sleep-timeline backfill -// lives in the integration-sync registrar. The dead-queue guard follows it. -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-integration-sync.ts", -); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { INTEGRATION_BACKFILL_ADMISSION_QUEUE } from "@/lib/jobs/integration-backfill-admission"; +import { + discoveryOf, + groupOf, + jobOf, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { + SLEEP_TIMELINE_BACKFILL_CONCURRENCY, + SLEEP_TIMELINE_BACKFILL_QUEUE, +} from "@/lib/jobs/sleep-timeline-backfill-queue"; -describe("reminder-worker — sleep-timeline backfill queue", () => { - it("registers the queue in allQueues (v1.4.37 dead-queue guard)", () => { - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - expect(block![1]!).toContain("SLEEP_TIMELINE_BACKFILL_QUEUE"); +describe("sleep-timeline backfill registration", () => { + it("declares the queue in the integration-sync group", () => { + expect(groupOf(SLEEP_TIMELINE_BACKFILL_QUEUE).name).toBe( + "integration_sync", + ); + const job = jobOf(SLEEP_TIMELINE_BACKFILL_QUEUE); + expect(job.policy?.policy).toBe("exclusive"); + expect(job.work).toEqual({ + localConcurrency: SLEEP_TIMELINE_BACKFILL_CONCURRENCY, + }); }); - it("wires a createAndWork handler through the shared admission queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,260}SLEEP_TIMELINE_BACKFILL_QUEUE[\s\S]{0,300}enqueueIntegrationBackfillAdmission/, - ); + it("forwards each job through the shared admission queue", async () => { + const { boss, sends } = recordingBoss(); + const handler = await jobOf(SLEEP_TIMELINE_BACKFILL_QUEUE).load({ boss }); + await handler([ + { + id: "j1", + name: SLEEP_TIMELINE_BACKFILL_QUEUE, + data: { userId: "u1", provider: "WHOOP" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + expect(sends).toHaveLength(1); + expect(sends[0]!.queue).toBe(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(sends[0]!.data).toMatchObject({ kind: "sleep-timeline-backfill" }); }); it("wires the boot discovery", () => { - expect(source).toMatch( - /enqueueBootTimeSleepTimelineBackfill\(\s*bootStaggerSecondsFor\("sleep-timeline-backfill"\)/, - ); - }); - - it("imports the backfill exports", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/jobs\/sleep-timeline-backfill["']/, + expect(discoveryOf("integration_sync")).toContain( + "sleep-timeline-backfill", ); }); }); diff --git a/src/lib/jobs/__tests__/status-cron-candidates.test.ts b/src/lib/jobs/__tests__/status-cron-candidates.test.ts index 331de5d62..b912a4c27 100644 --- a/src/lib/jobs/__tests__/status-cron-candidates.test.ts +++ b/src/lib/jobs/__tests__/status-cron-candidates.test.ts @@ -1,5 +1,5 @@ /** - * v1.15.20 — unit tests for the shared 02:xx status-cron discovery. + * Unit tests for the shared 02:xx status-cron discovery. * * Pins the three gates the discovery applies: * 1. the operator assistant kill-switch (`insightStatus` flag), @@ -9,8 +9,8 @@ * 3. the pregenerate-candidate skip (a stale or missing comprehensive * cache → the 04:30 pass owns the user; the 02:xx crons keep the * fresh-cache accounts), applied only while `briefing` is on, - * plus the mood-status queue registration in the worker source (the - * v1.4.37 dead-queue class guard). + * plus the nightly status batch's declaration in the worker registry (the + * dead-queue class guard). */ import { describe, it, expect, vi, beforeEach } from "vitest"; import fs from "node:fs"; @@ -25,6 +25,15 @@ vi.mock("@/lib/feature-flags", async () => import { findStatusCronCandidates } from "../status-cron-candidates"; import { PREGENERATE_STALE_MS } from "../insight-pregenerate"; +import { handleGeneralStatusGenerate } from "../reminder/insights-handlers"; +import { nightlyInsightPassOptions } from "../reminder/registrar-shared"; +import { allJobDefinitions } from "../registry"; +import { + groupNamed, + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; const NOW = new Date("2026-06-10T02:00:00.000Z"); const STALE_AT = new Date(NOW.getTime() - PREGENERATE_STALE_MS - 60_000); @@ -176,10 +185,7 @@ describe("findStatusCronCandidates — gates", () => { }); describe("the nightly status batch is the one status cron", () => { - const workerSrc = fs.readFileSync( - path.resolve(__dirname, "../reminder/register-status.ts"), - "utf8", - ); + const GENERAL_STATUS_QUEUE = "insights-general-status"; const RETIRED = [ "insights-blood-pressure-status", "insights-weight-status", @@ -189,27 +195,22 @@ describe("the nightly status batch is the one status cron", () => { "insights-medication-compliance-status", ]; - it("registers and schedules the 02:00 batch with the nightly pass options", () => { - const match = workerSrc.match(/const allQueues\s*=\s*\[([\s\S]*?)\];/); - expect(match).not.toBeNull(); - expect(match![1]).toMatch(/\bGENERAL_STATUS_QUEUE\b/); - expect(workerSrc).toMatch( - /\[\s*GENERAL_STATUS_QUEUE\s*,\s*GENERAL_STATUS_CRON\s*,\s*nightlyInsightPassOptions\s*\]/, + it("registers and schedules the 02:00 batch with the nightly pass options", async () => { + expect(groupOf(GENERAL_STATUS_QUEUE).name).toBe("status"); + const job = jobOf(GENERAL_STATUS_QUEUE); + expect(job.cron).toBe("0 2 * * *"); + expect(job.scheduleOptions).toBe(nightlyInsightPassOptions); + expect(await loadedHandler(GENERAL_STATUS_QUEUE)).toBe( + handleGeneralStatusGenerate, ); - expect(workerSrc).toMatch(/GENERAL_STATUS_CRON\s*=\s*"0 2 \* \* \*"/); }); it("retires the six per-metric crons on an instance that still has them", () => { - const retired = workerSrc.match( - /const RETIRED_STATUS_QUEUES\s*=\s*\[([\s\S]*?)\]/, + expect([...(groupNamed("status").retired ?? [])].sort()).toEqual( + [...RETIRED].sort(), ); - expect(retired).not.toBeNull(); - for (const name of RETIRED) expect(retired![1]).toContain(`"${name}"`); - expect(workerSrc).toMatch( - /await retireQueues\(boss,\s*RETIRED_STATUS_QUEUES\)/, - ); - const live = workerSrc.replace(retired![0], ""); - for (const name of RETIRED) expect(live).not.toContain(`"${name}"`); + const live = allJobDefinitions().map((job) => job.queue); + for (const name of RETIRED) expect(live).not.toContain(name); }); it("drives the batch through the shared discovery", () => { diff --git a/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts b/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts index e34c295cd..2f679fb63 100644 --- a/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts +++ b/src/lib/jobs/__tests__/step-consolidation-repair-queue.test.ts @@ -1,51 +1,46 @@ /** - * v1.28.37 — step-consolidation repair queue registration guard + - * `extractStatsDay` unit coverage. + * Step-consolidation repair queue registration guard + `extractStatsDay` + * unit coverage. * - * Same source-text-grep approach as the drain + step-consolidation + - * mean-consolidation guards: assert the repair queue is registered in - * `allQueues`, a `createAndWork` handler is wired against it, and the - * boot-discovery enqueue helper fires — without booting pg-boss + Prisma. - * An unregistered queue silently never drains (the dead-queue class). + * Reads the worker registry: the repair queue is declared in the rollup + * group, bound to the repair handler, and offered again by the rollup boot + * discovery. An unregistered queue silently never drains (the dead-queue + * class). */ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - import { describe, expect, it } from "vitest"; -import { extractStatsDay } from "../step-consolidation-repair"; +import { handleStepConsolidationRepair } from "@/lib/jobs/reminder/rollup-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; -const REGISTRAR_PATH = join(__dirname, "..", "reminder", "register-rollup.ts"); -const source = readFileSync(REGISTRAR_PATH, "utf8"); +import { + STEP_CONSOLIDATION_REPAIR_CONCURRENCY, + STEP_CONSOLIDATION_REPAIR_QUEUE, + extractStatsDay, +} from "../step-consolidation-repair"; -describe("reminder-worker — step-consolidation-repair wiring", () => { - it("imports the queue symbols from the repair module", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/jobs\/step-consolidation-repair["']/, - ); - expect(source).toMatch(/\bSTEP_CONSOLIDATION_REPAIR_QUEUE\b/); - expect(source).toMatch(/\brunStepConsolidationRepairForUser\b/); - expect(source).toMatch(/\benqueueBootTimeStepConsolidationRepair\b/); +describe("step-consolidation-repair registration", () => { + it("declares the repair queue in the rollup group", () => { + expect(groupOf(STEP_CONSOLIDATION_REPAIR_QUEUE).name).toBe("rollup"); + const job = jobOf(STEP_CONSOLIDATION_REPAIR_QUEUE); + expect(job.policy?.policy).toBe("exclusive"); + expect(job.work).toEqual({ + localConcurrency: STEP_CONSOLIDATION_REPAIR_CONCURRENCY, + }); }); - it("registers the repair queue in the allQueues loop", () => { - const allQueuesMatch = source.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bSTEP_CONSOLIDATION_REPAIR_QUEUE\b/); - }); - - it("registers a createAndWork handler against the repair queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}STEP_CONSOLIDATION_REPAIR_QUEUE[\s\S]{0,400}runStepConsolidationRepairForUser/, + it("binds the repair handler", async () => { + expect(await loadedHandler(STEP_CONSOLIDATION_REPAIR_QUEUE)).toBe( + handleStepConsolidationRepair, ); }); - it("fires the boot-discovery enqueue helper", () => { - expect(source).toMatch( - /await enqueueBootTimeStepConsolidationRepair\([^)]*\)/, - ); + it("runs the boot discovery pass", () => { + expect(discoveryOf("rollup")).toContain("step-consolidation-repair"); }); }); diff --git a/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts b/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts index 969d80fba..2595ac000 100644 --- a/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts +++ b/src/lib/jobs/__tests__/stress-strain-retention-queue.test.ts @@ -1,39 +1,125 @@ /** - * v1.10.0 — computed scores (WX-E). Queue-registration guards for the - * Stress + Strain nightly score jobs and the dense intra-day retention - * drain. + * Computed scores: queue-registration guards for the Stress + Strain nightly + * score jobs and the dense intra-day retention drain. * - * Same source-text-grep approach as the WX-C recovery-score guard: assert - * each queue is imported, registered in `allQueues`, scheduled (the two - * cron-driven score jobs), and wired to a `createAndWork` handler — without - * booting pg-boss + Prisma. An unregistered queue silently never drains - * (the recurring past bug this guards against). + * Reads the worker registry: each queue is declared in its group, scheduled + * (the two cron-driven score jobs), bound to its handler, and offered again by + * boot discovery where it has one — without booting pg-boss + Prisma. An + * unregistered queue silently never drains (the recurring past bug this + * guards against). The retention handler and the nightly drain run against + * mocked passes. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import type { Job } from "pg-boss"; +import { describe, expect, it, vi } from "vitest"; -// v1.18.1 — the queue wiring moved out of the 2143-LOC reminder-worker boot -// file into domain registrars. Stress + Strain live in the status registrar; -// the dense intra-day retention queue (and the nightly fold onto the -// drain-cumulative tick) lives in the rollup registrar. Concatenate both so the -// dead-queue guard follows the wiring into each. -const STATUS_REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-status.ts", +const { + runDenseIntradayRetention, + runDenseIntradayRetentionForUser, + workerLog, +} = vi.hoisted(() => ({ + runDenseIntradayRetention: vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + daysConsolidated: 0, + perSampleRowsSoftDeleted: 0, + hourlyRowsUpserted: 0, + dailyRowsRetired: 0, + }, + })), + runDenseIntradayRetentionForUser: vi.fn(async () => ({ + daysConsolidated: 1, + perSampleRowsSoftDeleted: 2, + derivedRestingRowsUpserted: 3, + stoppedEarly: false, + })), + workerLog: vi.fn(), +})); +vi.mock( + "@/lib/measurements/dense-intraday-retention", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/dense-intraday-retention") + >()), + runDenseIntradayRetention, + }), ); -const ROLLUP_REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-rollup.ts", +vi.mock("@/lib/jobs/dense-intraday-retention", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/jobs/dense-intraday-retention") + >()), + runDenseIntradayRetentionForUser, +})); +vi.mock( + "@/lib/measurements/drain-per-sample-cumulative", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/drain-per-sample-cumulative") + >()), + drainPerSampleCumulative: vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + bucketsCollapsed: 0, + perSampleRowsDeleted: 0, + dailyRowsUpserted: 0, + }, + })), + }), ); -const source = - readFileSync(STATUS_REGISTRAR_PATH, "utf8") + - readFileSync(ROLLUP_REGISTRAR_PATH, "utf8"); +vi.mock( + "@/lib/measurements/consolidate-daily-mean", + async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/measurements/consolidate-daily-mean") + >()), + consolidateDailyMean: vi.fn(async () => ({ + stoppedEarly: false, + totals: { + usersScanned: 0, + daysConsolidated: 0, + perSampleRowsSoftDeleted: 0, + dailyRowsUpserted: 0, + daysFailed: 0, + }, + })), + }), +); +vi.mock("@/lib/jobs/reminder/shared", async (importOriginal) => ({ + ...(await importOriginal()), + getWorkerPrisma: () => ({}), + workerLog, +})); + +import { DENSE_INTRADAY_HOURLY_REBUILD_QUEUE } from "@/lib/jobs/dense-intraday-hourly-rebuild-queue"; +import { DENSE_INTRADAY_RETENTION_QUEUE } from "@/lib/jobs/dense-intraday-retention-queue"; +import { DRAIN_CUMULATIVE_QUEUE } from "@/lib/jobs/drain-cumulative-queue"; +import { + handleDenseIntradayHourlyRebuild, + handleDenseIntradayRetention, +} from "@/lib/jobs/reminder/rollup-handlers"; +import { + handleStrainScore, + handleStressScore, +} from "@/lib/jobs/reminder/status-handlers"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { + STRAIN_SCORE_CRON, + STRAIN_SCORE_QUEUE, +} from "@/lib/jobs/strain-score-queue"; +import { + STRESS_SCORE_CRON, + STRESS_SCORE_QUEUE, +} from "@/lib/jobs/stress-score-queue"; const DENSE_RETENTION_PATH = join( __dirname, @@ -42,91 +128,70 @@ const DENSE_RETENTION_PATH = join( ); const denseRetentionSource = readFileSync(DENSE_RETENTION_PATH, "utf8"); -function allQueuesBlock(): string { - // Each registrar owns its own `const allQueues = [...]`; union every block so - // a queue registered in either the status or rollup registrar satisfies the - // membership assertion. - const blocks = [...source.matchAll(/const allQueues\s*=\s*\[([\s\S]*?)\];/g)]; - expect(blocks.length).toBeGreaterThan(0); - return blocks.map((m) => m[1]).join("\n"); +function job(name: string, data: T): Job { + return { + id: "j1", + name, + data, + expireInSeconds: 900, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }; } -describe("reminder-worker — stress-score wiring", () => { - it("imports the queue symbols from the stress-score module", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/stress-score["']/); - expect(source).toMatch(/\bSTRESS_SCORE_QUEUE\b/); - expect(source).toMatch(/\bSTRESS_SCORE_CRON\b/); - expect(source).toMatch(/\brunStressScore\b/); - }); - - it("registers the stress-score queue in allQueues", () => { - expect(allQueuesBlock()).toMatch(/\bSTRESS_SCORE_QUEUE\b/); +describe("stress-score registration", () => { + it("declares the stress-score queue in the status group", () => { + expect(groupOf(STRESS_SCORE_QUEUE).name).toBe("status"); }); it("schedules the stress-score cron", () => { - expect(source).toMatch(/\[STRESS_SCORE_QUEUE,\s*STRESS_SCORE_CRON\]/); + expect(jobOf(STRESS_SCORE_QUEUE).cron).toBe(STRESS_SCORE_CRON); }); - it("registers a createAndWork handler that runs the stress-score pass", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}STRESS_SCORE_QUEUE[\s\S]{0,400}runStressScore/, - ); + it("binds the handler that runs the stress-score pass", async () => { + expect(await loadedHandler(STRESS_SCORE_QUEUE)).toBe(handleStressScore); }); }); -describe("reminder-worker — strain-score wiring", () => { - it("imports the queue symbols from the strain-score module", () => { - expect(source).toMatch(/from\s*["']@\/lib\/jobs\/strain-score["']/); - expect(source).toMatch(/\bSTRAIN_SCORE_QUEUE\b/); - expect(source).toMatch(/\bSTRAIN_SCORE_CRON\b/); - expect(source).toMatch(/\brunStrainScore\b/); - }); - - it("registers the strain-score queue in allQueues", () => { - expect(allQueuesBlock()).toMatch(/\bSTRAIN_SCORE_QUEUE\b/); +describe("strain-score registration", () => { + it("declares the strain-score queue in the status group", () => { + expect(groupOf(STRAIN_SCORE_QUEUE).name).toBe("status"); }); it("schedules the strain-score cron", () => { - expect(source).toMatch(/\[STRAIN_SCORE_QUEUE,\s*STRAIN_SCORE_CRON\]/); + expect(jobOf(STRAIN_SCORE_QUEUE).cron).toBe(STRAIN_SCORE_CRON); }); - it("registers a createAndWork handler that runs the strain-score pass", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,200}STRAIN_SCORE_QUEUE[\s\S]{0,400}runStrainScore/, - ); + it("binds the handler that runs the strain-score pass", async () => { + expect(await loadedHandler(STRAIN_SCORE_QUEUE)).toBe(handleStrainScore); }); }); -describe("reminder-worker — dense intra-day retention wiring", () => { - it("imports the queue symbols from the dense-intraday-retention module", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/jobs\/dense-intraday-retention["']/, - ); - expect(source).toMatch(/\bDENSE_INTRADAY_RETENTION_QUEUE\b/); - expect(source).toMatch(/\brunDenseIntradayRetentionForUser\b/); - expect(source).toMatch(/\benqueueBootTimeDenseIntradayRetention\b/); - }); - - it("registers the dense-intraday-retention queue in allQueues", () => { - expect(allQueuesBlock()).toMatch(/\bDENSE_INTRADAY_RETENTION_QUEUE\b/); - }); - - it("registers a createAndWork handler against the dense-intraday-retention queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,250}DENSE_INTRADAY_RETENTION_QUEUE[\s\S]{0,500}runDenseIntradayRetentionForUser/, +describe("dense intra-day retention registration", () => { + it("declares the dense-intraday-retention queue in the rollup group", () => { + expect(groupOf(DENSE_INTRADAY_RETENTION_QUEUE).name).toBe("rollup"); + expect(jobOf(DENSE_INTRADAY_RETENTION_QUEUE).policy?.policy).toBe( + "exclusive", ); }); - it("fires the boot-discovery enqueue helper", () => { - expect(source).toMatch(/await enqueueBootTimeDenseIntradayRetention\(\)/); + it("binds the per-user retention handler", async () => { + expect(await loadedHandler(DENSE_INTRADAY_RETENTION_QUEUE)).toBe( + handleDenseIntradayRetention, + ); + }); + + it("fires the boot-discovery pass", () => { + expect(discoveryOf("rollup")).toContain("dense-intraday-retention"); }); it("defers the boot-discovery drain past the startup storm (P2028 guard)", () => { // The boot-time per-user retention jobs MUST carry a startAfter delay so // the transaction-per-day drain never contends with the deploy storm + // migration + other boot backfills + foreground health-checks for the - // shared connection pool. Dropping this re-opens the v1.10.0 pool- - // exhaustion (P2028) restart loop on a data-heavy tenant. + // shared connection pool. Dropping this re-opens the pool-exhaustion + // (P2028) restart loop on a data-heavy tenant. expect(denseRetentionSource).toMatch( /DENSE_INTRADAY_RETENTION_BOOT_DELAY_SECONDS\s*=\s*\d+/, ); @@ -135,44 +200,52 @@ describe("reminder-worker — dense intra-day retention wiring", () => { ); }); - it("folds the nightly dense-retention walk onto the drain-cumulative tick", () => { + it("folds the nightly dense-retention walk onto the drain-cumulative tick", async () => { // The steady-state nightly walk runs the global (no-userId) pass on the // same tick as the cumulative + mean drains. - expect(source).toMatch(/runDenseIntradayRetention\(\s*getWorkerPrisma\(\)/); + const { boss } = recordingBoss(); + const drain = await jobOf(DRAIN_CUMULATIVE_QUEUE).load({ boss }); + await drain([job(DRAIN_CUMULATIVE_QUEUE, {})]); + expect(runDenseIntradayRetention).toHaveBeenCalledTimes(1); + expect(runDenseIntradayRetention).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ dryRun: false }), + ); }); - it("surfaces the iOS#34 derived-resting count in the per-user worker log", () => { + it("surfaces the iOS#34 derived-resting count in the per-user worker log", async () => { // The PULSE facet mints derived RESTING_HEART_RATE rows for proxy users; // the per-user handler returns the count and the worker logs it so the // signal-preservation is observable in production. - expect(source).toMatch(/derivedRestingRowsUpserted/); + const outcome = await handleDenseIntradayRetention([ + job(DENSE_INTRADAY_RETENTION_QUEUE, { + userId: "u1", + enqueuedAt: "2026-01-01T00:00:00.000Z", + }), + ]); + expect(outcome).toMatchObject({ + did: { derived_resting_rows_upserted: 3 }, + }); + expect(workerLog).toHaveBeenCalledWith( + "info", + expect.stringContaining("derivedRestingRowsUpserted=3"), + ); }); }); -describe("reminder-worker — dense hourly-rebuild wiring", () => { - it("imports the queue symbols from the dense-intraday-hourly-rebuild module", () => { - expect(source).toMatch( - /from\s*["']@\/lib\/jobs\/dense-intraday-hourly-rebuild["']/, - ); - expect(source).toMatch(/\bDENSE_INTRADAY_HOURLY_REBUILD_QUEUE\b/); - expect(source).toMatch(/\brunDenseIntradayHourlyRebuildForUser\b/); - expect(source).toMatch(/\benqueueBootTimeDenseIntradayHourlyRebuild\b/); +describe("dense hourly-rebuild registration", () => { + it("declares the hourly-rebuild queue in the rollup group", () => { + expect(groupOf(DENSE_INTRADAY_HOURLY_REBUILD_QUEUE).name).toBe("rollup"); }); - it("registers the hourly-rebuild queue in allQueues", () => { - expect(allQueuesBlock()).toMatch(/\bDENSE_INTRADAY_HOURLY_REBUILD_QUEUE\b/); - }); - - it("registers a createAndWork handler against the hourly-rebuild queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,250}DENSE_INTRADAY_HOURLY_REBUILD_QUEUE[\s\S]{0,500}runDenseIntradayHourlyRebuildForUser/, + it("binds the hourly-rebuild handler", async () => { + expect(await loadedHandler(DENSE_INTRADAY_HOURLY_REBUILD_QUEUE)).toBe( + handleDenseIntradayHourlyRebuild, ); }); - it("fires the boot-discovery enqueue helper", () => { - expect(source).toMatch( - /await enqueueBootTimeDenseIntradayHourlyRebuild\(\)/, - ); + it("fires the boot-discovery pass", () => { + expect(discoveryOf("rollup")).toContain("dense-intraday-hourly-rebuild"); }); it("defers the boot-discovery rebuild past the startup storm (P2028 guard)", () => { diff --git a/src/lib/jobs/__tests__/subsystem-surface.test.ts b/src/lib/jobs/__tests__/subsystem-surface.test.ts index bb32673db..d416cb006 100644 --- a/src/lib/jobs/__tests__/subsystem-surface.test.ts +++ b/src/lib/jobs/__tests__/subsystem-surface.test.ts @@ -1,92 +1,17 @@ -import { existsSync, readFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +/** + * Every queue the worker registry declares has a failure audience in + * `SUBSYSTEM_SURFACE`, and the table names no queue the registry has dropped. + */ import { describe, expect, it } from "vitest"; +import { allJobDefinitions } from "@/lib/jobs/registry"; import { SUBSYSTEM_SURFACE, type SubsystemSurface, } from "@/lib/jobs/subsystem-surface"; -const REGISTRARS = [ - "register-integration-sync.ts", - "register-status.ts", - "register-rollup.ts", - "register-reminders.ts", - "register-maintenance.ts", -].map((file) => join(process.cwd(), "src/lib/jobs/reminder", file)); - -function withoutComments(source: string): string { - return source.replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/.*$/gm, ""); -} - -export function extractAllQueueBindings(source: string): string[] { - const match = /const\s+allQueues\s*=\s*\[([\s\S]*?)\n\s*\];/.exec(source); - if (!match) throw new Error("Registrar has no allQueues array"); - - return withoutComments(match[1]) - .split(",") - .map((entry) => entry.trim()) - .filter(Boolean) - .map((entry) => { - if (!/^[A-Z][A-Z0-9_]*$/.test(entry)) { - throw new Error(`Unsupported allQueues entry: ${entry}`); - } - return entry; - }); -} - -function literalPattern(binding: string): RegExp { - return new RegExp( - `(?:export\\s+)?const\\s+${binding}(?:\\s*:[^=;]+)?\\s*=\\s*["']([^"']+)["']`, - ); -} - -function modulePath(fromFile: string, specifier: string): string { - const base = specifier.startsWith("@/") - ? resolve(process.cwd(), "src", specifier.slice(2)) - : resolve(dirname(fromFile), specifier); - const candidates = [`${base}.ts`, `${base}.tsx`, join(base, "index.ts")]; - const found = candidates.find((candidate) => existsSync(candidate)); - if (!found) throw new Error(`Cannot resolve ${specifier} from ${fromFile}`); - return found; -} - -function resolveQueueBinding( - binding: string, - file: string, - seen: Set = new Set(), -): string { - const visit = `${file}:${binding}`; - if (seen.has(visit)) throw new Error(`Cyclic queue binding: ${visit}`); - seen.add(visit); - - const source = withoutComments(readFileSync(file, "utf8")); - const local = literalPattern(binding).exec(source); - if (local) return local[1]; - - const imports = /import\s*{([\s\S]*?)}\s*from\s*["']([^"']+)["'];?/g; - for (const match of source.matchAll(imports)) { - for (const rawToken of match[1].split(",")) { - const token = rawToken.trim().replace(/^type\s+/, ""); - if (!token) continue; - const [imported, alias] = token - .split(/\s+as\s+/) - .map((part) => part.trim()); - if ((alias ?? imported) !== binding) continue; - return resolveQueueBinding(imported, modulePath(file, match[2]), seen); - } - } - - throw new Error(`Cannot resolve queue binding ${binding} from ${file}`); -} - function registeredQueueNames(): string[] { - return REGISTRARS.flatMap((file) => { - const source = readFileSync(file, "utf8"); - return extractAllQueueBindings(source).map((binding) => - resolveQueueBinding(binding, file), - ); - }); + return allJobDefinitions().map((job) => job.queue); } export function findSurfaceDrift( @@ -113,8 +38,9 @@ function assertSurfaceComplete( } describe("SUBSYSTEM_SURFACE", () => { - it("classifies every current registrar queue with no stale entries", () => { + it("classifies every current registry queue with no stale entries", () => { const names = registeredQueueNames(); + expect(names.length).toBeGreaterThan(0); expect(new Set(names).size).toBe(names.length); expect(() => assertSurfaceComplete(names, SUBSYSTEM_SURFACE)).not.toThrow(); }); @@ -126,24 +52,9 @@ describe("SUBSYSTEM_SURFACE", () => { } }); - it("catches a new allQueues binding without a surface entry", () => { - const fixture = ` - const EXISTING_QUEUE = "existing-queue"; - const NEW_QUEUE = "new-queue"; - const allQueues = [ - EXISTING_QUEUE, - NEW_QUEUE, - ]; - `; - const names = extractAllQueueBindings(fixture).map((binding) => { - const match = literalPattern(binding).exec(withoutComments(fixture)); - if (!match) - throw new Error(`Fixture binding did not resolve: ${binding}`); - return match[1]; - }); - + it("catches a new registry queue without a surface entry", () => { expect(() => - assertSurfaceComplete(names, { + assertSurfaceComplete(["existing-queue", "new-queue"], { "existing-queue": { audience: "system" }, }), ).toThrow(/missing=\[new-queue\]/); diff --git a/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts b/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts index 9fe67d0b0..544707125 100644 --- a/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts +++ b/src/lib/jobs/__tests__/tls-pin-monitor-queue.test.ts @@ -1,63 +1,56 @@ /** - * v1.12.2 ios-coord — TLS-pin-monitor queue registration guard. + * TLS-pin-monitor queue registration guard. * - * Same source-text-grep approach as the other queue-wiring guards: assert - * the queue is imported, registered in `allQueues`, scheduled, and wired to - * a `createAndWork` handler — without booting pg-boss + Prisma. An unregistered - * queue silently never drains (the recurring v1.4.37 dead-queue bug), which - * would leave the pinned-leaf-rotation alarm dark. + * Reads the worker registry: the queue is declared in the maintenance group, + * scheduled on its cron with the default retries, and bound to the handler + * that runs the monitor pass. An unregistered queue silently never drains + * (the dead-queue class), which would leave the pinned-leaf-rotation alarm + * dark. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the TLS-pin-monitor wiring moved out of the 2143-LOC -// reminder-worker boot file into the maintenance registrar. The dead-queue -// guard follows the wiring there (the ops-handlers module is still concatenated -// for the handler-body `runTlsPinMonitor` assertion). -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-maintenance.ts", +import { handleTlsPinMonitor } from "@/lib/jobs/reminder/ops-handlers"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { + TLS_PIN_MONITOR_CRON, + TLS_PIN_MONITOR_QUEUE, +} from "@/lib/jobs/tls-pin-monitor-queue"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "ops-handlers.ts"), + "utf8", ); -const workerSource = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync(join(__dirname, "..", "reminder", "ops-handlers.ts"), "utf8"); -describe("reminder-worker — tls-pin-monitor wiring", () => { - it("imports the queue symbols from the tls-pin-monitor module", () => { - expect(workerSource).toMatch( - /from\s*["']@\/lib\/jobs\/tls-pin-monitor["']/, - ); - expect(workerSource).toMatch(/\bTLS_PIN_MONITOR_QUEUE\b/); - expect(workerSource).toMatch(/\bTLS_PIN_MONITOR_CRON\b/); - expect(workerSource).toMatch(/\brunTlsPinMonitor\b/); +describe("tls-pin-monitor registration", () => { + it("imports the monitor pass into the ops handlers", () => { + expect(handlerSource).toMatch(/\brunTlsPinMonitor\b/); }); - it("registers the tls-pin-monitor queue in the allQueues loop", () => { - const allQueuesMatch = workerSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueuesMatch).not.toBeNull(); - expect(allQueuesMatch![1]).toMatch(/\bTLS_PIN_MONITOR_QUEUE\b/); + it("declares the tls-pin-monitor queue in the maintenance group", () => { + expect(groupOf(TLS_PIN_MONITOR_QUEUE).name).toBe("maintenance"); }); - it("schedules the tls-pin-monitor cron", () => { - expect(workerSource).toMatch( - /\[TLS_PIN_MONITOR_QUEUE,\s*TLS_PIN_MONITOR_CRON\]/, - ); + it("schedules the tls-pin-monitor cron with the default retries", () => { + const job = jobOf(TLS_PIN_MONITOR_QUEUE); + expect(job.cron).toBe(TLS_PIN_MONITOR_CRON); + expect(job.scheduleOptions).toBeUndefined(); }); - it("registers a createAndWork handler for the queue", () => { - expect(workerSource).toMatch( - /createAndWork[\s\S]{0,200}TLS_PIN_MONITOR_QUEUE[\s\S]{0,200}handleTlsPinMonitor/, + it("binds the monitor handler", async () => { + expect(await loadedHandler(TLS_PIN_MONITOR_QUEUE)).toBe( + handleTlsPinMonitor, ); }); it("runs the monitor pass inside the handler", () => { - expect(workerSource).toMatch( + expect(handlerSource).toMatch( /handleTlsPinMonitor[\s\S]{0,400}runTlsPinMonitor/, ); }); diff --git a/src/lib/jobs/__tests__/whoop-queues.test.ts b/src/lib/jobs/__tests__/whoop-queues.test.ts index a1ad468b4..cdea7c9f2 100644 --- a/src/lib/jobs/__tests__/whoop-queues.test.ts +++ b/src/lib/jobs/__tests__/whoop-queues.test.ts @@ -1,113 +1,126 @@ /** - * v1.11.0 — pg-boss queue registration for the WHOOP sync layer. + * pg-boss queue registration for the WHOOP sync layer. * - * The reminder-worker imports heavy infrastructure (pg-boss, Prisma adapter, - * notification dispatchers, …) so we don't boot the whole worker here. Instead - * we read the source as text and assert the queue constants + cron schedules + - * handler wiring + allQueues + boot-enqueue are present — the fast guard that - * catches the v1.4.37 dead-queue class (a queue declared but never registered - * in `allQueues`, which silently never drains). + * Reads the worker registry instead of booting the worker: every WHOOP queue + * is declared in the integration-sync group with its cron and handler, the + * backfill forwards to the shared admission lane, and the boot discovery + * offers it again. This is the fast guard for the dead-queue class (a queue + * declared but never registered, which silently never drains). */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring + boot discovery moved out of the 2143-LOC -// reminder-worker boot file into domain registrars; the WHOOP queues live in -// the integration-sync registrar. The dead-queue guard follows the wiring there -// (the handler module is still concatenated for the import-routine assertions). -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-integration-sync.ts", -); -const source = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync(join(__dirname, "..", "reminder", "whoop-sync.ts"), "utf8"); +import { INTEGRATION_BACKFILL_ADMISSION_QUEUE } from "@/lib/jobs/integration-backfill-admission"; +import { + WHOOP_CYCLE_SYNC_CRON, + WHOOP_CYCLE_SYNC_QUEUE, + WHOOP_OAUTH_STATE_CLEANUP_CRON, + WHOOP_OAUTH_STATE_CLEANUP_QUEUE, + WHOOP_RECOVERY_SYNC_CRON, + WHOOP_RECOVERY_SYNC_QUEUE, + WHOOP_SLEEP_SYNC_CRON, + WHOOP_SLEEP_SYNC_QUEUE, + WHOOP_WORKOUT_SYNC_CRON, + WHOOP_WORKOUT_SYNC_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + handleWhoopCycleSync, + handleWhoopRecoverySync, + handleWhoopSleepSync, + handleWhoopWorkoutSync, +} from "@/lib/jobs/reminder/whoop-sync"; +import { + discoveryOf, + groupOf, + jobOf, + loadedHandler, + recordingBoss, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { WHOOP_BACKFILL_QUEUE } from "@/lib/jobs/whoop-backfill-queue"; -const WHOOP_QUEUE_CONSTS = [ - "WHOOP_RECOVERY_SYNC_QUEUE", - "WHOOP_SLEEP_SYNC_QUEUE", - "WHOOP_WORKOUT_SYNC_QUEUE", - "WHOOP_CYCLE_SYNC_QUEUE", - "WHOOP_BACKFILL_QUEUE", - "WHOOP_OAUTH_STATE_CLEANUP_QUEUE", +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "whoop-sync.ts"), + "utf8", +); + +const WHOOP_QUEUES = [ + WHOOP_RECOVERY_SYNC_QUEUE, + WHOOP_SLEEP_SYNC_QUEUE, + WHOOP_WORKOUT_SYNC_QUEUE, + WHOOP_CYCLE_SYNC_QUEUE, + WHOOP_BACKFILL_QUEUE, + WHOOP_OAUTH_STATE_CLEANUP_QUEUE, ] as const; -describe("reminder-worker — WHOOP sync queues", () => { +describe("WHOOP sync queues", () => { it("declares every WHOOP sync queue constant", () => { - expect(source).toMatch( - /WHOOP_RECOVERY_SYNC_QUEUE\s*=\s*["']whoop-recovery-sync["']/, - ); - expect(source).toMatch( - /WHOOP_SLEEP_SYNC_QUEUE\s*=\s*["']whoop-sleep-sync["']/, - ); - expect(source).toMatch( - /WHOOP_WORKOUT_SYNC_QUEUE\s*=\s*["']whoop-workout-sync["']/, - ); - expect(source).toMatch( - /WHOOP_CYCLE_SYNC_QUEUE\s*=\s*["']whoop-cycle-sync["']/, - ); + expect(WHOOP_RECOVERY_SYNC_QUEUE).toBe("whoop-recovery-sync"); + expect(WHOOP_SLEEP_SYNC_QUEUE).toBe("whoop-sleep-sync"); + expect(WHOOP_WORKOUT_SYNC_QUEUE).toBe("whoop-workout-sync"); + expect(WHOOP_CYCLE_SYNC_QUEUE).toBe("whoop-cycle-sync"); }); - it("registers EVERY WHOOP queue in allQueues (v1.4.37 dead-queue guard)", () => { - // Isolate the allQueues array body so the assertion can't be satisfied by - // an incidental mention elsewhere in the file. - const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); - expect(block).not.toBeNull(); - const body = block![1]!; - for (const q of WHOOP_QUEUE_CONSTS) { - expect(body).toContain(q); + it("declares EVERY WHOOP queue in the integration-sync group", () => { + for (const q of WHOOP_QUEUES) { + expect(groupOf(q).name).toBe("integration_sync"); } }); it("schedules the four poll crons + the daily oauth-state sweep", () => { - expect(source).toMatch( - /\[WHOOP_RECOVERY_SYNC_QUEUE,\s*WHOOP_RECOVERY_SYNC_CRON\]/, - ); - expect(source).toMatch( - /\[WHOOP_SLEEP_SYNC_QUEUE,\s*WHOOP_SLEEP_SYNC_CRON\]/, - ); - expect(source).toMatch( - /\[WHOOP_WORKOUT_SYNC_QUEUE,\s*WHOOP_WORKOUT_SYNC_CRON\]/, - ); - expect(source).toMatch( - /\[WHOOP_CYCLE_SYNC_QUEUE,\s*WHOOP_CYCLE_SYNC_CRON\]/, - ); - expect(source).toMatch( - /\[\s*WHOOP_OAUTH_STATE_CLEANUP_QUEUE,\s*WHOOP_OAUTH_STATE_CLEANUP_CRON,\s*cronIsTheRetry,?\s*\]/, + expect(jobOf(WHOOP_RECOVERY_SYNC_QUEUE).cron).toBe( + WHOOP_RECOVERY_SYNC_CRON, ); + expect(jobOf(WHOOP_SLEEP_SYNC_QUEUE).cron).toBe(WHOOP_SLEEP_SYNC_CRON); + expect(jobOf(WHOOP_WORKOUT_SYNC_QUEUE).cron).toBe(WHOOP_WORKOUT_SYNC_CRON); + expect(jobOf(WHOOP_CYCLE_SYNC_QUEUE).cron).toBe(WHOOP_CYCLE_SYNC_CRON); + const sweep = jobOf(WHOOP_OAUTH_STATE_CLEANUP_QUEUE); + expect(sweep.cron).toBe(WHOOP_OAUTH_STATE_CLEANUP_CRON); + expect(sweep.scheduleOptions).toBe(cronIsTheRetry); }); - it("registers a createAndWork handler for every WHOOP sync queue", () => { - expect(source).toMatch( - /createAndWork[\s\S]{0,160}WHOOP_RECOVERY_SYNC_QUEUE[\s\S]{0,160}handleWhoopRecoverySync/, + it("binds a handler for every WHOOP sync queue", async () => { + expect(await loadedHandler(WHOOP_RECOVERY_SYNC_QUEUE)).toBe( + handleWhoopRecoverySync, ); - expect(source).toMatch( - /createAndWork[\s\S]{0,160}WHOOP_SLEEP_SYNC_QUEUE[\s\S]{0,160}handleWhoopSleepSync/, + expect(await loadedHandler(WHOOP_SLEEP_SYNC_QUEUE)).toBe( + handleWhoopSleepSync, ); - expect(source).toMatch( - /createAndWork[\s\S]{0,160}WHOOP_WORKOUT_SYNC_QUEUE[\s\S]{0,160}handleWhoopWorkoutSync/, + expect(await loadedHandler(WHOOP_WORKOUT_SYNC_QUEUE)).toBe( + handleWhoopWorkoutSync, ); - expect(source).toMatch( - /createAndWork[\s\S]{0,160}WHOOP_CYCLE_SYNC_QUEUE[\s\S]{0,160}handleWhoopCycleSync/, - ); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}WHOOP_BACKFILL_QUEUE[\s\S]{0,260}enqueueIntegrationBackfillAdmission/, + expect(await loadedHandler(WHOOP_CYCLE_SYNC_QUEUE)).toBe( + handleWhoopCycleSync, ); + const { boss, sends } = recordingBoss(); + const backfill = await jobOf(WHOOP_BACKFILL_QUEUE).load({ boss }); + await backfill([ + { + id: "j1", + name: WHOOP_BACKFILL_QUEUE, + data: { userId: "u1" }, + expireInSeconds: 60, + heartbeatSeconds: null, + retryCount: 0, + signal: new AbortController().signal, + }, + ]); + expect(sends).toHaveLength(1); + expect(sends[0]!.queue).toBe(INTEGRATION_BACKFILL_ADMISSION_QUEUE); + expect(sends[0]!.data).toMatchObject({ kind: "whoop-backfill" }); }); it("wires the self-converging WHOOP backfill boot discovery", () => { - expect(source).toMatch( - /enqueueBootTimeWhoopBackfill\(\s*bootStaggerSecondsFor\("whoop-backfill"\)/, - ); + expect(discoveryOf("integration_sync")).toContain("whoop-backfill"); }); it("imports the WHOOP per-resource sync routines", () => { - expect(source).toMatch(/from\s*["']@\/lib\/whoop\/sync-recovery["']/); - expect(source).toMatch(/from\s*["']@\/lib\/whoop\/sync-cycle["']/); - expect(source).toMatch(/from\s*["']@\/lib\/whoop\/sync-workout["']/); + expect(handlerSource).toMatch( + /from\s*["']@\/lib\/whoop\/sync-recovery["']/, + ); + expect(handlerSource).toMatch(/from\s*["']@\/lib\/whoop\/sync-cycle["']/); + expect(handlerSource).toMatch(/from\s*["']@\/lib\/whoop\/sync-workout["']/); }); }); diff --git a/src/lib/jobs/__tests__/withings-ecg-worker.test.ts b/src/lib/jobs/__tests__/withings-ecg-worker.test.ts index 457a24121..73a2358bd 100644 --- a/src/lib/jobs/__tests__/withings-ecg-worker.test.ts +++ b/src/lib/jobs/__tests__/withings-ecg-worker.test.ts @@ -1,7 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { Job } from "pg-boss"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; const { syncUserEcg } = vi.hoisted(() => ({ syncUserEcg: vi.fn< @@ -28,7 +26,8 @@ vi.mock("@/lib/withings/sync-activity", () => ({ vi.mock("@/lib/withings/sync-sleep", () => ({ syncUserSleep: vi.fn(async () => 0), })); -vi.mock("@/lib/jobs/reminder-satisfy", () => ({ +vi.mock("@/lib/jobs/reminder-satisfy", async (importOriginal) => ({ + ...(await importOriginal()), enqueueReminderSatisfy: vi.fn(async () => undefined), })); vi.mock("@/lib/jobs/worker-status", () => ({ @@ -72,6 +71,13 @@ import { handleWithingsEcgSync, type WithingsEcgSyncPayload, } from "../reminder/withings-sync"; +import { WITHINGS_ECG_SYNC_CRON } from "../integration-poll-queues"; +import { + groupOf, + jobOf, + loadedHandler, +} from "../registry/__tests__/registry-helpers"; +import { WITHINGS_ECG_SYNC_QUEUE } from "../withings-ecg-queue"; function job(data: WithingsEcgSyncPayload): Job { return { data } as Job; @@ -82,10 +88,6 @@ beforeEach(() => { ecgFindMany.mockResolvedValue([]); isReauthRequired.mockResolvedValue(false); }); -const registrarSource = readFileSync( - join(process.cwd(), "src/lib/jobs/reminder/register-integration-sync.ts"), - "utf8", -); describe("handleWithingsEcgSync", () => { it("a webhook job (userId + window) does a targeted date-windowed sync", async () => { @@ -176,17 +178,13 @@ describe("handleWithingsEcgSync", () => { }); describe("withings ECG queue registration", () => { - it("deduplicates only queued replays and admits a rescue after work starts", () => { - const allQueues = registrarSource.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bWITHINGS_ECG_SYNC_QUEUE\b/); - expect(registrarSource).toMatch( - /\[WITHINGS_ECG_SYNC_QUEUE\]:\s*\{[\s\S]*?policy:\s*"short"/, - ); - expect(registrarSource).toMatch( - /createAndWork\(\s*boss,\s*WITHINGS_ECG_SYNC_QUEUE[\s\S]{0,120}handleWithingsEcgSync/, + it("deduplicates only queued replays and admits a rescue after work starts", async () => { + expect(groupOf(WITHINGS_ECG_SYNC_QUEUE).name).toBe("integration_sync"); + const definition = jobOf(WITHINGS_ECG_SYNC_QUEUE); + expect(definition.policy?.policy).toBe("short"); + expect(definition.cron).toBe(WITHINGS_ECG_SYNC_CRON); + expect(await loadedHandler(WITHINGS_ECG_SYNC_QUEUE)).toBe( + handleWithingsEcgSync, ); }); }); diff --git a/src/lib/jobs/__tests__/withings-queues.test.ts b/src/lib/jobs/__tests__/withings-queues.test.ts index 8a5bd52ec..0cd5a286b 100644 --- a/src/lib/jobs/__tests__/withings-queues.test.ts +++ b/src/lib/jobs/__tests__/withings-queues.test.ts @@ -1,90 +1,83 @@ /** - * v1.4.25 — pg-boss queue registration for Withings activity + - * sleep v2 syncs. + * pg-boss queue registration for Withings activity + sleep v2 syncs. * - * The reminder-worker imports heavy infrastructure (pg-boss, Prisma - * adapter, notification dispatchers, …) so we don't boot the whole - * worker in this test. Instead we read the source file as text and - * assert the queue constants + cron schedules + handler wiring are - * present — a fast regression guard that catches accidental deletion - * of the W17b/c plumbing without spinning up Postgres. + * Reads the worker registry instead of booting the worker (pg-boss, Prisma + * adapter, notification dispatchers, …): the queues are declared with their + * crons and bound to their handlers, a fast regression guard against + * accidental deletion of the plumbing without spinning up Postgres. The + * import-routine assertions read the handler module. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -// v1.18.1 — the queue wiring moved out of the 2143-LOC reminder-worker boot -// file into domain registrars; the Withings queues live in the integration-sync -// registrar. The dead-queue guard follows the wiring there (the handler module -// is still concatenated for the import-routine assertions below). -const REGISTRAR_PATH = join( - __dirname, - "..", - "reminder", - "register-integration-sync.ts", +import { + WITHINGS_ACTIVITY_CRON, + WITHINGS_ACTIVITY_QUEUE, + WITHINGS_ECG_SYNC_CRON, + WITHINGS_SLEEP_CRON, + WITHINGS_SLEEP_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; +import { + handleWithingsActivitySync, + handleWithingsSleepSync, +} from "@/lib/jobs/reminder/withings-sync"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; +import { WITHINGS_ECG_SYNC_QUEUE } from "@/lib/jobs/withings-ecg-queue"; + +const handlerSource = readFileSync( + join(__dirname, "..", "reminder", "withings-sync.ts"), + "utf8", ); -const source = - readFileSync(REGISTRAR_PATH, "utf8") + - readFileSync(join(__dirname, "..", "reminder", "withings-sync.ts"), "utf8"); - -describe("reminder-worker — Withings activity + sleep v2 queues", () => { +describe("Withings activity + sleep v2 queues", () => { it("declares the activity-sync queue at the documented cadence", () => { - expect(source).toMatch( - /WITHINGS_ACTIVITY_QUEUE\s*=\s*["']withings-activity-sync["']/, - ); - expect(source).toMatch( - /WITHINGS_ACTIVITY_CRON\s*=\s*["']0 \* \* \* \*["']/, - ); + expect(WITHINGS_ACTIVITY_QUEUE).toBe("withings-activity-sync"); + expect(WITHINGS_ACTIVITY_CRON).toBe("0 * * * *"); }); it("declares the sleep-sync queue at the offset cadence", () => { - expect(source).toMatch( - /WITHINGS_SLEEP_QUEUE\s*=\s*["']withings-sleep-sync["']/, - ); + expect(WITHINGS_SLEEP_QUEUE).toBe("withings-sleep-sync"); // :15 offset keeps it out of lockstep with the measure cron at :00. - expect(source).toMatch(/WITHINGS_SLEEP_CRON\s*=\s*["']15 \* \* \* \*["']/); + expect(WITHINGS_SLEEP_CRON).toBe("15 * * * *"); }); - it("registers the activity-sync handler against pg-boss", () => { - expect(source).toMatch(/handleWithingsActivitySync\b/); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}WITHINGS_ACTIVITY_QUEUE[\s\S]{0,200}handleWithingsActivitySync/, + it("binds the activity-sync handler", async () => { + expect(groupOf(WITHINGS_ACTIVITY_QUEUE).name).toBe("integration_sync"); + expect(await loadedHandler(WITHINGS_ACTIVITY_QUEUE)).toBe( + handleWithingsActivitySync, ); }); - it("registers the sleep-sync handler against pg-boss", () => { - expect(source).toMatch(/handleWithingsSleepSync\b/); - expect(source).toMatch( - /createAndWork[\s\S]{0,200}WITHINGS_SLEEP_QUEUE[\s\S]{0,200}handleWithingsSleepSync/, + it("binds the sleep-sync handler", async () => { + expect(groupOf(WITHINGS_SLEEP_QUEUE).name).toBe("integration_sync"); + expect(await loadedHandler(WITHINGS_SLEEP_QUEUE)).toBe( + handleWithingsSleepSync, ); }); - it("imports the new activity / sleep sync routines", () => { - expect(source).toMatch( + it("imports the activity / sleep sync routines", () => { + expect(handlerSource).toMatch( /import\s*\{\s*syncUserActivity\s*\}\s*from\s*["']@\/lib\/withings\/sync-activity["']/, ); - expect(source).toMatch( + expect(handlerSource).toMatch( /import\s*\{\s*syncUserSleep\s*\}\s*from\s*["']@\/lib\/withings\/sync-sleep["']/, ); }); - it("schedules both crons via boss.schedule (allQueues + schedules tables)", () => { - expect(source).toMatch( - /\[WITHINGS_ACTIVITY_QUEUE,\s*WITHINGS_ACTIVITY_CRON\]/, - ); - expect(source).toMatch(/\[WITHINGS_SLEEP_QUEUE,\s*WITHINGS_SLEEP_CRON\]/); + it("schedules both crons", () => { + expect(jobOf(WITHINGS_ACTIVITY_QUEUE).cron).toBe(WITHINGS_ACTIVITY_CRON); + expect(jobOf(WITHINGS_SLEEP_QUEUE).cron).toBe(WITHINGS_SLEEP_CRON); }); it("schedules the ECG catch-net cron at :41 so watch-only accounts pull strips", () => { - // The queue is already in allQueues + createAndWork-bound; W-1 adds the missing - // schedule entry so a payload-less cron tick walks the connection cohort. - expect(source).toMatch( - /WITHINGS_ECG_SYNC_CRON\s*=\s*["']41 \* \* \* \*["']/, - ); - expect(source).toMatch( - /\[WITHINGS_ECG_SYNC_QUEUE,\s*WITHINGS_ECG_SYNC_CRON\]/, - ); + // A payload-less cron tick walks the connection cohort. + expect(WITHINGS_ECG_SYNC_CRON).toBe("41 * * * *"); + expect(jobOf(WITHINGS_ECG_SYNC_QUEUE).cron).toBe(WITHINGS_ECG_SYNC_CRON); }); }); diff --git a/src/lib/jobs/__tests__/workout-insight-queue.test.ts b/src/lib/jobs/__tests__/workout-insight-queue.test.ts index 7af32f112..ce17b76df 100644 --- a/src/lib/jobs/__tests__/workout-insight-queue.test.ts +++ b/src/lib/jobs/__tests__/workout-insight-queue.test.ts @@ -1,40 +1,41 @@ /** - * v1.4.37 dead-queue guard for the `workout-insight-generate` queue. + * Dead-queue guard for the `workout-insight-generate` queue. * * This queue fails open in the worst possible way. Nothing warms it, no cron * touches it, and no read path triggers it — the ONLY thing that ever puts work - * on it is the arrival spine's dispatch. So if the queue name is missing from - * `allQueues`, pg-boss never provisions it, every `boss.send` resolves + * on it is the arrival spine's dispatch. So if the registry does not declare + * the queue, pg-boss never provisions it, every `boss.send` resolves * successfully, every job vanishes, and the only symptom is that no workout * ever gets a paragraph. That is indistinguishable from the feature's own * honest empty state, which is exactly why it needs a structural guard rather * than a behavioural one. * - * Source-text assertions over the registrar, matching the shape of the sibling - * `*-queue.test.ts` guards. + * Reads the worker registry for the queue facts. */ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -const statusRegistrar = readFileSync( - join(__dirname, "..", "reminder", "register-status.ts"), - "utf8", -); +import { handleWorkoutInsightGenerate } from "@/lib/jobs/workout-insight-generate"; +import { + WORKOUT_INSIGHT_GENERATE_CONCURRENCY, + WORKOUT_INSIGHT_GENERATE_QUEUE, +} from "@/lib/jobs/workout-insight-generate-shared"; +import { + groupOf, + jobOf, + loadedHandler, +} from "@/lib/jobs/registry/__tests__/registry-helpers"; describe("workout-insight-generate queue wiring", () => { - it("provisions the queue in the allQueues list", () => { - const allQueues = statusRegistrar.match( - /const allQueues\s*=\s*\[([\s\S]*?)\];/, - ); - expect(allQueues).not.toBeNull(); - expect(allQueues![1]).toMatch(/\bWORKOUT_INSIGHT_GENERATE_QUEUE\b/); + it("provisions the queue in the status group", () => { + expect(groupOf(WORKOUT_INSIGHT_GENERATE_QUEUE).name).toBe("status"); }); - it("binds a createAndWork handler that drains it", () => { - expect(statusRegistrar).toMatch( - /createAndWork[\s\S]{0,300}WORKOUT_INSIGHT_GENERATE_QUEUE[\s\S]{0,300}handleWorkoutInsightGenerate/, + it("binds the handler that drains it", async () => { + expect(await loadedHandler(WORKOUT_INSIGHT_GENERATE_QUEUE)).toBe( + handleWorkoutInsightGenerate, ); }); @@ -42,19 +43,17 @@ describe("workout-insight-generate queue wiring", () => { // This IS the provider path the spine deliberately does not walk. Widening // the concurrency would let one sync's worth of workouts fan out into // parallel completions. - expect(statusRegistrar).toMatch( - /WORKOUT_INSIGHT_GENERATE_QUEUE[\s\S]{0,300}localConcurrency:\s*WORKOUT_INSIGHT_GENERATE_CONCURRENCY/, - ); + expect(jobOf(WORKOUT_INSIGHT_GENERATE_QUEUE).work).toEqual({ + localConcurrency: WORKOUT_INSIGHT_GENERATE_CONCURRENCY, + }); }); it("has no cron schedule — it is dispatched by arrival only", () => { // A cron here would be a standing invitation to regenerate, which is the // saturation surface the design refuses. - // Tolerates the optional third schedule element, so a cron sneaking in - // with send options still trips this. - expect(statusRegistrar).not.toMatch( - /\[\s*WORKOUT_INSIGHT_GENERATE_QUEUE,\s*\w+_CRON\s*[,\]]/, - ); + const job = jobOf(WORKOUT_INSIGHT_GENERATE_QUEUE); + expect(job.cron).toBeUndefined(); + expect(job.scheduleOptions).toBeUndefined(); }); it("is reached from the spine only through the generator-free module", () => { diff --git a/src/lib/jobs/achievement-unlock-sweep-queue.ts b/src/lib/jobs/achievement-unlock-sweep-queue.ts new file mode 100644 index 000000000..4b64470b9 --- /dev/null +++ b/src/lib/jobs/achievement-unlock-sweep-queue.ts @@ -0,0 +1,8 @@ +/** + * The queue facts of `achievement-unlock-sweep.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const ACHIEVEMENT_UNLOCK_SWEEP_QUEUE = "achievement-unlock-sweep"; + +export const ACHIEVEMENT_UNLOCK_SWEEP_CRON = "25 4 * * *"; diff --git a/src/lib/jobs/achievement-unlock-sweep.ts b/src/lib/jobs/achievement-unlock-sweep.ts index 0a4a12685..190a17189 100644 --- a/src/lib/jobs/achievement-unlock-sweep.ts +++ b/src/lib/jobs/achievement-unlock-sweep.ts @@ -31,10 +31,6 @@ import { resolveModuleMap } from "@/lib/modules/gate"; import { buildAchievementsResult } from "@/lib/gamification/achievements-result"; import { getWorkerPrisma } from "./reminder/shared"; -export const ACHIEVEMENT_UNLOCK_SWEEP_QUEUE = "achievement-unlock-sweep"; - -export const ACHIEVEMENT_UNLOCK_SWEEP_CRON = "25 4 * * *"; - export interface AchievementUnlockSweepPayload { triggeredAt: string; } diff --git a/src/lib/jobs/apple-health-import-queue.ts b/src/lib/jobs/apple-health-import-queue.ts new file mode 100644 index 000000000..e862c9b7d --- /dev/null +++ b/src/lib/jobs/apple-health-import-queue.ts @@ -0,0 +1,36 @@ +/** + * The queue facts of `apple-health-import-worker.ts`: its queue names and + * concurrency. They live apart from the worker so the registry and the upload + * routes can name the queue without loading the parser. + */ +/** + * Queue + cron for the periodic orphan-ImportJob sweep. v1.32.1 + * (issue #588) — `reconcileOrphanImportJobs()` used to run only once, + * at worker boot. `restart: unless-stopped` in `docker-compose.yml` + * DOES bring a crashed/OOM-killed worker back up, so the boot-time + * pass usually catches a dead run — but only when that restart lands + * AFTER the row's heartbeat has already gone stale (30 minutes) or + * pg-boss no longer reports the backing job as live. A fast restart + * lands neither condition true yet, and because the pass runs + * exactly once per boot, a worker that then stays up and healthy + * NEVER re-evaluates that row — it is stuck in `unpacking` / + * `parsing` / `upserting` with "0 rows imported" forever and no + * failure ever surfaced to the status poll. A 15-minute cron closes + * that gap: within two ticks of the heartbeat going stale, the same + * idempotent reconcile flips the row to `failed` with an honest + * reason, so the UI's "Unpacking the archive…" spinner eventually + * turns into a visible, actionable error instead of an indefinite + * silent stall. + */ +export const IMPORT_JOB_RECONCILE_QUEUE = "apple-health-import-reconcile"; + +export const IMPORT_JOB_RECONCILE_CRON = "*/15 * * * *"; + +/** Queue name isolated from revision-1 workers during rolling deployments. */ +export const APPLE_HEALTH_IMPORT_V2_QUEUE = "apple-health-import-v2"; + +/** Queue used by pre-revision-2 binaries; new workers only bridge its backlog. */ +export const APPLE_HEALTH_IMPORT_LEGACY_QUEUE = "apple-health-import"; + +/** Concurrency cap per worker host. */ +export const APPLE_HEALTH_IMPORT_CONCURRENCY = 1; diff --git a/src/lib/jobs/apple-health-import-worker.ts b/src/lib/jobs/apple-health-import-worker.ts index f2cbfe0d9..01a3e3649 100644 --- a/src/lib/jobs/apple-health-import-worker.ts +++ b/src/lib/jobs/apple-health-import-worker.ts @@ -55,10 +55,12 @@ import { type StagingInUse, } from "@/lib/import/apple-health-staging"; import { caughtAs, logCaught } from "@/lib/logging/signal"; +import { HEALTH_CONNECT_IMPORT_KIND } from "@/lib/jobs/health-connect-import-worker"; +import { HEALTH_CONNECT_IMPORT_QUEUE } from "@/lib/jobs/health-connect-import-queue"; import { - HEALTH_CONNECT_IMPORT_KIND, - HEALTH_CONNECT_IMPORT_QUEUE, -} from "@/lib/jobs/health-connect-import-worker"; + APPLE_HEALTH_IMPORT_V2_QUEUE, + APPLE_HEALTH_IMPORT_LEGACY_QUEUE, +} from "./apple-health-import-queue"; /** `ImportJob.kind` of the rows this worker owns. */ export const APPLE_HEALTH_IMPORT_KIND = "apple_health"; @@ -76,28 +78,6 @@ function queuesForKind(kind: string): readonly string[] { : [APPLE_HEALTH_IMPORT_V2_QUEUE, APPLE_HEALTH_IMPORT_LEGACY_QUEUE]; } -/** - * Queue + cron for the periodic orphan-ImportJob sweep. v1.32.1 - * (issue #588) — `reconcileOrphanImportJobs()` used to run only once, - * at worker boot. `restart: unless-stopped` in `docker-compose.yml` - * DOES bring a crashed/OOM-killed worker back up, so the boot-time - * pass usually catches a dead run — but only when that restart lands - * AFTER the row's heartbeat has already gone stale (30 minutes) or - * pg-boss no longer reports the backing job as live. A fast restart - * lands neither condition true yet, and because the pass runs - * exactly once per boot, a worker that then stays up and healthy - * NEVER re-evaluates that row — it is stuck in `unpacking` / - * `parsing` / `upserting` with "0 rows imported" forever and no - * failure ever surfaced to the status poll. A 15-minute cron closes - * that gap: within two ticks of the heartbeat going stale, the same - * idempotent reconcile flips the row to `failed` with an honest - * reason, so the UI's "Unpacking the archive…" spinner eventually - * turns into a visible, actionable error instead of an indefinite - * silent stall. - */ -export const IMPORT_JOB_RECONCILE_QUEUE = "apple-health-import-reconcile"; -export const IMPORT_JOB_RECONCILE_CRON = "*/15 * * * *"; - /** * Cron handler for the periodic orphan-ImportJob sweep. Delegates to * the same `reconcileOrphanImportJobs()` the boot path calls — the @@ -145,12 +125,6 @@ export async function handleImportJobReconcileTick( ); } -/** Queue name isolated from revision-1 workers during rolling deployments. */ -export const APPLE_HEALTH_IMPORT_V2_QUEUE = "apple-health-import-v2"; - -/** Queue used by pre-revision-2 binaries; new workers only bridge its backlog. */ -export const APPLE_HEALTH_IMPORT_LEGACY_QUEUE = "apple-health-import"; - /** Parser semantics carried by every newly-created ImportJob. */ export const APPLE_HEALTH_IMPORT_PARSER_REVISION = 3; @@ -163,9 +137,6 @@ const APPLE_HEALTH_ECG_ARCHIVE_LIMITS = { } as const; const APPLE_HEALTH_ECG_MAX_SAMPLES = 2_000_000; -/** Concurrency cap per worker host. */ -export const APPLE_HEALTH_IMPORT_CONCURRENCY = 1; - /** * (issue #775) Memory preflight for the parse. The streaming pipeline * keeps per-record work off the heap, so what remains resident scales diff --git a/src/lib/jobs/backup-restore-queue.ts b/src/lib/jobs/backup-restore-queue.ts new file mode 100644 index 000000000..7993df548 --- /dev/null +++ b/src/lib/jobs/backup-restore-queue.ts @@ -0,0 +1,6 @@ +/** + * The queue facts of `backup-restore.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const BACKUP_RESTORE_QUEUE = "backup-restore"; diff --git a/src/lib/jobs/backup-restore.ts b/src/lib/jobs/backup-restore.ts index e17aa8865..ab721743e 100644 --- a/src/lib/jobs/backup-restore.ts +++ b/src/lib/jobs/backup-restore.ts @@ -78,8 +78,7 @@ import { type RestoreFailureCause, } from "@/lib/export/restore-failure-cause"; import { logCaught } from "@/lib/logging/signal"; - -export const BACKUP_RESTORE_QUEUE = "backup-restore"; +import { BACKUP_RESTORE_QUEUE } from "./backup-restore-queue"; /** Stable code of the 409 a second restore of the same account gets. */ export const BACKUP_RESTORE_ACTIVE_CODE = "backup.restore.active"; diff --git a/src/lib/jobs/coach-nudge-queue.ts b/src/lib/jobs/coach-nudge-queue.ts new file mode 100644 index 000000000..bef4d8ac7 --- /dev/null +++ b/src/lib/jobs/coach-nudge-queue.ts @@ -0,0 +1,10 @@ +/** + * The queue facts of `coach-nudge.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +/** Pg-boss queue + cron — imported by the reminder worker's bootstrap. */ +export const COACH_NUDGE_QUEUE = "coach-nudge"; + +/** 05:15 Europe/Berlin — after the 04:45–04:55 score crons settled. */ +export const COACH_NUDGE_CRON = "15 5 * * *"; diff --git a/src/lib/jobs/coach-nudge.ts b/src/lib/jobs/coach-nudge.ts index f8fb77cc5..b092ce85e 100644 --- a/src/lib/jobs/coach-nudge.ts +++ b/src/lib/jobs/coach-nudge.ts @@ -121,11 +121,6 @@ const SELF_CONTEXT_AI_SECTIONS = [ "COACH_FOCUS", ] as const satisfies readonly HealthProfileAiSection[]; -/** Pg-boss queue + cron — imported by the reminder worker's bootstrap. */ -export const COACH_NUDGE_QUEUE = "coach-nudge"; -/** 05:15 Europe/Berlin — after the 04:45–04:55 score crons settled. */ -export const COACH_NUDGE_CRON = "15 5 * * *"; - /** * Frequency cap default: one nudge per user per rolling week. v1.16.5 — * the per-user `nudgeFrequency` pref ("weekly" | "biweekly") can widen diff --git a/src/lib/jobs/coach-reminder-sweep-queue.ts b/src/lib/jobs/coach-reminder-sweep-queue.ts new file mode 100644 index 000000000..de3a9240e --- /dev/null +++ b/src/lib/jobs/coach-reminder-sweep-queue.ts @@ -0,0 +1,10 @@ +/** + * The queue facts of `coach-reminder-sweep.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const COACH_REMINDER_SWEEP_QUEUE = "coach-reminder-sweep"; + +// Daily at 05:20 Europe/Berlin — just after the 05:15 nudge tick so both +// proactive passes run on settled overnight rows. +export const COACH_REMINDER_SWEEP_CRON = "20 5 * * *"; diff --git a/src/lib/jobs/coach-reminder-sweep.ts b/src/lib/jobs/coach-reminder-sweep.ts index dab2112eb..fb47ce4b5 100644 --- a/src/lib/jobs/coach-reminder-sweep.ts +++ b/src/lib/jobs/coach-reminder-sweep.ts @@ -71,11 +71,6 @@ import { surfaceCoachReminders, } from "@/lib/ai/coach/reminder-surface"; -export const COACH_REMINDER_SWEEP_QUEUE = "coach-reminder-sweep"; -// Daily at 05:20 Europe/Berlin — just after the 05:15 nudge tick so both -// proactive passes run on settled overnight rows. -export const COACH_REMINDER_SWEEP_CRON = "20 5 * * *"; - /** Bound the plan-review fan-out per tick so one sweep can't run unbounded. */ const PLAN_REVIEW_BATCH = 200; /** Bound the surfacing fan-out per tick for the same reason. */ diff --git a/src/lib/jobs/compaction-tombstone-purge-queue.ts b/src/lib/jobs/compaction-tombstone-purge-queue.ts new file mode 100644 index 000000000..82b7b775e --- /dev/null +++ b/src/lib/jobs/compaction-tombstone-purge-queue.ts @@ -0,0 +1,6 @@ +/** + * The queue facts of `compaction-tombstone-purge.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const COMPACTION_TOMBSTONE_PURGE_QUEUE = "compaction-tombstone-purge"; diff --git a/src/lib/jobs/compaction-tombstone-purge.ts b/src/lib/jobs/compaction-tombstone-purge.ts index 415eee0dc..a43c080da 100644 --- a/src/lib/jobs/compaction-tombstone-purge.ts +++ b/src/lib/jobs/compaction-tombstone-purge.ts @@ -67,8 +67,7 @@ import { findCompactionTombstones, } from "@/lib/measurements/folded-window"; import { prisma as defaultPrisma } from "@/lib/db"; - -export const COMPACTION_TOMBSTONE_PURGE_QUEUE = "compaction-tombstone-purge"; +import { COMPACTION_TOMBSTONE_PURGE_QUEUE } from "./compaction-tombstone-purge-queue"; /** One key for every send, so a queued run absorbs a second boot's send. */ export const COMPACTION_TOMBSTONE_PURGE_SINGLETON_KEY = diff --git a/src/lib/jobs/daily-briefing-queue.ts b/src/lib/jobs/daily-briefing-queue.ts new file mode 100644 index 000000000..95948809e --- /dev/null +++ b/src/lib/jobs/daily-briefing-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `daily-briefing.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DAILY_BRIEFING_QUEUE = "daily-briefing"; + +/** Every 15 minutes; the per-user local-hour gate selects the fallback slot. */ +export const DAILY_BRIEFING_CRON = "*/15 * * * *"; diff --git a/src/lib/jobs/daily-briefing.ts b/src/lib/jobs/daily-briefing.ts index 522cb29c0..f8a50b9d1 100644 --- a/src/lib/jobs/daily-briefing.ts +++ b/src/lib/jobs/daily-briefing.ts @@ -30,10 +30,6 @@ import { } from "@/lib/daily/daily-briefing-push"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; -export const DAILY_BRIEFING_QUEUE = "daily-briefing"; -/** Every 15 minutes; the per-user local-hour gate selects the fallback slot. */ -export const DAILY_BRIEFING_CRON = "*/15 * * * *"; - export interface DailyBriefingTickSummary { candidatesScanned: number; inSlot: number; diff --git a/src/lib/jobs/dawarich-sync-queue.ts b/src/lib/jobs/dawarich-sync-queue.ts new file mode 100644 index 000000000..c597fab3b --- /dev/null +++ b/src/lib/jobs/dawarich-sync-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `dawarich-sync.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DAWARICH_SYNC_QUEUE = "dawarich-sync"; + +/** Daily 03:20 Europe/Berlin, after the night's environment fetch. */ +export const DAWARICH_SYNC_CRON = "20 3 * * *"; diff --git a/src/lib/jobs/dawarich-sync.ts b/src/lib/jobs/dawarich-sync.ts index e856a388a..ebd4bac30 100644 --- a/src/lib/jobs/dawarich-sync.ts +++ b/src/lib/jobs/dawarich-sync.ts @@ -30,11 +30,7 @@ import { import { daysBetweenDateKeys, shiftDateKey } from "@/lib/tz/format"; import { resolveUserTimezone } from "@/lib/tz/resolver"; import { DAWARICH_SYNC_MAX_DAYS } from "@/lib/validations/dawarich"; - -export const DAWARICH_SYNC_QUEUE = "dawarich-sync"; - -/** Daily 03:20 Europe/Berlin, after the night's environment fetch. */ -export const DAWARICH_SYNC_CRON = "20 3 * * *"; +import { DAWARICH_SYNC_QUEUE } from "./dawarich-sync-queue"; /** The nightly window: yesterday and the six days before it. */ const NIGHTLY_WINDOW_DAYS = 7; diff --git a/src/lib/jobs/dense-intraday-hourly-rebuild-queue.ts b/src/lib/jobs/dense-intraday-hourly-rebuild-queue.ts new file mode 100644 index 000000000..0d80fb3db --- /dev/null +++ b/src/lib/jobs/dense-intraday-hourly-rebuild-queue.ts @@ -0,0 +1,14 @@ +/** + * The queue facts of `dense-intraday-hourly-rebuild.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const DENSE_INTRADAY_HOURLY_REBUILD_QUEUE = + "dense-intraday-hourly-rebuild"; + +/** + * Serial concurrency — a rebuild walks every folded day for one account + * with a transaction per day; concurrency-1 keeps it from crowding the + * request pool, matching the retention drain's convention. + */ +export const DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY = 1; diff --git a/src/lib/jobs/dense-intraday-hourly-rebuild.ts b/src/lib/jobs/dense-intraday-hourly-rebuild.ts index a90de1ea7..14ead6307 100644 --- a/src/lib/jobs/dense-intraday-hourly-rebuild.ts +++ b/src/lib/jobs/dense-intraday-hourly-rebuild.ts @@ -34,16 +34,7 @@ import { DENSE_INTRADAY_RETENTION_TYPES, } from "@/lib/measurements/dense-intraday-retention"; import { runDenseIntradayHourlyRebuild } from "@/lib/measurements/dense-intraday-hourly-rebuild"; - -export const DENSE_INTRADAY_HOURLY_REBUILD_QUEUE = - "dense-intraday-hourly-rebuild"; - -/** - * Serial concurrency — a rebuild walks every folded day for one account - * with a transaction per day; concurrency-1 keeps it from crowding the - * request pool, matching the retention drain's convention. - */ -export const DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY = 1; +import { DENSE_INTRADAY_HOURLY_REBUILD_QUEUE } from "./dense-intraday-hourly-rebuild-queue"; /** * Boot-discovery jobs start only after this delay (seconds). The rebuild diff --git a/src/lib/jobs/dense-intraday-retention-queue.ts b/src/lib/jobs/dense-intraday-retention-queue.ts new file mode 100644 index 000000000..e2a6e80da --- /dev/null +++ b/src/lib/jobs/dense-intraday-retention-queue.ts @@ -0,0 +1,14 @@ +/** + * The queue facts of `dense-intraday-retention.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const DENSE_INTRADAY_RETENTION_QUEUE = "dense-intraday-retention"; + +/** + * Serial concurrency — the drain walks every out-of-window per-sample + * dense-tier row for a user and writes a transaction per day; + * concurrency-1 keeps it from crowding the request pool, matching the + * mean-consolidation + step-consolidation convention. + */ +export const DENSE_INTRADAY_RETENTION_CONCURRENCY = 1; diff --git a/src/lib/jobs/dense-intraday-retention.ts b/src/lib/jobs/dense-intraday-retention.ts index 7cbcdfad1..c807020a9 100644 --- a/src/lib/jobs/dense-intraday-retention.ts +++ b/src/lib/jobs/dense-intraday-retention.ts @@ -27,16 +27,7 @@ import { DENSE_INTRADAY_RETENTION_DAYS, DENSE_INTRADAY_FOLD_SOURCES, } from "@/lib/measurements/dense-intraday-retention"; - -export const DENSE_INTRADAY_RETENTION_QUEUE = "dense-intraday-retention"; - -/** - * Serial concurrency — the drain walks every out-of-window per-sample - * dense-tier row for a user and writes a transaction per day; - * concurrency-1 keeps it from crowding the request pool, matching the - * mean-consolidation + step-consolidation convention. - */ -export const DENSE_INTRADAY_RETENTION_CONCURRENCY = 1; +import { DENSE_INTRADAY_RETENTION_QUEUE } from "./dense-intraday-retention-queue"; /** * Daily at 03:50 Europe/Berlin — inside the existing 03:xx maintenance diff --git a/src/lib/jobs/document-ai-run-queue.ts b/src/lib/jobs/document-ai-run-queue.ts new file mode 100644 index 000000000..53583d010 --- /dev/null +++ b/src/lib/jobs/document-ai-run-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-ai-run.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DOCUMENT_AI_RUN_QUEUE = "document-ai-run"; + +/** Two reads at a time: provider calls, kept off the request pool. */ +export const DOCUMENT_AI_RUN_CONCURRENCY = 2; diff --git a/src/lib/jobs/document-ai-run-reaper-queue.ts b/src/lib/jobs/document-ai-run-reaper-queue.ts new file mode 100644 index 000000000..143869c04 --- /dev/null +++ b/src/lib/jobs/document-ai-run-reaper-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-ai-run-reaper.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the job so the worker + * registry and every sender can read them without loading it. + */ +export const DOCUMENT_AI_RUN_REAPER_QUEUE = "document-ai-run-reaper"; + +/** Every five minutes: a queued run fails within twenty minutes at worst. */ +export const DOCUMENT_AI_RUN_REAPER_CRON = "*/5 * * * *"; diff --git a/src/lib/jobs/document-ai-run-reaper.ts b/src/lib/jobs/document-ai-run-reaper.ts index f2e79eb40..ba6f8bd65 100644 --- a/src/lib/jobs/document-ai-run-reaper.ts +++ b/src/lib/jobs/document-ai-run-reaper.ts @@ -15,10 +15,6 @@ import { reapAiRuns } from "@/lib/documents/ai-runs/store"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; -export const DOCUMENT_AI_RUN_REAPER_QUEUE = "document-ai-run-reaper"; -/** Every five minutes: a queued run fails within twenty minutes at worst. */ -export const DOCUMENT_AI_RUN_REAPER_CRON = "*/5 * * * *"; - export interface DocumentAiRunReaperPayload { triggeredAt?: string; } diff --git a/src/lib/jobs/document-ai-run.ts b/src/lib/jobs/document-ai-run.ts index 2000054b4..6d9f25a52 100644 --- a/src/lib/jobs/document-ai-run.ts +++ b/src/lib/jobs/document-ai-run.ts @@ -97,11 +97,7 @@ import { executeSupplementLabelExtraction, requireSupplementLabelProvider, } from "@/lib/medications/ingredients/label-extract"; - -export const DOCUMENT_AI_RUN_QUEUE = "document-ai-run"; - -/** Two reads at a time: provider calls, kept off the request pool. */ -export const DOCUMENT_AI_RUN_CONCURRENCY = 2; +import { DOCUMENT_AI_RUN_QUEUE } from "./document-ai-run-queue"; /** * The job's expiry: one hour. Three model calls at the longest response diff --git a/src/lib/jobs/document-content-index-backfill-queue.ts b/src/lib/jobs/document-content-index-backfill-queue.ts new file mode 100644 index 000000000..513072f8d --- /dev/null +++ b/src/lib/jobs/document-content-index-backfill-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-content-index-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const CONTENT_INDEX_BACKFILL_QUEUE = "document-content-index-backfill"; + +/** Serial concurrency — provider calls, kept off the request pool. */ +export const CONTENT_INDEX_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-content-index-backfill.ts b/src/lib/jobs/document-content-index-backfill.ts index f355cb7d3..2402e7149 100644 --- a/src/lib/jobs/document-content-index-backfill.ts +++ b/src/lib/jobs/document-content-index-backfill.ts @@ -55,11 +55,7 @@ import { transcribeDocument } from "@/lib/documents/describe"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { resolveDocumentVisionProvider } from "@/lib/documents/provider-order"; import { annotate } from "@/lib/logging/context"; - -export const CONTENT_INDEX_BACKFILL_QUEUE = "document-content-index-backfill"; - -/** Serial concurrency — provider calls, kept off the request pool. */ -export const CONTENT_INDEX_BACKFILL_CONCURRENCY = 1; +import { CONTENT_INDEX_BACKFILL_QUEUE } from "./document-content-index-backfill-queue"; /** Discovery page size (id-only; blobs are loaded one at a time). */ const PAGE_SIZE = 50; diff --git a/src/lib/jobs/document-index-queue.ts b/src/lib/jobs/document-index-queue.ts new file mode 100644 index 000000000..caf1d936d --- /dev/null +++ b/src/lib/jobs/document-index-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-index.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DOCUMENT_INDEX_QUEUE = "document-index"; + +/** Serial concurrency — provider calls + PDF parse, kept off the request pool. */ +export const DOCUMENT_INDEX_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-index.ts b/src/lib/jobs/document-index.ts index abca1d45a..4cfacec37 100644 --- a/src/lib/jobs/document-index.ts +++ b/src/lib/jobs/document-index.ts @@ -17,11 +17,7 @@ import { indexDocumentContent } from "@/lib/documents/index-document"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { annotate } from "@/lib/logging/context"; import { caughtAs, logCaught } from "@/lib/logging/signal"; - -export const DOCUMENT_INDEX_QUEUE = "document-index"; - -/** Serial concurrency — provider calls + PDF parse, kept off the request pool. */ -export const DOCUMENT_INDEX_CONCURRENCY = 1; +import { DOCUMENT_INDEX_QUEUE } from "./document-index-queue"; export interface DocumentIndexPayload { userId: string; diff --git a/src/lib/jobs/document-summary-catchup-queue.ts b/src/lib/jobs/document-summary-catchup-queue.ts new file mode 100644 index 000000000..3fa82c31f --- /dev/null +++ b/src/lib/jobs/document-summary-catchup-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-summary-catchup.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DOCUMENT_SUMMARY_CATCHUP_QUEUE = "document-summary-catchup"; + +/** Serial concurrency — cheap discovery + fan-out enqueues. */ +export const DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-summary-catchup.ts b/src/lib/jobs/document-summary-catchup.ts index 3cb5c191a..9d8ce97d3 100644 --- a/src/lib/jobs/document-summary-catchup.ts +++ b/src/lib/jobs/document-summary-catchup.ts @@ -40,11 +40,7 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { enqueueDocumentSummary } from "@/lib/jobs/document-summary"; import { annotate } from "@/lib/logging/context"; import { logCaught } from "@/lib/logging/signal"; - -export const DOCUMENT_SUMMARY_CATCHUP_QUEUE = "document-summary-catchup"; - -/** Serial concurrency — cheap discovery + fan-out enqueues. */ -export const DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY = 1; +import { DOCUMENT_SUMMARY_CATCHUP_QUEUE } from "./document-summary-catchup-queue"; /** Discovery page size (id-only). */ const PAGE_SIZE = 100; diff --git a/src/lib/jobs/document-summary-queue.ts b/src/lib/jobs/document-summary-queue.ts new file mode 100644 index 000000000..c509924d7 --- /dev/null +++ b/src/lib/jobs/document-summary-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-summary.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const DOCUMENT_SUMMARY_QUEUE = "document-summary"; + +/** Serial concurrency — provider calls, kept off the request pool. */ +export const DOCUMENT_SUMMARY_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-summary-reaper-queue.ts b/src/lib/jobs/document-summary-reaper-queue.ts new file mode 100644 index 000000000..0eb2012db --- /dev/null +++ b/src/lib/jobs/document-summary-reaper-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-summary-reaper.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the job so the worker + * registry and every sender can read them without loading it. + */ +export const DOCUMENT_SUMMARY_REAPER_QUEUE = "document-summary-reaper"; + +/** Hourly at :50 — off the maintenance herd's minute marks. */ +export const DOCUMENT_SUMMARY_REAPER_CRON = "50 * * * *"; diff --git a/src/lib/jobs/document-summary-reaper.ts b/src/lib/jobs/document-summary-reaper.ts index f3e7b9362..775a90926 100644 --- a/src/lib/jobs/document-summary-reaper.ts +++ b/src/lib/jobs/document-summary-reaper.ts @@ -30,10 +30,6 @@ import { getWorkerPrisma } from "@/lib/jobs/reminder/shared"; import { SUMMARY_PENDING_TTL_MS } from "@/lib/documents/store"; import type { PrismaClient } from "@/generated/prisma/client"; -export const DOCUMENT_SUMMARY_REAPER_QUEUE = "document-summary-reaper"; -/** Hourly at :50 — off the maintenance herd's minute marks. */ -export const DOCUMENT_SUMMARY_REAPER_CRON = "50 * * * *"; - export interface DocumentSummaryReaperPayload { triggeredAt?: string; } diff --git a/src/lib/jobs/document-summary.ts b/src/lib/jobs/document-summary.ts index de889748f..dfebd34e6 100644 --- a/src/lib/jobs/document-summary.ts +++ b/src/lib/jobs/document-summary.ts @@ -49,8 +49,7 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { annotate } from "@/lib/logging/context"; import type { DocumentSummaryState } from "@/generated/prisma/client"; import { logCaught } from "@/lib/logging/signal"; - -export const DOCUMENT_SUMMARY_QUEUE = "document-summary"; +import { DOCUMENT_SUMMARY_QUEUE } from "./document-summary-queue"; /** * Record what became of this document's summary. Never touches a READY row — @@ -73,9 +72,6 @@ async function markSummaryState( }); } -/** Serial concurrency — provider calls, kept off the request pool. */ -export const DOCUMENT_SUMMARY_CONCURRENCY = 1; - export interface DocumentSummaryPayload { userId: string; documentId: string; diff --git a/src/lib/jobs/document-thumbnail-backfill-queue.ts b/src/lib/jobs/document-thumbnail-backfill-queue.ts new file mode 100644 index 000000000..d8c086432 --- /dev/null +++ b/src/lib/jobs/document-thumbnail-backfill-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-thumbnail-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the job so the worker + * registry and every sender can read them without loading it. + */ +export const DOCUMENT_THUMBNAIL_BACKFILL_QUEUE = "document-thumbnail-backfill"; + +/** Serial concurrency — cheap discovery + fan-out enqueues. */ +export const DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-thumbnail-backfill.ts b/src/lib/jobs/document-thumbnail-backfill.ts index e09568c6b..bdd373642 100644 --- a/src/lib/jobs/document-thumbnail-backfill.ts +++ b/src/lib/jobs/document-thumbnail-backfill.ts @@ -30,11 +30,7 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { enqueueDocumentThumbnail } from "@/lib/jobs/document-thumbnail"; import { annotate } from "@/lib/logging/context"; import { logCaught } from "@/lib/logging/signal"; - -export const DOCUMENT_THUMBNAIL_BACKFILL_QUEUE = "document-thumbnail-backfill"; - -/** Serial concurrency — cheap discovery + fan-out enqueues. */ -export const DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY = 1; +import { DOCUMENT_THUMBNAIL_BACKFILL_QUEUE } from "./document-thumbnail-backfill-queue"; /** Discovery page size (id-only). */ const PAGE_SIZE = 100; diff --git a/src/lib/jobs/document-thumbnail-queue.ts b/src/lib/jobs/document-thumbnail-queue.ts new file mode 100644 index 000000000..312b7269f --- /dev/null +++ b/src/lib/jobs/document-thumbnail-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `document-thumbnail.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the job so the worker + * registry and every sender can read them without loading it. + */ +export const DOCUMENT_THUMBNAIL_QUEUE = "document-thumbnail"; + +/** Serial concurrency — canvas/pdfjs decode, kept off the request pool. */ +export const DOCUMENT_THUMBNAIL_CONCURRENCY = 1; diff --git a/src/lib/jobs/document-thumbnail.ts b/src/lib/jobs/document-thumbnail.ts index f593ab760..4195820ee 100644 --- a/src/lib/jobs/document-thumbnail.ts +++ b/src/lib/jobs/document-thumbnail.ts @@ -34,11 +34,7 @@ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { withBackgroundEvent } from "@/lib/logging/background"; import { annotate } from "@/lib/logging/context"; import { logCaught } from "@/lib/logging/signal"; - -export const DOCUMENT_THUMBNAIL_QUEUE = "document-thumbnail"; - -/** Serial concurrency — canvas/pdfjs decode, kept off the request pool. */ -export const DOCUMENT_THUMBNAIL_CONCURRENCY = 1; +import { DOCUMENT_THUMBNAIL_QUEUE } from "./document-thumbnail-queue"; export interface DocumentThumbnailPayload { userId: string; diff --git a/src/lib/jobs/drain-cumulative-queue.ts b/src/lib/jobs/drain-cumulative-queue.ts new file mode 100644 index 000000000..352049c6c --- /dev/null +++ b/src/lib/jobs/drain-cumulative-queue.ts @@ -0,0 +1,20 @@ +/** + * The nightly drain of per-sample cumulative rows: its queue, cron and + * payload. It collapses each user × cumulative type × day into one `stats:` + * row, and the same tick folds the daily-mean and dense intra-day retention + * passes (`reminder/rollup-handlers.ts`). 03:45 sits between the 03:15 + * audit-log cleanup and the 04:00 feedback aggregator; the 36-hour grace + * window keeps today and the trailing watch-sync window raw. + */ +export const DRAIN_CUMULATIVE_QUEUE = "drain-per-sample-cumulative"; + +export const DRAIN_CUMULATIVE_CRON = "45 3 * * *"; + +export interface DrainCumulativePayload { + triggeredAt?: string; + /** + * How many times this tick has already handed itself on. Absent on the + * cron's own job. See `nextDrainContinuation`. + */ + continuation?: number; +} diff --git a/src/lib/jobs/encryption-key-rotate-queue.ts b/src/lib/jobs/encryption-key-rotate-queue.ts new file mode 100644 index 000000000..40e686ca1 --- /dev/null +++ b/src/lib/jobs/encryption-key-rotate-queue.ts @@ -0,0 +1,8 @@ +/** + * The queue facts of `encryption-key-rotate.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the job so the worker + * registry and every sender can read them without loading it. + */ +export const ENCRYPTION_KEY_ROTATE_QUEUE = "encryption-key-rotate"; + +export const ENCRYPTION_KEY_ROTATE_CONCURRENCY = 1; diff --git a/src/lib/jobs/encryption-key-rotate.ts b/src/lib/jobs/encryption-key-rotate.ts index cf1c52250..ccde5044a 100644 --- a/src/lib/jobs/encryption-key-rotate.ts +++ b/src/lib/jobs/encryption-key-rotate.ts @@ -36,9 +36,6 @@ import { retireCanariesWithoutData, type CanaryClient, } from "@/lib/crypto/canary"; - -export const ENCRYPTION_KEY_ROTATE_QUEUE = "encryption-key-rotate"; -export const ENCRYPTION_KEY_ROTATE_CONCURRENCY = 1; /** Fixed key so duplicate admin triggers coalesce into one queued run. */ export const ENCRYPTION_KEY_ROTATE_SINGLETON = "encryption-key-rotate"; diff --git a/src/lib/jobs/fitbit-backfill.ts b/src/lib/jobs/fitbit-backfill.ts index ad457bfae..02f84fa21 100644 --- a/src/lib/jobs/fitbit-backfill.ts +++ b/src/lib/jobs/fitbit-backfill.ts @@ -21,15 +21,7 @@ import { } from "@/lib/integrations/page-walk"; import { syncUserFitbit } from "@/lib/fitbit/sync"; import { isFitbitRetired } from "@/lib/fitbit/retirement"; - -export const FITBIT_BACKFILL_QUEUE = "fitbit-backfill"; - -/** - * Serial concurrency — a backfill walks years of history for one account and is - * rate-bounded by Google's per-app quota; concurrency-1 keeps it from crowding - * the request pool, matching `WHOOP_BACKFILL_CONCURRENCY`. - */ -export const FITBIT_BACKFILL_CONCURRENCY = 1; +import { FITBIT_BACKFILL_QUEUE } from "./../fitbit/jobs/queues"; export interface FitbitBackfillPayload { userId: string; diff --git a/src/lib/jobs/fitbit-sleep-repair.ts b/src/lib/jobs/fitbit-sleep-repair.ts index 0b681ff1b..f79ebb8f2 100644 --- a/src/lib/jobs/fitbit-sleep-repair.ts +++ b/src/lib/jobs/fitbit-sleep-repair.ts @@ -46,15 +46,7 @@ import { } from "@/lib/fitbit/sync-core"; import { syncUserSleep } from "@/lib/fitbit/sync-sleep"; import { isFitbitRetired } from "@/lib/fitbit/retirement"; - -export const FITBIT_SLEEP_REPAIR_QUEUE = "fitbit-sleep-repair"; - -/** - * Serial concurrency — a repair walks a year of sleep history for one account - * in 30-day requests against the tight per-user budget; concurrency-1 keeps it - * from crowding the request pool, matching `FITBIT_BACKFILL_CONCURRENCY`. - */ -export const FITBIT_SLEEP_REPAIR_CONCURRENCY = 1; +import { FITBIT_SLEEP_REPAIR_QUEUE } from "./../fitbit/jobs/queues"; export interface FitbitSleepRepairPayload { userId: string; diff --git a/src/lib/jobs/google-health-backfill-queue.ts b/src/lib/jobs/google-health-backfill-queue.ts new file mode 100644 index 000000000..26856d461 --- /dev/null +++ b/src/lib/jobs/google-health-backfill-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `google-health-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const GOOGLE_HEALTH_BACKFILL_QUEUE = "google-health-backfill"; + +/** + * Serial concurrency — a backfill walks years of history for one account and is + * rate-bounded by Google's per-app quota; concurrency-1 keeps it from crowding + * the request pool, matching `FITBIT_BACKFILL_CONCURRENCY`. + */ +export const GOOGLE_HEALTH_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/google-health-backfill.ts b/src/lib/jobs/google-health-backfill.ts index d070872de..94d44bb9e 100644 --- a/src/lib/jobs/google-health-backfill.ts +++ b/src/lib/jobs/google-health-backfill.ts @@ -44,15 +44,7 @@ import { import { reportJobProgress } from "@/lib/jobs/job-observer"; import { LANE_YIELD_DELAY_SECONDS, laneWanted } from "@/lib/jobs/heavy-lane"; import { GOOGLE_HEALTH_COLLECTION_KEYS } from "@/lib/google-health/collection-keys"; - -export const GOOGLE_HEALTH_BACKFILL_QUEUE = "google-health-backfill"; - -/** - * Serial concurrency — a backfill walks years of history for one account and is - * rate-bounded by Google's per-app quota; concurrency-1 keeps it from crowding - * the request pool, matching `FITBIT_BACKFILL_CONCURRENCY`. - */ -export const GOOGLE_HEALTH_BACKFILL_CONCURRENCY = 1; +import { GOOGLE_HEALTH_BACKFILL_QUEUE } from "./google-health-backfill-queue"; /** * One slice of a backfill: how long it walks and how many pages it reads diff --git a/src/lib/jobs/google-health-sleep-repair-queue.ts b/src/lib/jobs/google-health-sleep-repair-queue.ts new file mode 100644 index 000000000..4f83dcf38 --- /dev/null +++ b/src/lib/jobs/google-health-sleep-repair-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `google-health-sleep-repair.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE = "google-health-sleep-repair"; + +/** + * Serial concurrency — a repair walks the full sleep history for one account + * and is rate-bounded by Google's per-app quota; concurrency-1 keeps it from + * crowding the request pool, matching `GOOGLE_HEALTH_BACKFILL_CONCURRENCY`. + */ +export const GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY = 1; diff --git a/src/lib/jobs/google-health-sleep-repair.ts b/src/lib/jobs/google-health-sleep-repair.ts index 8c01adc9b..e4fe7dedf 100644 --- a/src/lib/jobs/google-health-sleep-repair.ts +++ b/src/lib/jobs/google-health-sleep-repair.ts @@ -37,15 +37,7 @@ import { GOOGLE_HEALTH_SYNC_LOCK_WAIT_MS, withGoogleHealthSyncLock, } from "@/lib/google-health/sync-lock"; - -export const GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE = "google-health-sleep-repair"; - -/** - * Serial concurrency — a repair walks the full sleep history for one account - * and is rate-bounded by Google's per-app quota; concurrency-1 keeps it from - * crowding the request pool, matching `GOOGLE_HEALTH_BACKFILL_CONCURRENCY`. - */ -export const GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY = 1; +import { GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE } from "./google-health-sleep-repair-queue"; export interface GoogleHealthSleepRepairPayload { userId: string; diff --git a/src/lib/jobs/health-connect-import-queue.ts b/src/lib/jobs/health-connect-import-queue.ts new file mode 100644 index 000000000..55b3f3c1b --- /dev/null +++ b/src/lib/jobs/health-connect-import-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `health-connect-import-worker.ts`: its queue names and + * concurrency. They live apart from the worker so the registry and the upload + * routes can name the queue without loading the parser. + */ +export const HEALTH_CONNECT_IMPORT_QUEUE = "health-connect-import"; + +/** One import at a time per worker: the SQLite read and the writes are heavy. */ +export const HEALTH_CONNECT_IMPORT_CONCURRENCY = 1; diff --git a/src/lib/jobs/health-connect-import-worker.ts b/src/lib/jobs/health-connect-import-worker.ts index 08850c72a..a989b56c5 100644 --- a/src/lib/jobs/health-connect-import-worker.ts +++ b/src/lib/jobs/health-connect-import-worker.ts @@ -33,8 +33,6 @@ import { annotate } from "@/lib/logging/context"; import { withBackgroundEvent } from "@/lib/logging/background"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; -export const HEALTH_CONNECT_IMPORT_QUEUE = "health-connect-import"; - /** `ImportJob.kind` of the rows this worker owns. */ export const HEALTH_CONNECT_IMPORT_KIND = "health_connect"; @@ -45,9 +43,6 @@ export const HEALTH_CONNECT_IMPORT_KIND = "health_connect"; */ export const HEALTH_CONNECT_IMPORT_PARSER_REVISION = 1; -/** One import at a time per worker: the SQLite read and the writes are heavy. */ -export const HEALTH_CONNECT_IMPORT_CONCURRENCY = 1; - /** Same shape and reasoning as `APPLE_HEALTH_IMPORT_SEND_OPTIONS`. */ export const HEALTH_CONNECT_IMPORT_SEND_OPTIONS = { retryLimit: 0, diff --git a/src/lib/jobs/insight-pregenerate-shared.ts b/src/lib/jobs/insight-pregenerate-shared.ts index 136d9353a..f15a9e605 100644 --- a/src/lib/jobs/insight-pregenerate-shared.ts +++ b/src/lib/jobs/insight-pregenerate-shared.ts @@ -249,3 +249,10 @@ export async function enqueuePregenerateFailureRetry(payload: { // catch-net, exactly the pre-retry behaviour. } } + +/** + * 04:30 Europe/Berlin. Slots between the cumulative drain (03:30) and + * the feedback aggregator (04:00) maintenance window's tail so the + * LLM-bound pass doesn't contend with the SQL-bound nightly folds. + */ +export const INSIGHT_PREGENERATE_CRON = "30 4 * * *"; diff --git a/src/lib/jobs/insight-pregenerate.ts b/src/lib/jobs/insight-pregenerate.ts index 509551b51..b8a6ce6b8 100644 --- a/src/lib/jobs/insight-pregenerate.ts +++ b/src/lib/jobs/insight-pregenerate.ts @@ -91,13 +91,6 @@ import { export { INSIGHT_PREGENERATE_QUEUE, enqueueForceWarm }; export type { InsightPregeneratePayload }; -/** - * 04:30 Europe/Berlin. Slots between the cumulative drain (03:30) and - * the feedback aggregator (04:00) maintenance window's tail so the - * LLM-bound pass doesn't contend with the SQL-bound nightly folds. - */ -export const INSIGHT_PREGENERATE_CRON = "30 4 * * *"; - /** Stale-cache threshold for the nightly discovery pass. * * The cron runs every 24 h and the advisor cache lives 24 h — so ANY diff --git a/src/lib/jobs/insight-status-generate-queue.ts b/src/lib/jobs/insight-status-generate-queue.ts new file mode 100644 index 000000000..bee53259d --- /dev/null +++ b/src/lib/jobs/insight-status-generate-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `insight-status-generate-shared.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const INSIGHT_STATUS_GENERATE_QUEUE = "insight-status-generate"; + +/** + * Low concurrency keeps the on-demand warm pass from saturating the + * Prisma pool or fanning out an unbounded number of concurrent provider + * calls when several cards request generation on the same first visit. + */ +export const INSIGHT_STATUS_GENERATE_CONCURRENCY = 2; diff --git a/src/lib/jobs/insight-status-generate-shared.ts b/src/lib/jobs/insight-status-generate-shared.ts index 5296732d1..e448872d9 100644 --- a/src/lib/jobs/insight-status-generate-shared.ts +++ b/src/lib/jobs/insight-status-generate-shared.ts @@ -21,15 +21,7 @@ import { type ProviderWorkAuthority, } from "@/lib/sharing/provider-work-authority"; import { logCaught } from "@/lib/logging/signal"; - -export const INSIGHT_STATUS_GENERATE_QUEUE = "insight-status-generate"; - -/** - * Low concurrency keeps the on-demand warm pass from saturating the - * Prisma pool or fanning out an unbounded number of concurrent provider - * calls when several cards request generation on the same first visit. - */ -export const INSIGHT_STATUS_GENERATE_CONCURRENCY = 2; +import { INSIGHT_STATUS_GENERATE_QUEUE } from "./insight-status-generate-queue"; /** * The seven per-metric status scopes the queue can generate. Mirrors the diff --git a/src/lib/jobs/insight-status-generate.ts b/src/lib/jobs/insight-status-generate.ts index 80118c473..33b525017 100644 --- a/src/lib/jobs/insight-status-generate.ts +++ b/src/lib/jobs/insight-status-generate.ts @@ -49,13 +49,15 @@ import { } from "@/lib/insights/derived/derived-assessment-ai"; import { isDerivedMetricId } from "@/lib/insights/derived/registry"; import { - INSIGHT_STATUS_GENERATE_QUEUE, - INSIGHT_STATUS_GENERATE_CONCURRENCY, INSIGHT_STATUS_METRICS, enqueueStatusGeneration, type InsightStatusMetric, type InsightStatusGeneratePayload, } from "@/lib/jobs/insight-status-generate-shared"; +import { + INSIGHT_STATUS_GENERATE_QUEUE, + INSIGHT_STATUS_GENERATE_CONCURRENCY, +} from "@/lib/jobs/insight-status-generate-queue"; import { aiCapabilityForJob } from "@/lib/ai/capabilities/gate"; import { mayDispatchProviderWork, diff --git a/src/lib/jobs/intake-auto-skip-queue.ts b/src/lib/jobs/intake-auto-skip-queue.ts new file mode 100644 index 000000000..6488f849d --- /dev/null +++ b/src/lib/jobs/intake-auto-skip-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `intake-auto-skip.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const INTAKE_AUTO_SKIP_QUEUE = "intake-auto-skip"; + +/** + * Cron schedule. Hourly at :05 so the tick doesn't collide with the + * top-of-the-hour reminder-check (:00) or the rollup fold (:30) + * cadences. Europe/Berlin matches the rest of the worker's schedules. + */ +export const INTAKE_AUTO_SKIP_CRON = "5 * * * *"; diff --git a/src/lib/jobs/intake-auto-skip.ts b/src/lib/jobs/intake-auto-skip.ts index 0220bc51e..db63b19c0 100644 --- a/src/lib/jobs/intake-auto-skip.ts +++ b/src/lib/jobs/intake-auto-skip.ts @@ -73,15 +73,6 @@ import { } from "@/lib/medications/scheduling/live-era"; import { INTAKE_AUTO_TAKE_LOOKBACK_MS } from "@/lib/jobs/intake-auto-take"; -export const INTAKE_AUTO_SKIP_QUEUE = "intake-auto-skip"; - -/** - * Cron schedule. Hourly at :05 so the tick doesn't collide with the - * top-of-the-hour reminder-check (:00) or the rollup fold (:30) - * cadences. Europe/Berlin matches the rest of the worker's schedules. - */ -export const INTAKE_AUTO_SKIP_CRON = "5 * * * *"; - /** * Grace floor before auto-skip kicks in. 24 h matches the spec and * the user's real-world cadence (late entries up to a full day after diff --git a/src/lib/jobs/intake-auto-take-queue.ts b/src/lib/jobs/intake-auto-take-queue.ts new file mode 100644 index 000000000..8572b837b --- /dev/null +++ b/src/lib/jobs/intake-auto-take-queue.ts @@ -0,0 +1,12 @@ +/** + * The queue facts of `intake-auto-take.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const INTAKE_AUTO_TAKE_QUEUE = "intake-auto-take"; + +/** + * Every 15 minutes, off the reminder tick (:00/:15/:30/:45) and the hourly + * auto-miss pass (:05). A 19:00 dose reads taken by 19:10 at the latest. + */ +export const INTAKE_AUTO_TAKE_CRON = "10,25,40,55 * * * *"; diff --git a/src/lib/jobs/intake-auto-take.ts b/src/lib/jobs/intake-auto-take.ts index e26a42dca..16265f1d2 100644 --- a/src/lib/jobs/intake-auto-take.ts +++ b/src/lib/jobs/intake-auto-take.ts @@ -98,14 +98,6 @@ export { type SlotDecision, } from "@/lib/medications/auto-take"; -export const INTAKE_AUTO_TAKE_QUEUE = "intake-auto-take"; - -/** - * Every 15 minutes, off the reminder tick (:00/:15/:30/:45) and the hourly - * auto-miss pass (:05). A 19:00 dose reads taken by 19:10 at the latest. - */ -export const INTAKE_AUTO_TAKE_CRON = "10,25,40,55 * * * *"; - const MINUTE_MS = 60 * 1000; const HOUR_MS = 60 * MINUTE_MS; const DAY_MS = 24 * HOUR_MS; diff --git a/src/lib/jobs/integration-backfill-forward.ts b/src/lib/jobs/integration-backfill-forward.ts new file mode 100644 index 000000000..054fc111b --- /dev/null +++ b/src/lib/jobs/integration-backfill-forward.ts @@ -0,0 +1,27 @@ +/** + * The handler of every full-history source queue (`whoop-backfill`, + * `strava-backfill`, the sleep repairs, ...): it forwards each job to the + * shared admission queue, where the import itself runs in the one + * cross-provider lane. The source queue keeps its per-connection singleton + * key, which is what boot discovery re-offers against. + */ +import type { Job, PgBoss } from "pg-boss"; + +import { + enqueueIntegrationBackfillAdmission, + type IntegrationBackfillData, + type IntegrationBackfillKind, +} from "@/lib/jobs/integration-backfill-admission"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; + +export function forwardToBackfillAdmission( + boss: PgBoss, + kind: IntegrationBackfillKind, +): (jobs: Job[]) => Promise { + return async (jobs) => { + for (const job of jobs) { + await enqueueIntegrationBackfillAdmission(boss, { kind, data: job.data }); + } + return jobDone({ jobs: jobs.length }); + }; +} diff --git a/src/lib/jobs/integration-poll-queues.ts b/src/lib/jobs/integration-poll-queues.ts index addd5e94d..415921c0e 100644 --- a/src/lib/jobs/integration-poll-queues.ts +++ b/src/lib/jobs/integration-poll-queues.ts @@ -1,27 +1,85 @@ /** - * Queue names for the poll-only integration syncs. + * Names and crons of the integration sync queues. * - * Split out of `reminder/register-integration-sync.ts` so a request-path caller - * (an OAuth callback enqueueing the first sync at connect time) can name the - * queue without importing the registrar — which would drag every worker handler - * and its transitive graph into the request bundle. Same shape as - * `STRAVA_BACKFILL_QUEUE`, which the Strava callback already imports from its - * own module. + * A light module on purpose: the worker registry declares these queues from + * here (`registry/integration-sync.ts`), and a request-path caller (an OAuth + * callback enqueueing the first sync, a webhook enqueueing one resource) names + * a queue without loading the sync it feeds. * - * The registrar remains the single place that creates, schedules, and binds a - * handler to each of these; the values here are its source. A name declared - * here but missing from `allQueues` there is the v1.4.37 dead-queue class, and - * the queue-wiring guards read both files for exactly that reason. + * The hourly minutes are staggered so the polls do not land on one boss poll: + * used are :00 :05 :08 :11 :13 :15 :17 :18 :20 :35 :41 :47 :50. */ -/** Hourly Nightscout CGM poll (:11). */ +/** Withings measure catch-net (the webhook is primary). */ +export const WITHINGS_SYNC_QUEUE = "withings-fallback-sync"; +export const WITHINGS_SYNC_CRON = "0 * * * *"; + +/** + * Withings activity and sleep: webhook-primary (appli 16 / 44), with an hourly + * catch-net for the deliveries Withings drops. + */ +export const WITHINGS_ACTIVITY_QUEUE = "withings-activity-sync"; +export const WITHINGS_ACTIVITY_CRON = "0 * * * *"; +export const WITHINGS_SLEEP_QUEUE = "withings-sleep-sync"; +export const WITHINGS_SLEEP_CRON = "15 * * * *"; + +/** + * Withings ECG catch-net. The queue is webhook-primary (appli 54), but appli 54 + * is not in the subscription set and the manual sync measures only, so a + * watch-only account would never have its strips pulled without this walk. + */ +export const WITHINGS_ECG_SYNC_CRON = "41 * * * *"; + +/** Daily sweep of the Withings OAuth state ledger. */ +export const WITHINGS_OAUTH_STATE_CLEANUP_QUEUE = + "withings-oauth-state-cleanup"; +export const WITHINGS_OAUTH_STATE_CLEANUP_CRON = "20 3 * * *"; + +/** + * WHOOP: recovery, sleep and workout are webhook-primary with an hourly + * catch-net; cycle has no webhook, so its cron is the only driver. + */ +export const WHOOP_RECOVERY_SYNC_QUEUE = "whoop-recovery-sync"; +export const WHOOP_RECOVERY_SYNC_CRON = "5 * * * *"; +export const WHOOP_SLEEP_SYNC_QUEUE = "whoop-sleep-sync"; +export const WHOOP_SLEEP_SYNC_CRON = "20 * * * *"; +export const WHOOP_WORKOUT_SYNC_QUEUE = "whoop-workout-sync"; +export const WHOOP_WORKOUT_SYNC_CRON = "35 * * * *"; +export const WHOOP_CYCLE_SYNC_QUEUE = "whoop-cycle-sync"; +export const WHOOP_CYCLE_SYNC_CRON = "50 * * * *"; + +/** Daily sweep of the WHOOP OAuth state ledger. */ +export const WHOOP_OAUTH_STATE_CLEANUP_QUEUE = "whoop-oauth-state-cleanup"; +export const WHOOP_OAUTH_STATE_CLEANUP_CRON = "22 3 * * *"; + +/** Google Health poll (no webhook). */ +export const GOOGLE_HEALTH_SYNC_QUEUE = "google-health-sync"; +export const GOOGLE_HEALTH_SYNC_CRON = "18 * * * *"; + +/** Daily sweep of the Google Health OAuth state ledger. */ +export const GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE = + "google-health-oauth-state-cleanup"; +export const GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON = "26 3 * * *"; + +/** Nightscout CGM poll. */ export const NIGHTSCOUT_SYNC_QUEUE = "nightscout-sync"; +export const NIGHTSCOUT_SYNC_CRON = "11 * * * *"; -/** Hourly Polar AccessLink poll (:13). */ +/** Polar AccessLink poll. */ export const POLAR_SYNC_QUEUE = "polar-sync"; +export const POLAR_SYNC_CRON = "13 * * * *"; -/** Hourly Oura Cloud poll (:15). */ +/** Oura Cloud poll. */ export const OURA_SYNC_QUEUE = "oura-sync"; +export const OURA_SYNC_CRON = "15 * * * *"; -/** Hourly Strava poll (:17). */ +/** Strava poll (the webhook is not wired). */ export const STRAVA_SYNC_QUEUE = "strava-sync"; +export const STRAVA_SYNC_CRON = "17 * * * *"; + +/** + * Daily sweep of the native OIDC handoff ledger. Handoff codes are single-use + * and expire at read, so this only reaps rows the app never came back for. + */ +export const OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE = "oidc-native-handoff-cleanup"; +export const OIDC_NATIVE_HANDOFF_CLEANUP_CRON = "28 3 * * *"; diff --git a/src/lib/jobs/integration-watchdog-queue.ts b/src/lib/jobs/integration-watchdog-queue.ts new file mode 100644 index 000000000..6324b1e1f --- /dev/null +++ b/src/lib/jobs/integration-watchdog-queue.ts @@ -0,0 +1,18 @@ +/** + * The queue facts of `integration-watchdog.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const INTEGRATION_WATCHDOG_HOURLY_QUEUE = "integration-watchdog-hourly"; + +export const INTEGRATION_WATCHDOG_DAILY_QUEUE = "integration-watchdog-daily"; + +/** + * :47 is a free minute off every hourly sync tick (used: 0, 5, 8, 11, 13, 15, + * 17, 18, 20, 30, 35, 41, 50), late enough in the hour that the syncs before it + * have written what they fetched. + */ +export const INTEGRATION_WATCHDOG_HOURLY_CRON = "47 * * * *"; + +/** Once a day, inside the maintenance window, off the hourly :47. */ +export const INTEGRATION_WATCHDOG_DAILY_CRON = "52 4 * * *"; diff --git a/src/lib/jobs/integration-watchdog.ts b/src/lib/jobs/integration-watchdog.ts index f4ce883e8..155ee95a5 100644 --- a/src/lib/jobs/integration-watchdog.ts +++ b/src/lib/jobs/integration-watchdog.ts @@ -16,18 +16,6 @@ import { type WatchdogMode, } from "@/lib/integrations/watchdog"; -export const INTEGRATION_WATCHDOG_HOURLY_QUEUE = "integration-watchdog-hourly"; -export const INTEGRATION_WATCHDOG_DAILY_QUEUE = "integration-watchdog-daily"; - -/** - * :47 is a free minute off every hourly sync tick (used: 0, 5, 8, 11, 13, 15, - * 17, 18, 20, 30, 35, 41, 50), late enough in the hour that the syncs before it - * have written what they fetched. - */ -export const INTEGRATION_WATCHDOG_HOURLY_CRON = "47 * * * *"; -/** Once a day, inside the maintenance window, off the hourly :47. */ -export const INTEGRATION_WATCHDOG_DAILY_CRON = "52 4 * * *"; - const LOCK_KEY = "job:integration-watchdog:pass"; export interface IntegrationWatchdogPayload { diff --git a/src/lib/jobs/measurement-fold-repair-queue.ts b/src/lib/jobs/measurement-fold-repair-queue.ts new file mode 100644 index 000000000..ff3060202 --- /dev/null +++ b/src/lib/jobs/measurement-fold-repair-queue.ts @@ -0,0 +1,6 @@ +/** + * The queue facts of `measurement-fold-repair.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const MEASUREMENT_FOLD_REPAIR_QUEUE = "measurement-fold-repair"; diff --git a/src/lib/jobs/measurement-fold-repair.ts b/src/lib/jobs/measurement-fold-repair.ts index 1efc75a9c..ffab94285 100644 --- a/src/lib/jobs/measurement-fold-repair.ts +++ b/src/lib/jobs/measurement-fold-repair.ts @@ -90,10 +90,8 @@ import { holdAccountFoldLock, } from "@/lib/export/restore-lock"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; -import { - COMPACTION_TOMBSTONE_PURGE_QUEUE, - COMPACTION_TOMBSTONE_PURGE_SINGLETON_KEY, -} from "@/lib/jobs/compaction-tombstone-purge"; +import { COMPACTION_TOMBSTONE_PURGE_SINGLETON_KEY } from "@/lib/jobs/compaction-tombstone-purge"; +import { COMPACTION_TOMBSTONE_PURGE_QUEUE } from "@/lib/jobs/compaction-tombstone-purge-queue"; import { jobBudget } from "@/lib/jobs/job-budget"; import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; import { reportJobRun, type JobRunCandidate } from "@/lib/jobs/job-run-report"; @@ -136,8 +134,7 @@ import { } from "@/lib/measurements/fold-constituents"; import { recomputeBucketsForMeasurement } from "@/lib/rollups/measurement-rollups"; import { localDayWindow } from "@/lib/tz/local-day"; - -export const MEASUREMENT_FOLD_REPAIR_QUEUE = "measurement-fold-repair"; +import { MEASUREMENT_FOLD_REPAIR_QUEUE } from "./measurement-fold-repair-queue"; /** Delay before the follow-up of a run that stopped early, in seconds. */ export const FOLD_REPAIR_CONTINUATION_DELAY_SECONDS = 30; diff --git a/src/lib/jobs/medication-intake-import-queue.ts b/src/lib/jobs/medication-intake-import-queue.ts new file mode 100644 index 000000000..da52962dc --- /dev/null +++ b/src/lib/jobs/medication-intake-import-queue.ts @@ -0,0 +1,8 @@ +/** + * The queue facts of `medication-intake-import.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const MEDICATION_INTAKE_IMPORT_QUEUE = "medication-intake-import"; + +export const MEDICATION_INTAKE_IMPORT_CONCURRENCY = 2; diff --git a/src/lib/jobs/medication-intake-import.ts b/src/lib/jobs/medication-intake-import.ts index 3fda3d1b2..bea85b8c1 100644 --- a/src/lib/jobs/medication-intake-import.ts +++ b/src/lib/jobs/medication-intake-import.ts @@ -9,9 +9,6 @@ import { jobBudget } from "@/lib/jobs/job-budget"; import { enqueueSupplementNutrientRecompute } from "@/lib/medications/ingredients/recompute-job"; import { consumeImportedIntakesBatch } from "@/lib/medications/inventory/consumption"; import { queueMedicationIntakeSync } from "@/lib/notifications/medication-intake-sync"; - -export const MEDICATION_INTAKE_IMPORT_QUEUE = "medication-intake-import"; -export const MEDICATION_INTAKE_IMPORT_CONCURRENCY = 2; export const MEDICATION_INTAKE_IMPORT_CHUNK_SIZE = 100; export const MEDICATION_INTAKE_IMPORT_STALE_AFTER_MS = 15 * 60_000; diff --git a/src/lib/jobs/medication-low-stock-queue.ts b/src/lib/jobs/medication-low-stock-queue.ts new file mode 100644 index 000000000..aeeca6dc3 --- /dev/null +++ b/src/lib/jobs/medication-low-stock-queue.ts @@ -0,0 +1,14 @@ +/** + * The queue facts of `medication-low-stock.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +/** Pg-boss queue + cron — imported by the reminder worker's bootstrap. */ +export const MEDICATION_LOW_STOCK_QUEUE = "medication-low-stock"; + +/** + * 09:00 Europe/Berlin — a supply alert is an errand prompt ("order a + * refill"), not a night signal; it fires once daily at a time the user + * can act on it. + */ +export const MEDICATION_LOW_STOCK_CRON = "0 9 * * *"; diff --git a/src/lib/jobs/medication-low-stock.ts b/src/lib/jobs/medication-low-stock.ts index 5c2e8af5c..42b2db001 100644 --- a/src/lib/jobs/medication-low-stock.ts +++ b/src/lib/jobs/medication-low-stock.ts @@ -86,15 +86,6 @@ import { resolveJobLocale } from "@/lib/i18n/job-locale"; import { annotate, getEvent } from "@/lib/logging/context"; import { TRACKED_INTAKE_WHERE } from "@/lib/medications/intake-tracking"; -/** Pg-boss queue + cron — imported by the reminder worker's bootstrap. */ -export const MEDICATION_LOW_STOCK_QUEUE = "medication-low-stock"; -/** - * 09:00 Europe/Berlin — a supply alert is an errand prompt ("order a - * refill"), not a night signal; it fires once daily at a time the user - * can act on it. - */ -export const MEDICATION_LOW_STOCK_CRON = "0 9 * * *"; - /** * An as-needed medication warns when this many doses or fewer are left: no * cadence says how many days that is, so the floor is a dose count. diff --git a/src/lib/jobs/mood-prognosis-refresh-queue.ts b/src/lib/jobs/mood-prognosis-refresh-queue.ts new file mode 100644 index 000000000..41f75c78c --- /dev/null +++ b/src/lib/jobs/mood-prognosis-refresh-queue.ts @@ -0,0 +1,8 @@ +/** + * The queue facts of `mood-prognosis-refresh.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const MOOD_PROGNOSIS_REFRESH_QUEUE = "mood-prognosis-refresh"; + +export const MOOD_PROGNOSIS_REFRESH_CRON = "35 4 * * *"; diff --git a/src/lib/jobs/mood-prognosis-refresh.ts b/src/lib/jobs/mood-prognosis-refresh.ts index 30ab95fba..e63ff411a 100644 --- a/src/lib/jobs/mood-prognosis-refresh.ts +++ b/src/lib/jobs/mood-prognosis-refresh.ts @@ -50,10 +50,6 @@ import { isModuleEnabled } from "@/lib/modules/gate"; import { getWorkerPrisma } from "./reminder/shared"; -export const MOOD_PROGNOSIS_REFRESH_QUEUE = "mood-prognosis-refresh"; - -export const MOOD_PROGNOSIS_REFRESH_CRON = "35 4 * * *"; - export interface MoodPrognosisRefreshPayload { triggeredAt: string; } diff --git a/src/lib/jobs/offhost-backup-queue.ts b/src/lib/jobs/offhost-backup-queue.ts new file mode 100644 index 000000000..13e6a0353 --- /dev/null +++ b/src/lib/jobs/offhost-backup-queue.ts @@ -0,0 +1,27 @@ +/** + * The queue facts of `offhost-backup.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy"; + +/** + * The off-host pass's expiry: four hours. It uploads every account and + * already stopped between accounts at three quarters of the default fifteen + * minutes, which on an instance of any size meant a retry per night; one + * large account past the default was retried beside itself. Four hours with + * the same budget and a lock (`lockedPass`) leaves room for the night's pass. + */ +export const OFFHOST_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60; + +export const OFFHOST_BACKUP_QUEUE = "data-backup-offhost"; + +/** + * What the nightly schedule sends with: the four hours above, and the + * heartbeat that notices a process that died under the pass within minutes + * rather than at the end of them (`BACKUP_HEARTBEAT_SECONDS`). + */ +export const OFFHOST_BACKUP_SEND_OPTIONS = { + expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS, + heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS, +} as const; diff --git a/src/lib/jobs/offhost-backup.ts b/src/lib/jobs/offhost-backup.ts index 1016ce032..cf6fbf1ba 100644 --- a/src/lib/jobs/offhost-backup.ts +++ b/src/lib/jobs/offhost-backup.ts @@ -41,35 +41,14 @@ import { orderInterruptedLast, readAccountsToTakeLast, } from "@/lib/jobs/backup-pass-attempts"; -import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy"; import { annotate, getEvent } from "@/lib/logging/context"; import { envValue } from "@/lib/env"; +import { OFFHOST_BACKUP_QUEUE } from "./offhost-backup-queue"; const ALGORITHM = "aes-256-gcm"; const IV_LENGTH = 12; const TAG_LENGTH = 16; -/** - * The off-host pass's expiry: four hours. It uploads every account and - * already stopped between accounts at three quarters of the default fifteen - * minutes, which on an instance of any size meant a retry per night; one - * large account past the default was retried beside itself. Four hours with - * the same budget and a lock (`lockedPass`) leaves room for the night's pass. - */ -export const OFFHOST_BACKUP_EXPIRE_SECONDS = 4 * 60 * 60; - -export const OFFHOST_BACKUP_QUEUE = "data-backup-offhost"; - -/** - * What the nightly schedule sends with: the four hours above, and the - * heartbeat that notices a process that died under the pass within minutes - * rather than at the end of them (`BACKUP_HEARTBEAT_SECONDS`). - */ -export const OFFHOST_BACKUP_SEND_OPTIONS = { - expireInSeconds: OFFHOST_BACKUP_EXPIRE_SECONDS, - heartbeatSeconds: BACKUP_HEARTBEAT_SECONDS, -} as const; - export interface OffhostBackupConfig { endpoint: string; bucket: string; diff --git a/src/lib/jobs/offhost-purge-queue.ts b/src/lib/jobs/offhost-purge-queue.ts new file mode 100644 index 000000000..4630131a5 --- /dev/null +++ b/src/lib/jobs/offhost-purge-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `offhost-purge.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge"; + +/** 03:40, after the nightly upload (02:30) has had its hour. */ +export const OFFHOST_PURGE_CRON = "40 3 * * *"; diff --git a/src/lib/jobs/offhost-purge.ts b/src/lib/jobs/offhost-purge.ts index cc08a3acc..2cdf06cd8 100644 --- a/src/lib/jobs/offhost-purge.ts +++ b/src/lib/jobs/offhost-purge.ts @@ -46,10 +46,7 @@ import { reportWorkerError } from "@/lib/jobs/report-worker-error"; import { getWorkerPrisma } from "@/lib/jobs/reminder/shared"; import { withBackgroundEvent } from "@/lib/logging/background"; import { logCaught } from "@/lib/logging/signal"; - -export const OFFHOST_PURGE_QUEUE = "offhost-backup-purge"; -/** 03:40, after the nightly upload (02:30) has had its hour. */ -export const OFFHOST_PURGE_CRON = "40 3 * * *"; +import { OFFHOST_PURGE_QUEUE } from "./offhost-purge-queue"; export type OffhostPurgeReason = "account_deleted" | "data_wiped" | "managed_profile_deleted"; diff --git a/src/lib/jobs/oura-backfill-queue.ts b/src/lib/jobs/oura-backfill-queue.ts new file mode 100644 index 000000000..47fdb4c66 --- /dev/null +++ b/src/lib/jobs/oura-backfill-queue.ts @@ -0,0 +1,9 @@ +/** + * The queue facts of `oura-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const OURA_BACKFILL_QUEUE = "oura-backfill"; + +/** One account at a time; the shared lane bounds the instance. */ +export const OURA_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/oura-backfill.ts b/src/lib/jobs/oura-backfill.ts index 6fad28bef..413267412 100644 --- a/src/lib/jobs/oura-backfill.ts +++ b/src/lib/jobs/oura-backfill.ts @@ -68,11 +68,7 @@ import { withoutDenied, type OuraBackfillProgress, } from "@/lib/oura/backfill-progress"; - -export const OURA_BACKFILL_QUEUE = "oura-backfill"; - -/** One account at a time; the shared lane bounds the instance. */ -export const OURA_BACKFILL_CONCURRENCY = 1; +import { OURA_BACKFILL_QUEUE } from "./oura-backfill-queue"; /** * One slice: five minutes of walking, then the account is let go and the diff --git a/src/lib/jobs/period-narrative-queue.ts b/src/lib/jobs/period-narrative-queue.ts new file mode 100644 index 000000000..5628280d9 --- /dev/null +++ b/src/lib/jobs/period-narrative-queue.ts @@ -0,0 +1,24 @@ +/** + * The queue facts of `period-narrative-shared.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const PERIOD_NARRATIVE_QUEUE = "period-narrative-warm"; + +/** + * The job's expiry: three hours. On a boundary night the scheduled pass + * writes up to two narratives for each of 200 accounts, which can outlast + * pg-boss's fifteen-minute default. The pass stops itself at three quarters + * of this (`jobBudget`) and holds a lock (`lockedPass`); every send and the + * schedule carry it. + */ +export const PERIOD_NARRATIVE_EXPIRE_SECONDS = 3 * 60 * 60; + +/** + * Nightly at 05:05 Europe/Berlin — inside the existing maintenance window, + * after the comprehensive insight pre-generation (04:30) and the computed + * scores (04:45–04:55) so the rollup signals it reads are already folded. + * The handler only does real work near a week/month boundary; on every other + * night it short-circuits cheaply. + */ +export const PERIOD_NARRATIVE_CRON = "5 5 * * *"; diff --git a/src/lib/jobs/period-narrative-shared.ts b/src/lib/jobs/period-narrative-shared.ts index 788ab0258..ac43bb66d 100644 --- a/src/lib/jobs/period-narrative-shared.ts +++ b/src/lib/jobs/period-narrative-shared.ts @@ -21,26 +21,10 @@ import { import type { NarrativePeriod } from "@/lib/insights/narrative/period-narrative"; import type { Locale } from "@/lib/i18n/config"; import { logCaught } from "@/lib/logging/signal"; - -export const PERIOD_NARRATIVE_QUEUE = "period-narrative-warm"; - -/** - * The job's expiry: three hours. On a boundary night the scheduled pass - * writes up to two narratives for each of 200 accounts, which can outlast - * pg-boss's fifteen-minute default. The pass stops itself at three quarters - * of this (`jobBudget`) and holds a lock (`lockedPass`); every send and the - * schedule carry it. - */ -export const PERIOD_NARRATIVE_EXPIRE_SECONDS = 3 * 60 * 60; - -/** - * Nightly at 05:05 Europe/Berlin — inside the existing maintenance window, - * after the comprehensive insight pre-generation (04:30) and the computed - * scores (04:45–04:55) so the rollup signals it reads are already folded. - * The handler only does real work near a week/month boundary; on every other - * night it short-circuits cheaply. - */ -export const PERIOD_NARRATIVE_CRON = "5 5 * * *"; +import { + PERIOD_NARRATIVE_QUEUE, + PERIOD_NARRATIVE_EXPIRE_SECONDS, +} from "./period-narrative-queue"; export interface PeriodNarrativePayload { /** Set on the nightly scheduled tick (informational). */ diff --git a/src/lib/jobs/period-narrative-warm.ts b/src/lib/jobs/period-narrative-warm.ts index d162d8506..51aa05dd9 100644 --- a/src/lib/jobs/period-narrative-warm.ts +++ b/src/lib/jobs/period-narrative-warm.ts @@ -37,11 +37,13 @@ import type { NarrativePeriod } from "@/lib/insights/narrative/period-narrative" import { defaultLocale } from "@/lib/i18n/config"; import { resolveJobLocale } from "@/lib/i18n/job-locale"; import { - PERIOD_NARRATIVE_QUEUE, - PERIOD_NARRATIVE_CRON, enqueueNarrativeWarm, type PeriodNarrativePayload, } from "@/lib/jobs/period-narrative-shared"; +import { + PERIOD_NARRATIVE_QUEUE, + PERIOD_NARRATIVE_CRON, +} from "@/lib/jobs/period-narrative-queue"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; export { PERIOD_NARRATIVE_QUEUE, PERIOD_NARRATIVE_CRON, enqueueNarrativeWarm }; diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 80b1feeb1..071481bfe 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -6,7 +6,7 @@ * it, and it does not stop the handler, so a job that legitimately needs longer * ends up running twice beside itself. No queue policy prevents that: the * policies constrain job rows, and an expired row is no longer active. Every - * queue bound with `createAndWork` is therefore declared here as one of: + * queue the worker registry declares is therefore declared here as one of: * * - `short`: a job cannot come near fifteen minutes (one account's bounded * window, one document, a forwarder, a bounded delete). It relies on the @@ -28,15 +28,15 @@ import { INTEGRATION_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/integration-backfill-admission"; import { NIGHTLY_INSIGHT_PASS_EXPIRE_SECONDS } from "@/lib/jobs/reminder/registrar-shared"; import { INSIGHT_PREGENERATE_EXPIRE_SECONDS } from "@/lib/jobs/insight-pregenerate-shared"; -import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "@/lib/jobs/period-narrative-shared"; +import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "@/lib/jobs/period-narrative-queue"; import { PR_DETECTION_EXPIRE_SECONDS } from "@/lib/jobs/pr-detection"; import { ENCRYPTION_KEY_ROTATE_EXPIRE_SECONDS } from "@/lib/jobs/encryption-key-rotate"; import { CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS } from "@/lib/jobs/document-content-index-backfill"; import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run"; import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy"; -import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup"; +import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup-queue"; import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore"; -import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill"; +import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill-queue"; import { APPLE_HEALTH_IMPORT_SEND_OPTIONS } from "@/lib/jobs/apple-health-import-worker"; import { MEDICATION_INTAKE_IMPORT_SEND_OPTIONS } from "@/lib/jobs/medication-intake-import"; import { HEALTH_CONNECT_IMPORT_SEND_OPTIONS } from "@/lib/jobs/health-connect-import-worker"; @@ -47,8 +47,8 @@ import { ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS } from "@/lib/rollups/measurement-r /** pg-boss's default job expiry, in seconds. */ export const DEFAULT_JOB_EXPIRE_SECONDS = 15 * 60; -/** Where a check lives: `binding` is the `createAndWork` call itself. */ -export type SourceSite = "binding" | { file: string; fn: string }; +/** Where a check lives: a function (or const) in a file under `src/`. */ +export type SourceSite = { file: string; fn: string }; export type QueueRuntime = | { runtime: "short"; why: string } @@ -64,8 +64,9 @@ export type QueueRuntime = /** Where the handler stops on the job's budget, or `oneShot`. */ stop: SourceSite | { oneShot: string }; /** - * What keeps a second run from working beside this one: `lockedPass` - * at the binding, or a marker in the named function. + * What keeps a second run from working beside this one: the `lock` + * its registry definition declares (`lockedPass`), or a marker in the + * named function. */ exclusive: "lockedPass" | { marker: string; at: SourceSite }; why: string; @@ -177,7 +178,10 @@ export const QUEUE_RUNTIME: Readonly> = { runtime: "long", expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS, expiryVia: ["PERIOD_NARRATIVE_EXPIRE_SECONDS"], - stop: "binding", + stop: { + file: "lib/jobs/reminder/status-handlers.ts", + fn: "handlePeriodNarrative", + }, exclusive: "lockedPass", why: "On a boundary night up to 200 accounts with a provider call per period each; a single-account warm is short.", }, @@ -198,7 +202,10 @@ export const QUEUE_RUNTIME: Readonly> = { runtime: "long", expireInSeconds: ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS, expiryVia: ["ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS"], - stop: "binding", + stop: { + file: "lib/jobs/reminder/rollup-handlers.ts", + fn: "handleRollupFullBackfill", + }, exclusive: "lockedPass", why: "One account's five-year fold: four aggregates under the statement timeout and four persist transactions of up to 120 s each, about thirteen minutes at worst under the defaults and unbounded when the operator lifts the statement timeout.", }, @@ -220,22 +227,34 @@ export const QUEUE_RUNTIME: Readonly> = { ), "mean-consolidation": { runtime: "budgeted", - budget: "binding", + budget: { + file: "lib/jobs/reminder/rollup-handlers.ts", + fn: "handleMeanConsolidation", + }, why: "One account's whole history, a day at a time; stops at three quarters of the default expiry and resumes.", }, "dense-intraday-retention": { runtime: "budgeted", - budget: "binding", + budget: { + file: "lib/jobs/reminder/rollup-handlers.ts", + fn: "handleDenseIntradayRetention", + }, why: "A dense account's history, a day at a time; stops at three quarters of the default expiry and resumes.", }, "dense-intraday-hourly-rebuild": { runtime: "budgeted", - budget: "binding", + budget: { + file: "lib/jobs/reminder/rollup-handlers.ts", + fn: "handleDenseIntradayHourlyRebuild", + }, why: "A dense account's hourly rows, a day at a time; stops at three quarters of the default expiry and resumes.", }, "drain-per-sample-cumulative": { runtime: "budgeted", - budget: "binding", + budget: { + file: "lib/jobs/reminder/rollup-handlers.ts", + fn: "drainCumulativeHandler", + }, why: "Every account's per-sample drains; stops at three quarters of the default expiry and hands the rest to a continuation job.", }, @@ -471,7 +490,10 @@ export const QUEUE_RUNTIME: Readonly> = { runtime: "long", expireInSeconds: CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS, expiryVia: ["CONTENT_INDEX_BACKFILL_EXPIRE_SECONDS"], - stop: "binding", + stop: { + file: "lib/jobs/reminder/document-handlers.ts", + fn: "handleContentIndexBackfill", + }, exclusive: "lockedPass", why: "Up to 200 documents of one account, a provider transcription each.", }, diff --git a/src/lib/jobs/recovery-score-queue.ts b/src/lib/jobs/recovery-score-queue.ts new file mode 100644 index 000000000..181e66c5f --- /dev/null +++ b/src/lib/jobs/recovery-score-queue.ts @@ -0,0 +1,14 @@ +/** + * The queue facts of `recovery-score.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const RECOVERY_SCORE_QUEUE = "recovery-score-compute"; + +/** + * Daily at 04:45 Europe/Berlin — inside the existing 03:xx–04:xx maintenance + * window, after the cumulative drain (03:45) + the daily-mean consolidation + * and the insight pre-generation (04:30) so the nightly signals it reads are + * already folded into the rollup tier by the time it runs. + */ +export const RECOVERY_SCORE_CRON = "45 4 * * *"; diff --git a/src/lib/jobs/recovery-score.ts b/src/lib/jobs/recovery-score.ts index cc2a2fb2f..c0a4a796f 100644 --- a/src/lib/jobs/recovery-score.ts +++ b/src/lib/jobs/recovery-score.ts @@ -24,16 +24,6 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { persistRecoveryScore } from "@/lib/insights/recovery-score"; import { runScoreBatch } from "@/lib/insights/score-row"; -export const RECOVERY_SCORE_QUEUE = "recovery-score-compute"; - -/** - * Daily at 04:45 Europe/Berlin — inside the existing 03:xx–04:xx maintenance - * window, after the cumulative drain (03:45) + the daily-mean consolidation - * and the insight pre-generation (04:30) so the nightly signals it reads are - * already folded into the rollup tier by the time it runs. - */ -export const RECOVERY_SCORE_CRON = "45 4 * * *"; - /** * Per-run user cap. A nightly fan-out across every user could starve the * pool; the discovery query is ordered so users with the most recent vital diff --git a/src/lib/jobs/registry/__tests__/registry-contract.test.ts b/src/lib/jobs/registry/__tests__/registry-contract.test.ts new file mode 100644 index 000000000..afcbdc995 --- /dev/null +++ b/src/lib/jobs/registry/__tests__/registry-contract.test.ts @@ -0,0 +1,129 @@ +/** + * The contract of the worker registry, checked over every declaration. + * + * The dead-queue class (a queue named but never created, scheduled or + * drained) cannot be written any more: a definition is the queue, its cron, + * its policy and its handler in one object, and `registerJobGroup` turns each + * into all four. What is left to check is what the type system cannot see: + * + * - every queue is declared once, under a well-formed name, and no retired + * queue is declared again; + * - every cron has five fields, every policy names one this codebase uses + * and says why; + * - the groups register in the order the worker always used; + * - loading the registry loads none of the work: its static import graph + * stays inside a small budget and reaches no handler module. A handler is + * named only through its definition's dynamic `import("…")`. + * + * What pg-boss is left holding after registration is pinned byte for byte by + * `tests/integration/queue-registration-parity.test.ts`, which also loads + * every handler once. + */ +import { describe, expect, it } from "vitest"; + +import { staticImportClosure } from "@/__tests__/helpers/source-metrics"; +import { JOB_GROUP_NAMES } from "../define-job"; +import { allJobDefinitions, JOB_GROUPS } from "../index"; + +const definitions = allJobDefinitions(); + +/** Modules the registry may not load: the work the definitions point at. */ +const HANDLER_GRAPH = + /^lib\/(?:jobs\/reminder\/(?!registrar-shared\.ts|shared\.ts)|ai\/|insights\/|withings\/|whoop\/|oura\/|polar\/|strava\/|google-health\/|nightscout\/|dawarich\/|documents\/|export\/|import\/|measurements\/import)/; + +/** + * Light modules inside those folders the registry may load, with the reason. + */ +const ALLOWED: Readonly> = { + "lib/ai/coach/coach-memory-shared.ts": + "the Coach memory queue's name and payload, with its byte codec; no model code", + "lib/ai/coach/bytes-codec.ts": "pulled by coach-memory-shared; a codec", + "lib/export/restore-lock.ts": + "the per-account restore lock the consolidation passes take; a lock primitive", + "lib/export/restore-failure-cause.ts": + "pulled by restore-lock; an error type", +}; + +/** + * The registry's static graph: the declarations, the light queue-fact + * modules and the boot infrastructure they share (the database client, the + * logger, the lane and the lock). Measured at 288 on 2026-10-23, against the + * 1,164 modules the five hand-written registrars loaded; it may only fall. + */ +const REGISTRY_GRAPH_BUDGET = 300; + +describe("worker registry", () => { + it("declares every queue once, under a well-formed name", () => { + expect(definitions.length).toBeGreaterThan(100); + const names = definitions.map((d) => d.queue); + const twice = names.filter((n, i) => names.indexOf(n) !== i); + expect(twice).toEqual([]); + expect(names.filter((n) => !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(n))).toEqual( + [], + ); + }); + + it("declares no retired queue again", () => { + const declared = new Set(definitions.map((d) => d.queue)); + const retired = JOB_GROUPS.flatMap((g) => g.retired ?? []); + expect(retired.length).toBeGreaterThan(0); + expect(retired.filter((q) => declared.has(q))).toEqual([]); + }); + + it("gives every cron five fields and every policy a known kind and a reason", () => { + const crons = definitions.filter((d) => d.cron); + expect(crons.length).toBeGreaterThan(60); + expect( + crons + .filter((d) => d.cron!.trim().split(/\s+/).length !== 5) + .map((d) => d.queue), + ).toEqual([]); + const policies = definitions.filter((d) => d.policy); + expect(policies.length).toBeGreaterThan(30); + expect( + policies + .filter( + (d) => + !["short", "exclusive"].includes(d.policy!.policy) || + d.policy!.reason.length < 30, + ) + .map((d) => d.queue), + ).toEqual([]); + }); + + it("binds every queue with an explicit concurrency", () => { + expect( + definitions + .filter( + (d) => + !Number.isInteger(d.work.localConcurrency) || + (d.work.localConcurrency ?? 0) < 1, + ) + .map((d) => d.queue), + ).toEqual([]); + }); + + it("registers the groups in the worker's order", () => { + expect(JOB_GROUPS.map((g) => g.name)).toEqual([...JOB_GROUP_NAMES]); + }); + + it("loads none of the work when the registry loads", () => { + const graph = staticImportClosure(["lib/jobs/registry/index.ts"]); + // A floor, so a walk that resolves nothing cannot pass. + expect(graph.size).toBeGreaterThan(20); + expect(graph.has("lib/jobs/registry/maintenance.ts")).toBe(true); + expect( + [...graph].filter((m) => HANDLER_GRAPH.test(m) && !(m in ALLOWED)).sort(), + ).toEqual([]); + expect(graph.size).toBeLessThanOrEqual(REGISTRY_GRAPH_BUDGET); + }); + + it("catches a handler module in the registry graph (planted)", () => { + // The same check over the graph a registrar that imported its handlers + // would have. + const graph = staticImportClosure(["lib/jobs/reminder/status-handlers.ts"]); + expect( + [...graph].filter((m) => HANDLER_GRAPH.test(m)).length, + ).toBeGreaterThan(0); + }); +}); diff --git a/src/lib/jobs/registry/__tests__/registry-helpers.ts b/src/lib/jobs/registry/__tests__/registry-helpers.ts new file mode 100644 index 000000000..eba79f863 --- /dev/null +++ b/src/lib/jobs/registry/__tests__/registry-helpers.ts @@ -0,0 +1,61 @@ +/** + * Reading the worker registry from a test: the declaration of one queue, the + * group that holds it, the handler its loader resolves to, and a group's boot + * discovery passes. Every helper fails loudly on a queue the registry does + * not declare, so a test cannot pass by asserting on nothing. + */ +import type { PgBoss } from "pg-boss"; + +import { JOB_GROUPS } from "../index"; +import type { JobDefinition, JobGroup, JobGroupName } from "../define-job"; + +export function jobOf(queue: string): JobDefinition { + for (const group of JOB_GROUPS) { + const job = group.jobs.find((j) => j.queue === queue); + if (job) return job; + } + throw new Error(`the registry declares no queue "${queue}"`); +} + +export function groupOf(queue: string): JobGroup { + const group = JOB_GROUPS.find((g) => g.jobs.some((j) => j.queue === queue)); + if (!group) throw new Error(`the registry declares no queue "${queue}"`); + return group; +} + +export function groupNamed(name: JobGroupName): JobGroup { + const group = JOB_GROUPS.find((g) => g.name === name); + if (!group) throw new Error(`no registry group "${name}"`); + return group; +} + +/** The names of a group's boot discovery passes, in run order. */ +export function discoveryOf(name: JobGroupName): string[] { + return (groupNamed(name).discovery ?? []).map((pass) => pass.name); +} + +/** + * A boss that records sends and refuses everything else, for loaders that + * close over it (the forwarders, the drain's follow-up). + */ +export function recordingBoss(): { + boss: PgBoss; + sends: Array<{ queue: string; data: unknown; options: unknown }>; +} { + const sends: Array<{ queue: string; data: unknown; options: unknown }> = []; + const boss = { + send: async (queue: string, data: unknown, options: unknown) => { + sends.push({ queue, data, options }); + return `job-${sends.length}`; + }, + } as unknown as PgBoss; + return { boss, sends }; +} + +/** The handler the queue's loader resolves to (its module is loaded). */ +export async function loadedHandler( + queue: string, + boss: PgBoss = recordingBoss().boss, +): Promise { + return jobOf(queue).load({ boss }); +} diff --git a/src/lib/jobs/registry/define-job.ts b/src/lib/jobs/registry/define-job.ts new file mode 100644 index 000000000..acaa67254 --- /dev/null +++ b/src/lib/jobs/registry/define-job.ts @@ -0,0 +1,109 @@ +/** + * The declaration of one pg-boss queue: everything the worker needs to + * provision, schedule and drain it, in one object. + * + * Every queue the worker runs is a `defineJob(...)` entry in one of the + * registry groups (`./integration-sync.ts`, `./status.ts`, `./rollup.ts`, + * `./reminders.ts`, `./maintenance.ts`). `registerJobGroup` turns a group into + * the four facts the dead-queue class keeps losing: the queue exists, its cron + * is scheduled with the send options it carries, its de-duplication policy is + * set, and a handler drains it. A queue cannot be declared without all four. + * + * The handler is not imported here. `load` names the module it lives in with + * a dynamic import, and the worker evaluates it when the queue's first job + * arrives, so booting the worker loads the registry and nothing of the work. + * The facts a declaration needs at boot (names, crons, concurrency, send + * options) therefore come from light modules (`*-queue.ts`, `*-shared.ts`), + * never from a handler module; `job-registry-graph.test.ts` holds that. + */ +import type { Job, PgBoss, WorkOptions } from "pg-boss"; + +import type { JobOutcome } from "@/lib/jobs/job-outcome"; +import type { QueuePolicyDecision } from "@/lib/jobs/reminder/registrar-shared"; + +/** The groups, in the order the worker registers them. */ +export const JOB_GROUP_NAMES = [ + "integration_sync", + "status", + "rollup", + "reminders", + "maintenance", +] as const; + +export type JobGroupName = (typeof JOB_GROUP_NAMES)[number]; + +/** What a handler loader is handed: the worker's own boss, for re-sends. */ +export interface JobContext { + readonly boss: PgBoss; +} + +/** A handler as pg-boss calls it: a batch of jobs, one outcome. */ +export type BatchHandler> = ( + jobs: J[], +) => Promise; + +export interface JobSpec> { + readonly queue: string; + /** The cron that feeds the queue, in the default zone. */ + readonly cron?: string; + /** + * Send options every job the cron mints carries (retries, expiry). They ride + * on the schedule because `createQueue` cannot change an existing queue. + */ + readonly scheduleOptions?: Readonly>; + /** De-duplication policy; absent keeps pg-boss's `standard`. */ + readonly policy?: QueuePolicyDecision; + /** Work options of the binding (concurrency, metadata, poll). */ + readonly work: WorkOptions; + /** + * One run per identity at a time (`lockedPass`), held outside the lane so a + * retry pg-boss starts after an expiry finds the run still going. + */ + readonly lock?: (job: J) => string; + /** + * The job runs in the heavy lane (`heavyLane`): `true` for every job, or a + * filter of the jobs that are heavy on a queue that also carries light ones. + */ + readonly lane?: true | ((job: J) => boolean); + /** Loads the handler that drains the queue. */ + readonly load: (ctx: JobContext) => Promise>; +} + +/** A declaration with its payload type erased, as the registry holds it. */ +export type JobDefinition = JobSpec>; + +export function defineJob>( + spec: JobSpec, +): JobDefinition { + // The payload type is the declaration's own business; the registry only + // passes jobs through to the handler that declared it. + return spec as unknown as JobDefinition; +} + +/** What a boot discovery pass reports, as the backfill helpers return it. */ +export interface DiscoveryResult { + /** How many jobs it enqueued, or whether it enqueued its one job. */ + readonly enqueued: number | boolean; + readonly skipped?: number; + readonly error?: string | null; +} + +/** + * A self-converging pass the worker runs once at boot (after every queue is + * bound): it finds the accounts that still need a backfill or repair and + * enqueues one job each. `name` is the log prefix. + */ +export interface BootDiscovery { + readonly name: string; + readonly run: () => Promise; +} + +export interface JobGroup { + readonly name: JobGroupName; + readonly jobs: readonly JobDefinition[]; + /** Queues an older release created and this one removes, with whatever they hold. */ + readonly retired?: readonly string[]; + /** Work that needs the group's handlers bound, before the boot discovery. */ + readonly afterBind?: (ctx: JobContext) => Promise; + readonly discovery?: readonly BootDiscovery[]; +} diff --git a/src/lib/jobs/registry/index.ts b/src/lib/jobs/registry/index.ts new file mode 100644 index 000000000..a6d02a764 --- /dev/null +++ b/src/lib/jobs/registry/index.ts @@ -0,0 +1,55 @@ +/** + * The worker registry: every queue the worker runs, by group, in the order + * the worker registers them. `reminder-worker.ts` walks it at boot; + * `tests/integration/queue-registration-parity.test.ts` holds what it leaves + * in pg-boss to the golden. + */ +import type { PgBoss } from "pg-boss"; + +import type { JobDefinition, JobGroup } from "./define-job"; +import { INTEGRATION_SYNC_GROUP } from "./integration-sync"; +import { MAINTENANCE_GROUP } from "./maintenance"; +import { REMINDERS_GROUP } from "./reminders"; +import { registerJobGroup, runBootDiscovery } from "./register"; +import { ROLLUP_GROUP } from "./rollup"; +import { STATUS_GROUP } from "./status"; + +export const JOB_GROUPS: readonly JobGroup[] = [ + INTEGRATION_SYNC_GROUP, + STATUS_GROUP, + ROLLUP_GROUP, + REMINDERS_GROUP, + MAINTENANCE_GROUP, +]; + +/** Every definition, across the groups. */ +export function allJobDefinitions(): readonly JobDefinition[] { + return JOB_GROUPS.flatMap((group) => group.jobs); +} + +/** + * Register every group in order. `onGroup` runs before each one, so the boot + * can time the groups as steps. Returns every queue name created. + */ +export async function registerAllJobGroups( + boss: PgBoss, + onGroup: (group: JobGroup) => void = () => {}, +): Promise { + const queues: string[] = []; + for (const group of JOB_GROUPS) { + onGroup(group); + queues.push(...(await registerJobGroup(boss, group))); + } + return queues; +} + +/** + * Run the boot discovery of every group: rollup first, then the integration + * backfills, then maintenance, the order the worker always used. + */ +export async function runAllBootDiscovery(): Promise { + for (const name of ["rollup", "integration_sync", "maintenance"] as const) { + const group = JOB_GROUPS.find((g) => g.name === name); + if (group?.discovery) await runBootDiscovery(group.discovery); + } +} diff --git a/src/lib/jobs/registry/integration-sync.ts b/src/lib/jobs/registry/integration-sync.ts new file mode 100644 index 000000000..d5ec529a3 --- /dev/null +++ b/src/lib/jobs/registry/integration-sync.ts @@ -0,0 +1,391 @@ +/** + * The third-party provider sync queues: Withings, WHOOP, Fitbit, Google + * Health, Nightscout, Polar, Oura, Strava, the full-history backfills and + * repairs that run through the shared admission lane, the integration + * watchdog, and the OAuth-state and handoff prunes. + * + * The recurring polls are keyless cron ticks, so they carry no policy. The + * one-shot backfills and repairs share one shape: one job per connection, + * keyed per user, enqueued by a boot discovery that only stops offering the + * job once it has completed. `exclusive` covers queued, active and + * retry-backoff, so a worker restarting during a long pass cannot append a + * second identical job, and since discovery re-offers outstanding work a + * suppressed send never loses any. + * + * The prunes carry `cronIsTheRetry`: a DELETE that could not run will not run + * better a second later, and the daily tick is the retry. The sync crons keep + * the default retries; a provider outage is transient. + */ +import { + FITBIT_BOOT_DISCOVERY, + FITBIT_JOBS, +} from "@/lib/fitbit/jobs/definitions"; +import { + INTEGRATION_BACKFILL_ADMISSION_QUEUE, + INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, + bootStaggerSecondsFor, + type IntegrationBackfillKind, +} from "@/lib/jobs/integration-backfill-admission"; +import { + GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON, + GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, + GOOGLE_HEALTH_SYNC_CRON, + GOOGLE_HEALTH_SYNC_QUEUE, + NIGHTSCOUT_SYNC_CRON, + NIGHTSCOUT_SYNC_QUEUE, + OIDC_NATIVE_HANDOFF_CLEANUP_CRON, + OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, + OURA_SYNC_CRON, + OURA_SYNC_QUEUE, + POLAR_SYNC_CRON, + POLAR_SYNC_QUEUE, + STRAVA_SYNC_CRON, + STRAVA_SYNC_QUEUE, + WHOOP_CYCLE_SYNC_CRON, + WHOOP_CYCLE_SYNC_QUEUE, + WHOOP_OAUTH_STATE_CLEANUP_CRON, + WHOOP_OAUTH_STATE_CLEANUP_QUEUE, + WHOOP_RECOVERY_SYNC_CRON, + WHOOP_RECOVERY_SYNC_QUEUE, + WHOOP_SLEEP_SYNC_CRON, + WHOOP_SLEEP_SYNC_QUEUE, + WHOOP_WORKOUT_SYNC_CRON, + WHOOP_WORKOUT_SYNC_QUEUE, + WITHINGS_ACTIVITY_CRON, + WITHINGS_ACTIVITY_QUEUE, + WITHINGS_ECG_SYNC_CRON, + WITHINGS_OAUTH_STATE_CLEANUP_CRON, + WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, + WITHINGS_SLEEP_CRON, + WITHINGS_SLEEP_QUEUE, + WITHINGS_SYNC_CRON, + WITHINGS_SYNC_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; +import { + INTEGRATION_WATCHDOG_DAILY_CRON, + INTEGRATION_WATCHDOG_DAILY_QUEUE, + INTEGRATION_WATCHDOG_HOURLY_CRON, + INTEGRATION_WATCHDOG_HOURLY_QUEUE, +} from "@/lib/jobs/integration-watchdog-queue"; +import { + GOOGLE_HEALTH_BACKFILL_CONCURRENCY, + GOOGLE_HEALTH_BACKFILL_QUEUE, +} from "@/lib/jobs/google-health-backfill-queue"; +import { + GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY, + GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, +} from "@/lib/jobs/google-health-sleep-repair-queue"; +import { + LAB_BIOMARKER_BACKFILL_CONCURRENCY, + LAB_BIOMARKER_BACKFILL_QUEUE, +} from "@/lib/jobs/lab-biomarker-backfill"; +import { + OURA_BACKFILL_CONCURRENCY, + OURA_BACKFILL_QUEUE, +} from "@/lib/jobs/oura-backfill-queue"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + SLEEP_TIMELINE_BACKFILL_CONCURRENCY, + SLEEP_TIMELINE_BACKFILL_QUEUE, +} from "@/lib/jobs/sleep-timeline-backfill-queue"; +import { + STRAVA_BACKFILL_CONCURRENCY, + STRAVA_BACKFILL_QUEUE, +} from "@/lib/jobs/strava-backfill-queue"; +import { + WHOOP_BACKFILL_CONCURRENCY, + WHOOP_BACKFILL_QUEUE, +} from "@/lib/jobs/whoop-backfill-queue"; +import { WITHINGS_ECG_SYNC_QUEUE } from "@/lib/jobs/withings-ecg-queue"; + +import { + defineJob, + type BootDiscovery, + type DiscoveryResult, + type JobDefinition, + type JobGroup, +} from "./define-job"; + +const SELF_CONVERGING_BACKFILL = + "Per-connection full-history backfill; boot-discovery driven and self-converging."; + +/** A source queue that forwards each job to the shared admission lane. */ +function backfillSource( + queue: string, + concurrency: number, + kind: IntegrationBackfillKind, + reason: string = SELF_CONVERGING_BACKFILL, +): JobDefinition { + return defineJob({ + queue, + policy: { policy: "exclusive", reason }, + work: { localConcurrency: concurrency }, + load: async ({ boss }) => + ( + await import("@/lib/jobs/integration-backfill-forward") + ).forwardToBackfillAdmission(boss, kind), + }); +} + +const withingsSync = () => import("@/lib/jobs/reminder/withings-sync"); +const whoopSync = () => import("@/lib/jobs/reminder/whoop-sync"); +const googleHealthSync = () => import("@/lib/jobs/reminder/google-health-sync"); +const cleanups = () => import("@/lib/jobs/reminder/cleanup-handlers"); + +const jobs: readonly JobDefinition[] = [ + // pg-boss scopes groupConcurrency to one queue name, so every provider + // reaches this shared queue before doing full-history work. The group is + // enforced in Postgres across workers; the heavy lane keeps an import off + // the heap while a backup, a restore or an upload is on it. + defineJob({ + queue: INTEGRATION_BACKFILL_ADMISSION_QUEUE, + policy: { + policy: "exclusive", + reason: + "Shared full-history drain; keyed per source job so boot rediscovery cannot duplicate queued, active, or retrying work.", + }, + work: { + localConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, + groupConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, + }, + lane: true, + load: async () => + (await import("@/lib/jobs/integration-backfill-drain")) + .drainIntegrationBackfillAdmission, + }), + defineJob({ + queue: WITHINGS_SYNC_QUEUE, + cron: WITHINGS_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await withingsSync()).handleWithingsFallbackSync, + }), + defineJob({ + queue: WITHINGS_ACTIVITY_QUEUE, + cron: WITHINGS_ACTIVITY_CRON, + work: { localConcurrency: 1 }, + load: async () => (await withingsSync()).handleWithingsActivitySync, + }), + defineJob({ + queue: WITHINGS_SLEEP_QUEUE, + cron: WITHINGS_SLEEP_CRON, + work: { localConcurrency: 1 }, + load: async () => (await withingsSync()).handleWithingsSleepSync, + }), + defineJob({ + queue: WITHINGS_ECG_SYNC_QUEUE, + cron: WITHINGS_ECG_SYNC_CRON, + policy: { + policy: "short", + reason: + "Webhook retries share one identity while queued; once work starts, a later replay remains admissible as a rescue if the original exhausts retries.", + }, + work: { localConcurrency: 1 }, + load: async () => (await withingsSync()).handleWithingsEcgSync, + }), + defineJob({ + queue: WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, + cron: WITHINGS_OAUTH_STATE_CLEANUP_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => (await cleanups()).handleWithingsOAuthStateCleanup, + }), + // WHOOP per-resource syncs: a webhook job is one user, the cron walks + // everyone. Serial, inside WHOOP's 100 requests a minute per app. + defineJob({ + queue: WHOOP_RECOVERY_SYNC_QUEUE, + cron: WHOOP_RECOVERY_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await whoopSync()).handleWhoopRecoverySync, + }), + defineJob({ + queue: WHOOP_SLEEP_SYNC_QUEUE, + cron: WHOOP_SLEEP_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await whoopSync()).handleWhoopSleepSync, + }), + defineJob({ + queue: WHOOP_WORKOUT_SYNC_QUEUE, + cron: WHOOP_WORKOUT_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await whoopSync()).handleWhoopWorkoutSync, + }), + defineJob({ + queue: WHOOP_CYCLE_SYNC_QUEUE, + cron: WHOOP_CYCLE_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await whoopSync()).handleWhoopCycleSync, + }), + backfillSource( + WHOOP_BACKFILL_QUEUE, + WHOOP_BACKFILL_CONCURRENCY, + "whoop-backfill", + "Per-connection full-history backfill; long-running, and boot discovery re-enqueues until the connection is marked backfilled.", + ), + defineJob({ + queue: WHOOP_OAUTH_STATE_CLEANUP_QUEUE, + cron: WHOOP_OAUTH_STATE_CLEANUP_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => (await cleanups()).handleWhoopOAuthStateCleanup, + }), + ...FITBIT_JOBS, + defineJob({ + queue: GOOGLE_HEALTH_SYNC_QUEUE, + cron: GOOGLE_HEALTH_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => (await googleHealthSync()).handleGoogleHealthSync, + }), + backfillSource( + GOOGLE_HEALTH_BACKFILL_QUEUE, + GOOGLE_HEALTH_BACKFILL_CONCURRENCY, + "google-health-backfill", + ), + backfillSource( + GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, + GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY, + "google-health-sleep-repair", + "Per-connection one-shot sleep re-read; discovery drops the connection once it is marked repaired.", + ), + defineJob({ + queue: GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, + cron: GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await googleHealthSync()).handleGoogleHealthOAuthStateCleanup, + }), + backfillSource( + SLEEP_TIMELINE_BACKFILL_QUEUE, + SLEEP_TIMELINE_BACKFILL_CONCURRENCY, + "sleep-timeline-backfill", + "Per-connection one-shot timeline backfill, keyed per provider+user; discovery drops the connection once its rows carry the new shape.", + ), + backfillSource( + LAB_BIOMARKER_BACKFILL_QUEUE, + LAB_BIOMARKER_BACKFILL_CONCURRENCY, + "lab-biomarker-backfill", + "Per-user one-shot catalog link-up; discovery drops the user once no unlinked lab readings remain.", + ), + // The cohort polls: the hourly tick (no `userId`) walks every connection; + // one connection's failure is booked on its ledger, not the tick's. + defineJob({ + queue: NIGHTSCOUT_SYNC_QUEUE, + cron: NIGHTSCOUT_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/nightscout-sync")) + .handleNightscoutSync, + }), + defineJob({ + queue: POLAR_SYNC_QUEUE, + cron: POLAR_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/polar-sync")).handlePolarSync, + }), + defineJob({ + queue: OURA_SYNC_QUEUE, + cron: OURA_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/oura-sync")).handleOuraSync, + }), + backfillSource( + OURA_BACKFILL_QUEUE, + OURA_BACKFILL_CONCURRENCY, + "oura-backfill", + ), + defineJob({ + queue: STRAVA_SYNC_QUEUE, + cron: STRAVA_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/strava-sync")).handleStravaSync, + }), + backfillSource( + STRAVA_BACKFILL_QUEUE, + STRAVA_BACKFILL_CONCURRENCY, + "strava-backfill", + ), + defineJob({ + queue: OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, + cron: OIDC_NATIVE_HANDOFF_CLEANUP_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => (await cleanups()).handleOidcNativeHandoffCleanup, + }), + // The watchdog: an hourly cheap pass and a daily one that re-reads the + // history behind standing silence flags. Both share one advisory lock, so + // they never run side by side on any replica; the next tick is the retry. + defineJob({ + queue: INTEGRATION_WATCHDOG_HOURLY_QUEUE, + cron: INTEGRATION_WATCHDOG_HOURLY_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/integration-watchdog")) + .handleIntegrationWatchdogHourly, + }), + defineJob({ + queue: INTEGRATION_WATCHDOG_DAILY_QUEUE, + cron: INTEGRATION_WATCHDOG_DAILY_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/integration-watchdog")) + .handleIntegrationWatchdogDaily, + }), +]; + +/** A backfill discovery staggered by its kind's place in the boot order. */ +function backfillDiscovery( + kind: IntegrationBackfillKind, + enqueue: (staggerSeconds: number) => Promise, +): BootDiscovery { + return { name: kind, run: () => enqueue(bootStaggerSecondsFor(kind)) }; +} + +const discovery: readonly BootDiscovery[] = [ + backfillDiscovery("whoop-backfill", async (stagger) => + (await import("@/lib/jobs/whoop-backfill")).enqueueBootTimeWhoopBackfill( + stagger, + ), + ), + backfillDiscovery("google-health-backfill", async (stagger) => + ( + await import("@/lib/jobs/google-health-backfill") + ).enqueueBootTimeGoogleHealthBackfill(stagger), + ), + backfillDiscovery("google-health-sleep-repair", async (stagger) => + ( + await import("@/lib/jobs/google-health-sleep-repair") + ).enqueueBootTimeGoogleHealthSleepRepair(stagger), + ), + backfillDiscovery("sleep-timeline-backfill", async (stagger) => + ( + await import("@/lib/jobs/sleep-timeline-backfill") + ).enqueueBootTimeSleepTimelineBackfill(stagger), + ), + backfillDiscovery("strava-backfill", async (stagger) => + (await import("@/lib/jobs/strava-backfill")).enqueueBootTimeStravaBackfill( + stagger, + ), + ), + backfillDiscovery("oura-backfill", async (stagger) => + (await import("@/lib/jobs/oura-backfill")).enqueueBootTimeOuraBackfill( + stagger, + ), + ), + backfillDiscovery("lab-biomarker-backfill", async (stagger) => + ( + await import("@/lib/jobs/lab-biomarker-backfill") + ).enqueueBootTimeLabBiomarkerBackfill(stagger), + ), + ...FITBIT_BOOT_DISCOVERY, +]; + +export const INTEGRATION_SYNC_GROUP: JobGroup = { + name: "integration_sync", + jobs, + discovery, +}; diff --git a/src/lib/jobs/registry/maintenance-discovery.ts b/src/lib/jobs/registry/maintenance-discovery.ts new file mode 100644 index 000000000..aa137ea13 --- /dev/null +++ b/src/lib/jobs/registry/maintenance-discovery.ts @@ -0,0 +1,100 @@ +/** + * The boot discovery of the maintenance group: the duplicate dose-slot + * cleanup, the encryption backfills, the thumbnail backfill, the air-quality + * history, the fold repair and the purge that waits for it, the GeoLite2 + * fetch, and the detached climb backfill. Each pass is idempotent across + * boots; the heavy ones start past the boot storm. + */ +import { BOOT_BACKFILL_STAGGER_SECONDS } from "@/lib/jobs/reminder/shared"; + +import type { BootDiscovery } from "./define-job"; + +export const MAINTENANCE_DISCOVERY: readonly BootDiscovery[] = [ + { + name: "intake-slot-dedup", + run: async () => + ( + await import("@/lib/medications/intake-slot-dedup") + ).enqueueBootTimeIntakeSlotDedup(), + }, + // The heavy loads start after the boot storm; the daily ticks pass no + // offset and stay immediate. + { + name: "note-encryption-backfill", + run: async () => + ( + await import("@/lib/jobs/note-encryption-backfill") + ).enqueueBootTimeNoteEncryptionBackfill( + BOOT_BACKFILL_STAGGER_SECONDS * 5, + ), + }, + { + name: "med-notes-encryption-backfill", + run: async () => + ( + await import("@/lib/jobs/med-notes-encryption-backfill") + ).enqueueBootTimeMedNotesEncryptionBackfill(), + }, + { + name: "free-text-encryption-backfill", + run: async () => + ( + await import("@/lib/jobs/free-text-encryption-backfill") + ).enqueueBootTimeFreeTextEncryptionBackfill( + BOOT_BACKFILL_STAGGER_SECONDS * 6, + ), + }, + { + name: "document-thumbnail-backfill", + run: async () => { + const { enqueued } = await ( + await import("@/lib/jobs/document-thumbnail-backfill") + ).enqueueBootTimeThumbnailBackfill(); + return { enqueued }; + }, + }, + { + name: "environment-aq-history", + run: async () => + ( + await import("@/lib/jobs/environment-air-quality-history") + ).enqueueBootTimeAirQualityHistory(BOOT_BACKFILL_STAGGER_SECONDS * 7), + }, + // Once per account; the purge below waits for it. + { + name: "measurement-fold-repair", + run: async () => + ( + await import("@/lib/jobs/measurement-fold-repair") + ).enqueueBootTimeMeasurementFoldRepair(), + }, + // Self-limiting and idempotent; a boot with no backlog costs one query. + { + name: "compaction-tombstone-purge", + run: async () => { + const { enqueued } = await ( + await import("@/lib/jobs/compaction-tombstone-purge") + ).enqueueBootTimeCompactionTombstonePurge(); + return { enqueued }; + }, + }, + // Only with a MaxMind key, egress allowed and no offline database present. + { + name: "geolite2-fetch", + run: async () => { + const { enqueued } = await ( + await import("@/lib/jobs/geolite2-fetch") + ).enqueueGeolite2FetchBootDiscovery(); + return { enqueued }; + }, + }, + // Detached: reads one track at a time with a pause between batches and + // resumes from its cursor on the next boot; a finished pass returns at once. + { + name: "workout-climb-backfill", + run: async () => + ( + await import("@/lib/jobs/reminder/climb-backfill-at-boot") + ).startClimbBackfill(), + }, +]; diff --git a/src/lib/jobs/registry/maintenance.ts b/src/lib/jobs/registry/maintenance.ts new file mode 100644 index 000000000..d0f2ec23e --- /dev/null +++ b/src/lib/jobs/registry/maintenance.ts @@ -0,0 +1,744 @@ +/** + * The housekeeping queues: backups (weekly in-database, nightly off-host), + * the restore drill and restores, every ledger and retention cleanup, the + * medication maintenance passes, the imports (Apple Health, Health Connect, + * medication intake), the converging encryption backfills, the document vault + * work, the environment and Dawarich context, the geo and TLS probes, PR + * detection and the measurement table maintenance. + * + * `cronIsTheRetry` (no retries) marks the passes whose failure repeats: a bulk + * DELETE or UPDATE that timed out will time out again on an immediate retry, + * and the next tick arrives with the whole gap as backoff. The transient ones + * (the TLS probe, the off-host upload, a provider fetch) keep the default + * retries. The restore drill sends `retryLimit: 0` in its own options, because + * it pages each failed account itself. + * + * Policies. The retention ticks are keyless, so they carry none. Deliberately + * left at `standard`: the environment fetch (an explicit-range backfill sends + * with no key so it always runs; a policy would merge two requested ranges), + * and the imports, the backup and PR detection, whose runs are each distinct + * work. The per-user converging backfills are `exclusive`; the per-document + * jobs and the runs that queue their own follow-up while active are `short`. + */ +import type { Job } from "pg-boss"; + +import { + ACHIEVEMENT_UNLOCK_SWEEP_CRON, + ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, +} from "@/lib/jobs/achievement-unlock-sweep-queue"; +import { + APPLE_HEALTH_IMPORT_CONCURRENCY, + APPLE_HEALTH_IMPORT_LEGACY_QUEUE, + APPLE_HEALTH_IMPORT_V2_QUEUE, + IMPORT_JOB_RECONCILE_CRON, + IMPORT_JOB_RECONCILE_QUEUE, +} from "@/lib/jobs/apple-health-import-queue"; +import { + AUTH_LEDGER_CLEANUP_CRON, + AUTH_LEDGER_CLEANUP_QUEUE, +} from "@/lib/jobs/auth-ledger-cleanup"; +import { BACKUP_RESTORE_QUEUE } from "@/lib/jobs/backup-restore-queue"; +import { COMPACTION_TOMBSTONE_PURGE_QUEUE } from "@/lib/jobs/compaction-tombstone-purge-queue"; +import { + CYCLE_PREDICTION_REFRESH_CRON, + CYCLE_PREDICTION_REFRESH_QUEUE, +} from "@/lib/jobs/cycle-prediction-refresh"; +import { + DATA_BACKUP_QUEUE, + DATA_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/data-backup-policy"; +import { + DAWARICH_SYNC_CRON, + DAWARICH_SYNC_QUEUE, +} from "@/lib/jobs/dawarich-sync-queue"; +import { + DOCUMENT_AI_RUN_CONCURRENCY, + DOCUMENT_AI_RUN_QUEUE, +} from "@/lib/jobs/document-ai-run-queue"; +import { + DOCUMENT_AI_RUN_REAPER_CRON, + DOCUMENT_AI_RUN_REAPER_QUEUE, +} from "@/lib/jobs/document-ai-run-reaper-queue"; +import { + CONTENT_INDEX_BACKFILL_CONCURRENCY, + CONTENT_INDEX_BACKFILL_QUEUE, +} from "@/lib/jobs/document-content-index-backfill-queue"; +import { + DOCUMENT_INDEX_CONCURRENCY, + DOCUMENT_INDEX_QUEUE, +} from "@/lib/jobs/document-index-queue"; +import { + DOCUMENT_PURGE_CRON, + DOCUMENT_PURGE_QUEUE, +} from "@/lib/jobs/document-purge"; +import { + DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY, + DOCUMENT_SUMMARY_CATCHUP_QUEUE, +} from "@/lib/jobs/document-summary-catchup-queue"; +import { + DOCUMENT_SUMMARY_CONCURRENCY, + DOCUMENT_SUMMARY_QUEUE, +} from "@/lib/jobs/document-summary-queue"; +import { + DOCUMENT_SUMMARY_REAPER_CRON, + DOCUMENT_SUMMARY_REAPER_QUEUE, +} from "@/lib/jobs/document-summary-reaper-queue"; +import { + DOCUMENT_THUMBNAIL_CONCURRENCY, + DOCUMENT_THUMBNAIL_QUEUE, +} from "@/lib/jobs/document-thumbnail-queue"; +import { + DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY, + DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, +} from "@/lib/jobs/document-thumbnail-backfill-queue"; +import { + ENCRYPTION_KEY_ROTATE_CONCURRENCY, + ENCRYPTION_KEY_ROTATE_QUEUE, +} from "@/lib/jobs/encryption-key-rotate-queue"; +import { + ENVIRONMENT_AQ_HISTORY_CRON, + ENVIRONMENT_AQ_HISTORY_QUEUE, +} from "@/lib/jobs/environment-air-quality-history"; +import { + ENVIRONMENT_FETCH_CONCURRENCY, + ENVIRONMENT_FETCH_CRON, + ENVIRONMENT_FETCH_QUEUE, +} from "@/lib/jobs/environment-fetch"; +import { + FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY, + FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, +} from "@/lib/jobs/free-text-encryption-backfill"; +import { GEO_BACKFILL_CRON, GEO_BACKFILL_QUEUE } from "@/lib/jobs/geo-backfill"; +import { + GEOLITE2_FETCH_CRON, + GEOLITE2_FETCH_QUEUE, +} from "@/lib/jobs/geolite2-fetch"; +import { + HEALTH_CONNECT_IMPORT_CONCURRENCY, + HEALTH_CONNECT_IMPORT_QUEUE, +} from "@/lib/jobs/health-connect-import-queue"; +import { + INTAKE_AUTO_SKIP_CRON, + INTAKE_AUTO_SKIP_QUEUE, +} from "@/lib/jobs/intake-auto-skip-queue"; +import { + INTAKE_AUTO_TAKE_CRON, + INTAKE_AUTO_TAKE_QUEUE, +} from "@/lib/jobs/intake-auto-take-queue"; +import { WHOLE_PASS } from "@/lib/jobs/long-pass"; +import { + MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY, + MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, +} from "@/lib/jobs/med-notes-encryption-backfill"; +import { MEASUREMENT_FOLD_REPAIR_QUEUE } from "@/lib/jobs/measurement-fold-repair-queue"; +import { + MEASUREMENT_MAINTENANCE_CRON, + MEASUREMENT_MAINTENANCE_QUEUE, + MEASUREMENT_MAINTENANCE_SEND_OPTIONS, +} from "@/lib/jobs/measurement-maintenance"; +import { + MEDICATION_INTAKE_IMPORT_CONCURRENCY, + MEDICATION_INTAKE_IMPORT_QUEUE, +} from "@/lib/jobs/medication-intake-import-queue"; +import { + MEDICATION_INVENTORY_EXPIRE_CRON, + MEDICATION_INVENTORY_EXPIRE_QUEUE, +} from "@/lib/jobs/medication-inventory-expire"; +import { + MOOD_PROGNOSIS_REFRESH_CRON, + MOOD_PROGNOSIS_REFRESH_QUEUE, +} from "@/lib/jobs/mood-prognosis-refresh-queue"; +import { + NOTE_ENCRYPTION_BACKFILL_CONCURRENCY, + NOTE_ENCRYPTION_BACKFILL_QUEUE, +} from "@/lib/jobs/note-encryption-backfill"; +import { + OFFHOST_BACKUP_QUEUE, + OFFHOST_BACKUP_SEND_OPTIONS, +} from "@/lib/jobs/offhost-backup-queue"; +import { + OFFHOST_PURGE_CRON, + OFFHOST_PURGE_QUEUE, +} from "@/lib/jobs/offhost-purge-queue"; +import { + PR_DETECTION_CONCURRENCY, + PR_DETECTION_EXPIRE_SECONDS, + PR_DETECTION_FALLBACK_CRON, + PR_DETECTION_QUEUE, + type PrDetectionPayload, +} from "@/lib/jobs/pr-detection"; +import { PULSE_DAY_MEAN_QUEUE } from "@/lib/jobs/pulse-day-mean"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + RESTORE_DRILL_CRON, + RESTORE_DRILL_QUEUE, + RESTORE_DRILL_SEND_OPTIONS, +} from "@/lib/jobs/restore-drill-queue"; +import { + SOFT_DELETE_PURGE_CRON, + SOFT_DELETE_PURGE_QUEUE, +} from "@/lib/jobs/soft-delete-purge"; +import { + TLS_PIN_MONITOR_CRON, + TLS_PIN_MONITOR_QUEUE, +} from "@/lib/jobs/tls-pin-monitor-queue"; +import { + INTAKE_SLOT_DEDUP_CONCURRENCY, + INTAKE_SLOT_DEDUP_QUEUE, +} from "@/lib/medications/intake-slot-dedup"; + +import { MAINTENANCE_DISCOVERY } from "./maintenance-discovery"; +import { + defineJob, + type JobDefinition, + type JobGroup, + type JobSpec, +} from "./define-job"; + +/** Weekly, Sunday 03:00. The send options carry the backup's long expiry. */ +const DATA_BACKUP_CRON = "0 3 * * 0"; +/** 02:30: after the audit and idempotency cleanups, before the weekly backup. */ +const OFFHOST_BACKUP_CRON = "30 2 * * *"; +const RATE_LIMIT_CLEANUP_QUEUE = "rate-limit-cleanup"; +const RATE_LIMIT_CLEANUP_CRON = "*/5 * * * *"; +const IDEMPOTENCY_CLEANUP_QUEUE = "idempotency-cleanup"; +const IDEMPOTENCY_CLEANUP_CRON = "0 3 * * *"; +/** An elevation lives five minutes; the sweep only keeps the table small. */ +const STEP_UP_ELEVATION_CLEANUP_QUEUE = "step-up-elevation-cleanup"; +const STEP_UP_ELEVATION_CLEANUP_CRON = "*/15 * * * *"; +const AUDIT_LOG_CLEANUP_QUEUE = "audit-log-cleanup"; +const AUDIT_LOG_CLEANUP_CRON = "15 3 * * *"; +/** Daily duplicate dose-slot discovery; the empty payload is the fan-out. */ +const INTAKE_SLOT_DEDUP_CRON = "28 3 * * *"; +/** Every five minutes; the admin chart polls at the same interval. */ +const HOST_METRIC_QUEUE = "host-metric-sample"; +const HOST_METRIC_CRON = "*/5 * * * *"; +/** After the cleanups, so the previous day's noise is gone. */ +const FEEDBACK_AGGREGATOR_QUEUE = "feedback-aggregator"; +const FEEDBACK_AGGREGATOR_CRON = "0 4 * * *"; +const MOOD_REMINDER_CLEANUP_QUEUE = "mood-reminder-cleanup"; +const MOOD_REMINDER_CLEANUP_CRON = "25 3 * * *"; +const PUSH_ATTEMPT_CLEANUP_QUEUE = "push-attempt-cleanup"; +const PUSH_ATTEMPT_CLEANUP_CRON = "35 3 * * *"; +const ARRIVAL_REACTION_CLEANUP_QUEUE = "arrival-reaction-cleanup"; +const ARRIVAL_REACTION_CLEANUP_CRON = "10 4 * * *"; +const MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE = "measurement-tombstone-cleanup"; +const MEASUREMENT_TOMBSTONE_CLEANUP_CRON = "40 3 * * *"; +const COACH_MESSAGE_CLEANUP_QUEUE = "coach-message-cleanup"; +const COACH_MESSAGE_CLEANUP_CRON = "50 3 * * *"; +const WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE = "workout-insight-claim-cleanup"; +const WORKOUT_INSIGHT_CLAIM_CLEANUP_CRON = "52 3 * * *"; +const NOTE_ENCRYPTION_BACKFILL_CRON = "55 3 * * *"; +const MCP_TOKEN_CLEANUP_QUEUE = "mcp-token-cleanup"; +const MCP_TOKEN_CLEANUP_CRON = "0 4 * * *"; +const MED_NOTES_ENCRYPTION_BACKFILL_CRON = "5 4 * * *"; +const FREE_TEXT_ENCRYPTION_BACKFILL_CRON = "7 4 * * *"; + +const backups = () => import("@/lib/jobs/reminder/backup-handlers"); +const cleanups = () => import("@/lib/jobs/reminder/cleanup-handlers"); +const ops = () => import("@/lib/jobs/reminder/ops-handlers"); +const medication = () => import("@/lib/jobs/reminder/medication-maintenance"); +const maintenance = () => import("@/lib/jobs/reminder/maintenance-handlers"); +const encryption = () => + import("@/lib/jobs/reminder/encryption-backfill-handlers"); +const documents = () => import("@/lib/jobs/reminder/document-handlers"); + +/** A daily keyless prune: one bounded, idempotent delete, single-flight. */ +function prune>( + queue: string, + cron: string, + load: JobSpec["load"], + options: { retry: "cron" | "default" } = { retry: "cron" }, +): JobDefinition { + return defineJob({ + queue, + cron, + ...(options.retry === "cron" ? { scheduleOptions: cronIsTheRetry } : {}), + work: { localConcurrency: 1 }, + load, + }); +} + +const perUserBackfill = (reason: string) => + ({ policy: "exclusive", reason }) as const; + +const jobs: readonly JobDefinition[] = [ + defineJob({ + queue: ENVIRONMENT_FETCH_QUEUE, + cron: ENVIRONMENT_FETCH_CRON, + work: { localConcurrency: ENVIRONMENT_FETCH_CONCURRENCY }, + load: async ({ boss }) => + (await maintenance()).environmentFetchHandler(boss), + }), + // A run with work left sends its own follow-up while still active, which + // `exclusive` would swallow. + defineJob({ + queue: ENVIRONMENT_AQ_HISTORY_QUEUE, + cron: ENVIRONMENT_AQ_HISTORY_CRON, + policy: { + policy: "short", + reason: + "Per-account singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", + }, + work: { localConcurrency: 1 }, + load: async ({ boss }) => + (await maintenance()).environmentAqHistoryHandler(boss), + }), + // A failed run is an outage at the instance; the default retries apply. + defineJob({ + queue: DAWARICH_SYNC_QUEUE, + cron: DAWARICH_SYNC_CRON, + work: { localConcurrency: 1 }, + load: async ({ boss }) => (await maintenance()).dawarichSyncHandler(boss), + }), + defineJob({ + queue: DATA_BACKUP_QUEUE, + cron: DATA_BACKUP_CRON, + scheduleOptions: DATA_BACKUP_SEND_OPTIONS, + work: { localConcurrency: 1 }, + lock: () => WHOLE_PASS, + lane: true, + load: async () => (await backups()).handleDataBackup, + }), + prune( + RATE_LIMIT_CLEANUP_QUEUE, + RATE_LIMIT_CLEANUP_CRON, + async () => (await cleanups()).handleRateLimitCleanup, + ), + prune( + IDEMPOTENCY_CLEANUP_QUEUE, + IDEMPOTENCY_CLEANUP_CRON, + async () => (await cleanups()).handleIdempotencyCleanup, + ), + prune( + AUDIT_LOG_CLEANUP_QUEUE, + AUDIT_LOG_CLEANUP_CRON, + async () => (await cleanups()).handleAuditLogCleanup, + { retry: "default" }, + ), + prune( + STEP_UP_ELEVATION_CLEANUP_QUEUE, + STEP_UP_ELEVATION_CLEANUP_CRON, + async () => (await cleanups()).handleStepUpElevationCleanup, + ), + // Bound with metadata: a retry resumes from the job's creation time. + defineJob({ + queue: OFFHOST_BACKUP_QUEUE, + cron: OFFHOST_BACKUP_CRON, + scheduleOptions: OFFHOST_BACKUP_SEND_OPTIONS, + work: { localConcurrency: 1, includeMetadata: true }, + lock: () => WHOLE_PASS, + lane: true, + load: async () => (await backups()).handleOffhostBackup, + }), + // Locked: a drill that outlives a restart never reads the bucket beside + // another one. + defineJob({ + queue: RESTORE_DRILL_QUEUE, + cron: RESTORE_DRILL_CRON, + scheduleOptions: RESTORE_DRILL_SEND_OPTIONS, + work: { localConcurrency: 1 }, + lock: () => WHOLE_PASS, + lane: true, + load: async () => + (await import("@/lib/jobs/restore-drill")).handleRestoreDrill, + }), + // The nightly backstop for the purge an account deletion kicks at once. + prune( + OFFHOST_PURGE_QUEUE, + OFFHOST_PURGE_CRON, + async () => (await import("@/lib/jobs/offhost-purge")).handleOffhostPurge, + ), + // One restore at a time: each holds a transaction over a whole account. + defineJob({ + queue: BACKUP_RESTORE_QUEUE, + work: { localConcurrency: 1 }, + lane: true, + load: async () => + (await import("@/lib/jobs/backup-restore")).handleBackupRestore, + }), + prune( + HOST_METRIC_QUEUE, + HOST_METRIC_CRON, + async () => (await ops()).handleHostMetricSample, + ), + prune( + FEEDBACK_AGGREGATOR_QUEUE, + FEEDBACK_AGGREGATOR_CRON, + async () => (await ops()).handleFeedbackAggregator, + ), + // Hourly at :40, capped per pass, for audit rows that landed without a geo. + prune( + GEO_BACKFILL_QUEUE, + GEO_BACKFILL_CRON, + async () => (await ops()).handleGeoBackfill, + ), + // Monthly; a month is far too long to be the retry, so default retries. + prune( + GEOLITE2_FETCH_QUEUE, + GEOLITE2_FETCH_CRON, + async () => (await ops()).handleGeolite2Fetch, + { retry: "default" }, + ), + // Every six hours, inside the re-pin window the app's leaf pin needs. + prune( + TLS_PIN_MONITOR_QUEUE, + TLS_PIN_MONITOR_CRON, + async () => (await ops()).handleTlsPinMonitor, + { retry: "default" }, + ), + // An ingest job is one account; the fallback rescan every 30 minutes walks + // every account, one run at a time. + defineJob({ + queue: PR_DETECTION_QUEUE, + cron: PR_DETECTION_FALLBACK_CRON, + scheduleOptions: { expireInSeconds: PR_DETECTION_EXPIRE_SECONDS }, + work: { localConcurrency: PR_DETECTION_CONCURRENCY }, + lock: (job: Job) => { + const userId = (job.data as PrDetectionPayload | undefined)?.userId; + return userId ? `user:${userId}` : WHOLE_PASS; + }, + load: async () => (await ops()).handlePrDetection, + }), + prune( + MEDICATION_INVENTORY_EXPIRE_QUEUE, + MEDICATION_INVENTORY_EXPIRE_CRON, + async () => (await medication()).handleMedicationInventoryExpire, + ), + prune( + INTAKE_AUTO_SKIP_QUEUE, + INTAKE_AUTO_SKIP_CRON, + async () => (await medication()).handleIntakeAutoSkip, + ), + prune( + INTAKE_AUTO_TAKE_QUEUE, + INTAKE_AUTO_TAKE_CRON, + async () => (await medication()).handleIntakeAutoTake, + ), + // CPU-bound parse of up to a gigabyte; a second at once would race the + // first for memory. + defineJob({ + queue: APPLE_HEALTH_IMPORT_V2_QUEUE, + work: { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, + lane: true, + load: async () => (await maintenance()).handleAppleHealthImportJobs, + }), + defineJob({ + queue: APPLE_HEALTH_IMPORT_LEGACY_QUEUE, + work: { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, + load: async () => (await maintenance()).handleLegacyAppleHealthImportJobs, + }), + // Every 15 minutes: an import a crashed worker left mid-run converges to a + // visible `failed` within two ticks. + prune( + IMPORT_JOB_RECONCILE_QUEUE, + IMPORT_JOB_RECONCILE_CRON, + async () => + (await import("@/lib/jobs/apple-health-import-worker")) + .handleImportJobReconcileTick, + ), + defineJob({ + queue: MEDICATION_INTAKE_IMPORT_QUEUE, + work: { + localConcurrency: MEDICATION_INTAKE_IMPORT_CONCURRENCY, + includeMetadata: true, + }, + load: async () => (await maintenance()).handleMedicationIntakeImportJobs, + }), + defineJob({ + queue: HEALTH_CONNECT_IMPORT_QUEUE, + work: { localConcurrency: HEALTH_CONNECT_IMPORT_CONCURRENCY }, + lane: true, + load: async () => + (await import("@/lib/jobs/health-connect-import-worker")) + .handleHealthConnectImport, + }), + // A run that stops early sends its own follow-up while still active, which + // `exclusive` would swallow. Serial: each batch holds an account against a + // concurrent restore. + defineJob({ + queue: COMPACTION_TOMBSTONE_PURGE_QUEUE, + policy: { + policy: "short", + reason: + "Fixed singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/compaction-tombstone-purge")) + .handleCompactionTombstonePurge, + }), + defineJob({ + queue: MEASUREMENT_FOLD_REPAIR_QUEUE, + policy: { + policy: "short", + reason: + "Per-account singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/measurement-fold-repair")) + .handleMeasurementFoldRepair, + }), + // The nightly check vacuums when dead rows pile up; the same singleton key + // as the operator's run, so the two collapse. + defineJob({ + queue: MEASUREMENT_MAINTENANCE_QUEUE, + cron: MEASUREMENT_MAINTENANCE_CRON, + scheduleOptions: MEASUREMENT_MAINTENANCE_SEND_OPTIONS, + policy: { + policy: "exclusive", + reason: + "Fixed singleton key on the admin send and the nightly tick alike. Two maintenance passes would vacuum or rebuild side by side; the route reports a suppressed send back as enqueued: false.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/measurement-maintenance")) + .handleMeasurementMaintenance, + }), + defineJob({ + queue: PULSE_DAY_MEAN_QUEUE, + policy: { + policy: "exclusive", + reason: + "Fixed singleton key, admin-triggered. A dry run beside a rebuild would compare against rollups being rewritten; the route answers 409 busy instead.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/pulse-day-mean")).handlePulseDayMean, + }), + defineJob({ + queue: INTAKE_SLOT_DEDUP_QUEUE, + cron: INTAKE_SLOT_DEDUP_CRON, + policy: perUserBackfill( + "Per-user duplicate-slot cleanup; discovery drops the user once no colliding intake pair remains.", + ), + work: { localConcurrency: INTAKE_SLOT_DEDUP_CONCURRENCY }, + load: async () => (await maintenance()).handleIntakeSlotDedup, + }), + prune( + MOOD_REMINDER_CLEANUP_QUEUE, + MOOD_REMINDER_CLEANUP_CRON, + async () => (await cleanups()).handleMoodReminderCleanup, + ), + prune( + PUSH_ATTEMPT_CLEANUP_QUEUE, + PUSH_ATTEMPT_CLEANUP_CRON, + async () => (await cleanups()).handlePushAttemptCleanup, + ), + prune( + ARRIVAL_REACTION_CLEANUP_QUEUE, + ARRIVAL_REACTION_CLEANUP_CRON, + async () => (await cleanups()).handleArrivalReactionCleanup, + ), + // The nightly refreshes: tomorrow's tick is the retry. + prune( + CYCLE_PREDICTION_REFRESH_QUEUE, + CYCLE_PREDICTION_REFRESH_CRON, + async () => + (await import("@/lib/jobs/cycle-prediction-refresh")) + .handleCyclePredictionRefresh, + ), + prune( + MOOD_PROGNOSIS_REFRESH_QUEUE, + MOOD_PROGNOSIS_REFRESH_CRON, + async () => + (await import("@/lib/jobs/mood-prognosis-refresh")) + .handleMoodPrognosisRefresh, + ), + prune( + ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, + ACHIEVEMENT_UNLOCK_SWEEP_CRON, + async () => + (await import("@/lib/jobs/achievement-unlock-sweep")) + .handleAchievementUnlockSweep, + ), + prune( + MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, + MEASUREMENT_TOMBSTONE_CLEANUP_CRON, + async () => (await cleanups()).handleMeasurementTombstoneCleanup, + { retry: "default" }, + ), + prune( + COACH_MESSAGE_CLEANUP_QUEUE, + COACH_MESSAGE_CLEANUP_CRON, + async () => (await cleanups()).handleCoachMessageCleanup, + ), + prune( + WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE, + WORKOUT_INSIGHT_CLAIM_CLEANUP_CRON, + async () => (await cleanups()).handleWorkoutInsightClaimCleanup, + ), + // The converging encryption backfills: boot discovery plus a daily tick + // whose empty payload fans out one job per account still holding a + // readable value. + defineJob({ + queue: NOTE_ENCRYPTION_BACKFILL_QUEUE, + cron: NOTE_ENCRYPTION_BACKFILL_CRON, + policy: perUserBackfill( + "Per-user note-encryption backfill; discovery drops the user once no plaintext note remains.", + ), + work: { localConcurrency: NOTE_ENCRYPTION_BACKFILL_CONCURRENCY }, + load: async () => (await encryption()).handleNoteEncryptionBackfill, + }), + // Admin-triggered, one corpus rotation at a time. + defineJob({ + queue: ENCRYPTION_KEY_ROTATE_QUEUE, + policy: { + policy: "exclusive", + reason: + "Fixed singleton key, admin-triggered. Two concurrent corpus rotations must never overlap; the route already reports a suppressed send back as alreadyQueued.", + }, + work: { localConcurrency: ENCRYPTION_KEY_ROTATE_CONCURRENCY }, + lock: () => WHOLE_PASS, + load: async () => + (await import("@/lib/jobs/encryption-key-rotate")) + .handleEncryptionKeyRotate, + }), + prune( + MCP_TOKEN_CLEANUP_QUEUE, + MCP_TOKEN_CLEANUP_CRON, + async () => (await cleanups()).handleMcpTokenCleanup, + ), + defineJob({ + queue: MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, + cron: MED_NOTES_ENCRYPTION_BACKFILL_CRON, + policy: perUserBackfill( + "Per-user medication-note encryption backfill; discovery drops the user once no plaintext note remains.", + ), + work: { localConcurrency: MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY }, + load: async () => (await encryption()).handleMedNotesEncryptionBackfill, + }), + defineJob({ + queue: FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, + cron: FREE_TEXT_ENCRYPTION_BACKFILL_CRON, + policy: perUserBackfill( + "Per-user encryption backfill (Coach title, custom-metric note, practitioner phone + address, workout GPS track); discovery drops the user once no readable value remains.", + ), + work: { localConcurrency: FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY }, + load: async () => (await encryption()).handleFreeTextEncryptionBackfill, + }), + // Physical purge of documents past the 30-day undo grace. + prune( + DOCUMENT_PURGE_QUEUE, + DOCUMENT_PURGE_CRON, + async () => (await import("@/lib/jobs/document-purge")).handleDocumentPurge, + ), + prune( + SOFT_DELETE_PURGE_QUEUE, + SOFT_DELETE_PURGE_CRON, + async () => + (await import("@/lib/jobs/soft-delete-purge")).handleSoftDeletePurge, + ), + prune( + AUTH_LEDGER_CLEANUP_QUEUE, + AUTH_LEDGER_CLEANUP_CRON, + async () => + (await import("@/lib/jobs/auth-ledger-cleanup")).handleAuthLedgerCleanup, + ), + // Hourly heal of summaries stuck on PENDING. + prune( + DOCUMENT_SUMMARY_REAPER_QUEUE, + DOCUMENT_SUMMARY_REAPER_CRON, + async () => + (await import("@/lib/jobs/document-summary-reaper")) + .handleDocumentSummaryReaper, + ), + // Index-all for one account, idempotent: a second press while one runs is + // correctly a no-op. + defineJob({ + queue: CONTENT_INDEX_BACKFILL_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user index-all run, triggered from the documents surface. Idempotent (it indexes only not-yet-indexed documents), so a second press while one runs is correctly a no-op.", + }, + work: { localConcurrency: CONTENT_INDEX_BACKFILL_CONCURRENCY }, + lock: (job: Job<{ userId?: string }>) => `user:${job.data.userId ?? ""}`, + load: async () => (await documents()).handleContentIndexBackfill, + }), + // Per-document jobs re-read the document when they start, so only queued + // duplicates may collapse; nothing re-offers a dropped re-process. + defineJob({ + queue: DOCUMENT_INDEX_QUEUE, + policy: { + policy: "short", + reason: + "Per-document, no re-converging discovery pass. Collapse queued duplicates only, so a re-index after a content change is never dropped.", + }, + work: { localConcurrency: DOCUMENT_INDEX_CONCURRENCY }, + load: async () => (await documents()).handleDocumentIndex, + }), + // One run per job under its run id's lock, so a redelivery never reads + // beside the first delivery. + defineJob({ + queue: DOCUMENT_AI_RUN_QUEUE, + work: { localConcurrency: DOCUMENT_AI_RUN_CONCURRENCY }, + lock: (job: Job<{ runId?: string }>) => `run:${job.data.runId ?? ""}`, + load: async () => + (await import("@/lib/jobs/document-ai-run")).handleDocumentAiRunJobs, + }), + prune( + DOCUMENT_AI_RUN_REAPER_QUEUE, + DOCUMENT_AI_RUN_REAPER_CRON, + async () => + (await import("@/lib/jobs/document-ai-run-reaper")) + .handleDocumentAiRunReaper, + ), + defineJob({ + queue: DOCUMENT_THUMBNAIL_QUEUE, + policy: { + policy: "short", + reason: + "Per-document, no re-converging discovery pass. Collapse queued duplicates only.", + }, + work: { localConcurrency: DOCUMENT_THUMBNAIL_CONCURRENCY }, + load: async () => (await documents()).handleDocumentThumbnail, + }), + defineJob({ + queue: DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, + policy: perUserBackfill( + "Per-user thumbnail backfill; discovery drops the user once every thumbnailable document has a preview.", + ), + work: { localConcurrency: DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY }, + load: async () => (await documents()).handleDocumentThumbnailBackfill, + }), + defineJob({ + queue: DOCUMENT_SUMMARY_QUEUE, + policy: { + policy: "short", + reason: + "Per-document, no re-converging discovery pass. Collapse queued duplicates only.", + }, + work: { localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY }, + load: async () => (await documents()).handleDocumentSummary, + }), + defineJob({ + queue: DOCUMENT_SUMMARY_CATCHUP_QUEUE, + policy: { + policy: "short", + reason: + "Per-user, keyed on the opt-in flip. Collapse a double toggle into one pass; the pass re-reads its candidate set when it starts.", + }, + work: { localConcurrency: DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY }, + load: async () => (await documents()).handleDocumentSummaryCatchUp, + }), +]; + +/** A restore the previous process was running is picked up once its heartbeat is stale. */ +async function sweepInterruptedRestores({ + boss, +}: { + boss: Parameters[0]["boss"]; +}): Promise { + await ( + await import("@/lib/jobs/backup-restore") + ).enqueueBackupRestoreSweepAtBoot(boss); +} + +export const MAINTENANCE_GROUP: JobGroup = { + name: "maintenance", + jobs, + afterBind: sweepInterruptedRestores, + discovery: MAINTENANCE_DISCOVERY, +}; diff --git a/src/lib/jobs/registry/register.ts b/src/lib/jobs/registry/register.ts new file mode 100644 index 000000000..3a2a36bef --- /dev/null +++ b/src/lib/jobs/registry/register.ts @@ -0,0 +1,146 @@ +/** + * Turn the registry into running queues. + * + * For one group, in this order: create every queue (policy and retention + * included), reconcile the policies of queues that already exist, schedule + * every cron, remove the retired queues, then bind a handler to every queue. + * The order is the one the hand-written registrars kept, and + * `tests/integration/queue-registration-parity.test.ts` holds what it leaves + * in pg-boss to the golden they produced. + * + * A handler is loaded on its queue's first job, not here. The wrappers that + * shape how a job runs (the lock, the heavy lane) are applied at binding, so a + * queue's protections do not depend on whether its module has loaded yet. A + * module that fails to load fails the job that asked for it, with the queue's + * name on the error, and the next job tries the import again. + */ +import type { Job, PgBoss } from "pg-boss"; + +import { heavyLane } from "@/lib/jobs/heavy-lane"; +import { lockedPass } from "@/lib/jobs/long-pass"; +import { + createAndSchedule, + createAndWork, + retireQueues, + type QueuePolicyTable, + type ScheduleEntry, +} from "@/lib/jobs/reminder/registrar-shared"; +import { workerLog } from "@/lib/jobs/reminder/shared"; + +import type { + BatchHandler, + BootDiscovery, + JobContext, + JobDefinition, + JobGroup, +} from "./define-job"; + +/** + * The handler a definition binds: its loaded module behind the lock and the + * lane it declares. + */ +export function bindableHandler( + definition: JobDefinition, + ctx: JobContext, +): BatchHandler> { + let loading: Promise>> | null = null; + const load = (): Promise>> => { + loading ??= definition.load(ctx).catch((err: unknown) => { + loading = null; + throw err; + }); + return loading; + }; + let handler: BatchHandler> = async (jobs) => + (await load())(jobs); + if (definition.lane) { + handler = heavyLane( + definition.queue, + handler, + definition.lane === true ? undefined : definition.lane, + ); + } + if (definition.lock) { + handler = lockedPass(definition.queue, definition.lock, handler); + } + return handler; +} + +/** The schedule tuples of a group, in declaration order. */ +export function schedulesOf(jobs: readonly JobDefinition[]): ScheduleEntry[] { + return jobs.flatMap((job): ScheduleEntry[] => + job.cron + ? [ + job.scheduleOptions + ? [job.queue, job.cron, { ...job.scheduleOptions }] + : [job.queue, job.cron], + ] + : [], + ); +} + +/** The policy table of a group. */ +export function policiesOf(jobs: readonly JobDefinition[]): QueuePolicyTable { + return Object.fromEntries( + jobs.flatMap((job) => (job.policy ? [[job.queue, job.policy]] : [])), + ); +} + +/** + * Register one group: create, schedule, retire, bind. Returns the queue names + * it created. + */ +export async function registerJobGroup( + boss: PgBoss, + group: JobGroup, +): Promise { + const queues = group.jobs.map((job) => job.queue); + await createAndSchedule( + boss, + queues, + schedulesOf(group.jobs), + policiesOf(group.jobs), + ); + if (group.retired) await retireQueues(boss, group.retired); + const ctx: JobContext = { boss }; + for (const job of group.jobs) { + await createAndWork(boss, job.queue, job.work, bindableHandler(job, ctx)); + } + if (group.afterBind) await group.afterBind(ctx); + return queues; +} + +/** + * Run one group's boot discovery. A pass that fails is logged and the next + * one runs: a discovery miss never fails the boot, and the next boot or cron + * tick offers the work again. + */ +export async function runBootDiscovery( + passes: readonly BootDiscovery[], +): Promise { + for (const pass of passes) { + try { + const result = await pass.run(); + if (!result) continue; + if (result.error) { + workerLog( + "error", + `[${pass.name}] boot discovery failed: ${result.error}`, + ); + } else { + const skipped = + result.skipped === undefined ? "" : ` skipped=${result.skipped}`; + workerLog( + "info", + `[${pass.name}] boot discovery: enqueued=${result.enqueued}${skipped}`, + ); + } + } catch (err) { + workerLog( + "error", + `[${pass.name}] boot discovery threw an unexpected error`, + err, + ); + } + } +} diff --git a/src/lib/jobs/registry/reminders.ts b/src/lib/jobs/registry/reminders.ts new file mode 100644 index 000000000..e66f90434 --- /dev/null +++ b/src/lib/jobs/registry/reminders.ts @@ -0,0 +1,70 @@ +/** + * The user-facing reminder ticks: the medication overdue check, the daily + * mood reminder, the cycle reminder, the checkup (measurement) reminder, and + * the eventful reminder-satisfy queue. + * + * The three daily reminders tick every 15 minutes and short-circuit unless + * the user's local time is the reminder's hour, so one UTC cron serves every + * zone; each keeps its own once-a-day anchor (a dispatch ledger, the + * `push_attempts` row, the reminder's `nextDueAt`). Every tick is + * single-flight so two ticks never race that anchor. + * + * Every queue here keeps pg-boss's default retries: a job-level failure means + * the pass itself could not run, a transient. The satisfy queue sets its own + * `retryLimit` at send time (`reminder-satisfy.ts`). + */ +import { + REMINDER_SATISFY_CONCURRENCY, + REMINDER_SATISFY_QUEUE, +} from "@/lib/jobs/reminder-satisfy"; + +import { defineJob, type JobDefinition, type JobGroup } from "./define-job"; + +const MEDICATION_REMINDER_CHECK_QUEUE = "medication-reminder-check"; +const MEDICATION_REMINDER_CHECK_CRON = "*/15 * * * *"; +const MOOD_REMINDER_QUEUE = "mood-reminder-check"; +const MOOD_REMINDER_CRON = "*/15 * * * *"; +const CYCLE_REMINDER_QUEUE = "cycle-reminder-check"; +const CYCLE_REMINDER_CRON = "*/15 * * * *"; +const MEASUREMENT_REMINDER_QUEUE = "measurement-reminder-check"; +const MEASUREMENT_REMINDER_CRON = "*/15 * * * *"; + +const checks = () => import("@/lib/jobs/reminder/mood-cycle-checks"); + +const jobs: readonly JobDefinition[] = [ + defineJob({ + queue: MEDICATION_REMINDER_CHECK_QUEUE, + cron: MEDICATION_REMINDER_CHECK_CRON, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reminder/medication-reminder-check")) + .handleReminderCheck, + }), + defineJob({ + queue: MOOD_REMINDER_QUEUE, + cron: MOOD_REMINDER_CRON, + work: { localConcurrency: 1 }, + load: async () => (await checks()).handleMoodReminderCheck, + }), + defineJob({ + queue: CYCLE_REMINDER_QUEUE, + cron: CYCLE_REMINDER_CRON, + work: { localConcurrency: 1 }, + load: async () => (await checks()).handleCycleReminderCheck, + }), + defineJob({ + queue: MEASUREMENT_REMINDER_QUEUE, + cron: MEASUREMENT_REMINDER_CRON, + work: { localConcurrency: 1 }, + load: async () => (await checks()).handleMeasurementReminderCheck, + }), + // No cron: enqueued by the ingest paths when a measurement or lab lands; + // the 15-minute checkup tick is the safety net. + defineJob({ + queue: REMINDER_SATISFY_QUEUE, + work: { localConcurrency: REMINDER_SATISFY_CONCURRENCY }, + load: async () => (await checks()).handleReminderSatisfy, + }), +]; + +export const REMINDERS_GROUP: JobGroup = { name: "reminders", jobs }; diff --git a/src/lib/jobs/registry/rollup.ts b/src/lib/jobs/registry/rollup.ts new file mode 100644 index 000000000..cb885c0dc --- /dev/null +++ b/src/lib/jobs/registry/rollup.ts @@ -0,0 +1,281 @@ +/** + * The pre-aggregation and consolidation queues: the measurement and mood + * rollup folds with their boot backfills, the legacy step consolidation and + * its repair, the cumulative-record re-derivation, the daily-mean + * consolidation, the dense intra-day retention and hourly rebuild, the + * supplement nutrient recompute, and the nightly drain. + * + * The two recompute queues and the supplement recompute are `short`: the + * handler reads the bucket's live rows when it starts, so sends that pile up + * while an identical job is still queued may collapse, while a send issued + * after the job started reading must still queue the next one. The per-user + * backfills are `exclusive`: a second run is duplicated work and discovery + * re-offers whatever is outstanding. The drain is a keyless cron tick. + * + * Boot discovery is staggered: each backfill starts a multiple of + * `BOOT_BACKFILL_STAGGER_SECONDS` later, so a heavy account's full-history + * loads do not all land on the first poll. The dense passes defer themselves + * further, past the boot window. + */ +import type { Job } from "pg-boss"; + +import { + DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY, + DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, +} from "@/lib/jobs/dense-intraday-hourly-rebuild-queue"; +import { + DENSE_INTRADAY_RETENTION_CONCURRENCY, + DENSE_INTRADAY_RETENTION_QUEUE, +} from "@/lib/jobs/dense-intraday-retention-queue"; +import { + DRAIN_CUMULATIVE_CRON, + DRAIN_CUMULATIVE_QUEUE, +} from "@/lib/jobs/drain-cumulative-queue"; +import { + MEAN_CONSOLIDATION_CONCURRENCY, + MEAN_CONSOLIDATION_QUEUE, +} from "@/lib/jobs/mean-consolidation"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { BOOT_BACKFILL_STAGGER_SECONDS } from "@/lib/jobs/reminder/shared"; +import { + STEP_CONSOLIDATION_CONCURRENCY, + STEP_CONSOLIDATION_QUEUE, +} from "@/lib/jobs/step-consolidation"; +import { + STEP_CONSOLIDATION_REPAIR_CONCURRENCY, + STEP_CONSOLIDATION_REPAIR_QUEUE, +} from "@/lib/jobs/step-consolidation-repair"; +import { + SUPPLEMENT_NUTRIENT_RECOMPUTE_CONCURRENCY, + SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE, +} from "@/lib/medications/ingredients/recompute-job"; +import { + CUMULATIVE_PR_REDERIVE_CONCURRENCY, + CUMULATIVE_PR_REDERIVE_QUEUE, +} from "@/lib/personal-records/cumulative-pr-rederivation"; +import { + ROLLUP_FULL_BACKFILL_CONCURRENCY, + ROLLUP_FULL_BACKFILL_QUEUE, + ROLLUP_RECOMPUTE_CONCURRENCY, + ROLLUP_RECOMPUTE_QUEUE, + type RollupFullBackfillPayload, +} from "@/lib/rollups/measurement-rollups"; +import { + MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY, + MOOD_ROLLUP_FULL_BACKFILL_QUEUE, + MOOD_ROLLUP_RECOMPUTE_CONCURRENCY, + MOOD_ROLLUP_RECOMPUTE_QUEUE, +} from "@/lib/rollups/mood-rollups"; + +import { + defineJob, + type BootDiscovery, + type JobDefinition, + type JobGroup, +} from "./define-job"; + +/** + * The boot backfill of the per-day medication-compliance rollup, whose table + * had no reader left and was dropped. Its queue is removed where it exists. + */ +const RETIRED_ROLLUP_QUEUES = ["medication-compliance-full-backfill"] as const; + +const handlers = () => import("@/lib/jobs/reminder/rollup-handlers"); + +const jobs: readonly JobDefinition[] = [ + defineJob({ + queue: ROLLUP_RECOMPUTE_QUEUE, + policy: { + policy: "short", + reason: + "Burst coalescing for the write-path fan-out. The handler re-reads the bucket at run time, so collapsing queued duplicates cannot strand a write; exclusive could.", + }, + work: { localConcurrency: ROLLUP_RECOMPUTE_CONCURRENCY }, + load: async () => (await handlers()).handleRollupRecompute, + }), + // The lock sits outside the lane: a retry pg-boss starts after the expiry + // while the first fold still runs finds the account's lock and settles, + // instead of waiting in the lane and folding the account again. + defineJob({ + queue: ROLLUP_FULL_BACKFILL_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user full-history backfill; long-running. Boot discovery re-enqueues while coverage is still missing, so suppressing a duplicate of an ACTIVE pass is safe.", + }, + work: { localConcurrency: ROLLUP_FULL_BACKFILL_CONCURRENCY }, + lock: (job: Job) => job.data.userId, + lane: true, + load: async () => (await handlers()).handleRollupFullBackfill, + }), + defineJob({ + queue: MOOD_ROLLUP_RECOMPUTE_QUEUE, + policy: { + policy: "short", + reason: + "Same shape as the measurement recompute: re-reads the bucket at run time, so only queued duplicates may be collapsed.", + }, + work: { localConcurrency: MOOD_ROLLUP_RECOMPUTE_CONCURRENCY }, + load: async () => (await handlers()).handleMoodRollupRecompute, + }), + defineJob({ + queue: MOOD_ROLLUP_FULL_BACKFILL_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user mood backfill; boot-discovery driven and self-converging on rollup coverage.", + }, + work: { localConcurrency: MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY }, + load: async () => (await handlers()).handleMoodRollupFullBackfill, + }), + defineJob({ + queue: STEP_CONSOLIDATION_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user legacy step consolidation; discovery drops the user once no live granular rows remain.", + }, + work: { localConcurrency: STEP_CONSOLIDATION_CONCURRENCY }, + load: async () => (await handlers()).handleStepConsolidation, + }), + defineJob({ + queue: STEP_CONSOLIDATION_REPAIR_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user one-shot repair; discovery drops the user once the affected provider rows are repaired.", + }, + work: { localConcurrency: STEP_CONSOLIDATION_REPAIR_CONCURRENCY }, + load: async () => (await handlers()).handleStepConsolidationRepair, + }), + defineJob({ + queue: CUMULATIVE_PR_REDERIVE_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user one-shot re-derivation; discovery drops the user once the inflated records are rewritten.", + }, + work: { localConcurrency: CUMULATIVE_PR_REDERIVE_CONCURRENCY }, + load: async () => (await handlers()).handleCumulativePrRederive, + }), + defineJob({ + queue: MEAN_CONSOLIDATION_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user daily-mean consolidation; discovery drops the user once the spot rows are folded.", + }, + work: { localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY }, + load: async () => (await handlers()).handleMeanConsolidation, + }), + defineJob({ + queue: DENSE_INTRADAY_RETENTION_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user dense-tier drain; the multi-hour pass on a heavy account is exactly the case where a restart used to append a duplicate full-history job.", + }, + work: { localConcurrency: DENSE_INTRADAY_RETENTION_CONCURRENCY }, + load: async () => (await handlers()).handleDenseIntradayRetention, + }), + defineJob({ + queue: DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, + policy: { + policy: "exclusive", + reason: + "Per-user one-shot hourly rebuild; discovery drops the day once its retired daily row is gone.", + }, + work: { localConcurrency: DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY }, + load: async () => (await handlers()).handleDenseIntradayHourlyRebuild, + }), + // One sequential walk, so it never crowds the dashboard request pool. + defineJob({ + queue: DRAIN_CUMULATIVE_QUEUE, + cron: DRAIN_CUMULATIVE_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async ({ boss }) => (await handlers()).drainCumulativeHandler(boss), + }), + defineJob({ + queue: SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE, + policy: { + policy: "short", + reason: + "Per-account recompute of supplement nutrient rows; the pass reads the live doses and composition when it starts, so a queued duplicate may be collapsed, while a send during a running pass must still queue the next one.", + }, + work: { localConcurrency: SUPPLEMENT_NUTRIENT_RECOMPUTE_CONCURRENCY }, + load: async () => (await handlers()).handleSupplementNutrientRecompute, + }), +]; + +const discovery: readonly BootDiscovery[] = [ + { + name: "rollup-full-backfill", + run: async () => + ( + await import("@/lib/rollups/measurement-rollups") + ).enqueueBootTimeRollupBackfill(BOOT_BACKFILL_STAGGER_SECONDS * 0), + }, + { + name: "step-consolidation", + run: async () => + ( + await import("@/lib/jobs/step-consolidation") + ).enqueueBootTimeStepConsolidation(BOOT_BACKFILL_STAGGER_SECONDS * 1), + }, + { + name: "mean-consolidation", + run: async () => + ( + await import("@/lib/jobs/mean-consolidation") + ).enqueueBootTimeMeanConsolidation(BOOT_BACKFILL_STAGGER_SECONDS * 2), + }, + // Defers itself past the boot window (the pool-exhaustion fix). + { + name: "dense-intraday-retention", + run: async () => + ( + await import("@/lib/jobs/dense-intraday-retention") + ).enqueueBootTimeDenseIntradayRetention(), + }, + // Defers past the boot window and the retention drain's own stage. + { + name: "dense-intraday-hourly-rebuild", + run: async () => + ( + await import("@/lib/jobs/dense-intraday-hourly-rebuild") + ).enqueueBootTimeDenseIntradayHourlyRebuild(), + }, + { + name: "mood-rollup-full-backfill", + run: async () => + ( + await import("@/lib/rollups/mood-rollups") + ).enqueueBootTimeMoodRollupBackfill(BOOT_BACKFILL_STAGGER_SECONDS * 3), + }, + { + name: "step-consolidation-repair", + run: async () => + ( + await import("@/lib/jobs/step-consolidation-repair") + ).enqueueBootTimeStepConsolidationRepair( + BOOT_BACKFILL_STAGGER_SECONDS * 5, + ), + }, + { + name: "cumulative-pr-rederive", + run: async () => + ( + await import("@/lib/personal-records/cumulative-pr-rederivation") + ).enqueueBootTimeCumulativePrRederivation( + BOOT_BACKFILL_STAGGER_SECONDS * 6, + ), + }, +]; + +export const ROLLUP_GROUP: JobGroup = { + name: "rollup", + jobs, + retired: RETIRED_ROLLUP_QUEUES, + discovery, +}; diff --git a/src/lib/jobs/registry/status.ts b/src/lib/jobs/registry/status.ts new file mode 100644 index 000000000..104007bcd --- /dev/null +++ b/src/lib/jobs/registry/status.ts @@ -0,0 +1,260 @@ +/** + * The status, insight and computed-score queues: the nightly status batch, + * the comprehensive-insight pre-generation, the on-demand per-metric status, + * the morning digest refresh, the data-arrival spine with its reaction line + * and workout insight, the nightly scores, the period-narrative warm, the + * Coach memory refresh, nudge, reminder sweep and plan review, the low-stock + * pass and the daily-briefing fallback. + * + * Most queues here are keyless cron ticks, so they carry no policy. The + * LLM-bound warm queues (pre-generate, per-metric status, period narrative, + * Coach memory) pass `singletonSeconds` on every send, which pg-boss + * constrains whatever the policy, so they are left alone. The three that carry + * one key on the user's local date, which no wall-clock window can express. + */ +import type { Job } from "pg-boss"; + +import { COACH_MEMORY_REFRESH_QUEUE } from "@/lib/ai/coach/coach-memory-shared"; +import { DATA_ARRIVAL_QUEUE } from "@/lib/arrivals/emit-shared"; +import { REACTION_LINE_QUEUE } from "@/lib/arrivals/reaction-line-shared"; +import { + COACH_NUDGE_CRON, + COACH_NUDGE_QUEUE, +} from "@/lib/jobs/coach-nudge-queue"; +import { + COACH_PLAN_REVIEW_CRON, + COACH_PLAN_REVIEW_QUEUE, +} from "@/lib/jobs/coach-plan-review"; +import { + COACH_REMINDER_SWEEP_CRON, + COACH_REMINDER_SWEEP_QUEUE, +} from "@/lib/jobs/coach-reminder-sweep-queue"; +import { + DAILY_BRIEFING_CRON, + DAILY_BRIEFING_QUEUE, +} from "@/lib/jobs/daily-briefing-queue"; +import { + INSIGHT_PREGENERATE_CRON, + INSIGHT_PREGENERATE_EXPIRE_SECONDS, + INSIGHT_PREGENERATE_QUEUE, + type InsightPregeneratePayload, +} from "@/lib/jobs/insight-pregenerate-shared"; +import { + INSIGHT_STATUS_GENERATE_CONCURRENCY, + INSIGHT_STATUS_GENERATE_QUEUE, +} from "@/lib/jobs/insight-status-generate-queue"; +import { WHOLE_PASS } from "@/lib/jobs/long-pass"; +import { + MEDICATION_LOW_STOCK_CRON, + MEDICATION_LOW_STOCK_QUEUE, +} from "@/lib/jobs/medication-low-stock-queue"; +import { MORNING_DIGEST_REFRESH_QUEUE } from "@/lib/jobs/morning-digest-refresh-shared"; +import { + PERIOD_NARRATIVE_CRON, + PERIOD_NARRATIVE_EXPIRE_SECONDS, + PERIOD_NARRATIVE_QUEUE, +} from "@/lib/jobs/period-narrative-queue"; +import { + RECOVERY_SCORE_CRON, + RECOVERY_SCORE_QUEUE, +} from "@/lib/jobs/recovery-score-queue"; +import { + insightRetryOptions, + nightlyInsightPassOptions, +} from "@/lib/jobs/reminder/registrar-shared"; +import { + STRAIN_SCORE_CRON, + STRAIN_SCORE_QUEUE, +} from "@/lib/jobs/strain-score-queue"; +import { + STRESS_SCORE_CRON, + STRESS_SCORE_QUEUE, +} from "@/lib/jobs/stress-score-queue"; +import { + WORKOUT_INSIGHT_GENERATE_CONCURRENCY, + WORKOUT_INSIGHT_GENERATE_QUEUE, +} from "@/lib/jobs/workout-insight-generate-shared"; + +import { defineJob, type JobDefinition, type JobGroup } from "./define-job"; + +const GENERAL_STATUS_QUEUE = "insights-general-status"; +const GENERAL_STATUS_CRON = "0 2 * * *"; + +/** + * The six per-metric status crons that followed the 02:00 batch. The batch + * builds every card, so they only re-read cards it had just written; their + * schedules and queues are removed where they still exist. + */ +const RETIRED_STATUS_QUEUES = [ + "insights-blood-pressure-status", + "insights-weight-status", + "insights-pulse-status", + "insights-bmi-status", + "insights-mood-status", + "insights-medication-compliance-status", +] as const; + +const insights = () => import("@/lib/jobs/reminder/insights-handlers"); +const handlers = () => import("@/lib/jobs/reminder/status-handlers"); + +/** A pre-generation job that warms one account on a visit. */ +const isForcedWarm = (job: Job) => + Boolean(job.data?.force && job.data.userId); + +const jobs: readonly JobDefinition[] = [ + // Every per-metric status card, one provider call per account. The options + // ride on the schedule because the cron is the only sender. + defineJob({ + queue: GENERAL_STATUS_QUEUE, + cron: GENERAL_STATUS_CRON, + scheduleOptions: nightlyInsightPassOptions, + work: { localConcurrency: 1 }, + lock: () => WHOLE_PASS, + lane: true, + load: async () => (await insights()).handleGeneralStatusGenerate, + }), + // Two slots: the cohort walk and a visitor's forced warm. The walk is one + // pass at a time and runs in the lane; a forced warm is one per account and + // never waits behind the walk. + defineJob({ + queue: INSIGHT_PREGENERATE_QUEUE, + cron: INSIGHT_PREGENERATE_CRON, + scheduleOptions: { + ...insightRetryOptions, + expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS, + }, + work: { localConcurrency: 2 }, + lock: (job: Job) => + isForcedWarm(job) ? `user:${job.data.userId}` : WHOLE_PASS, + lane: (job: Job) => !isForcedWarm(job), + load: async () => (await insights()).handleInsightPregenerateJob, + }), + // Enqueued by the read-only status route on a cold card; low concurrency so + // a first visit cannot fan out unbounded provider calls. + defineJob({ + queue: INSIGHT_STATUS_GENERATE_QUEUE, + work: { localConcurrency: INSIGHT_STATUS_GENERATE_CONCURRENCY }, + load: async () => (await insights()).handleInsightStatusGenerate, + }), + // Seven sleep-write seams enqueue the same user + local-date key. + defineJob({ + queue: MORNING_DIGEST_REFRESH_QUEUE, + policy: { + policy: "exclusive", + reason: + "Seven sleep-write seams enqueue the same user+localDate key; exclusive makes the at-most-once-per-morning contract real. A wall-clock singletonSeconds window cannot express a local date.", + }, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleMorningDigestRefresh, + }), + // The arrival spine claims a durable unique row, so a second slot would only + // race the claim; it makes no provider call. + defineJob({ + queue: DATA_ARRIVAL_QUEUE, + policy: { + policy: "exclusive", + reason: + "Day-scoped arrival keys carry the user's local date; under standard they coalesce nothing and a chatty ingest queues a job per batch. The worker claims a durable unique row, so a concurrent second run is pure duplicated work.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/data-arrival")).handleDataArrival, + }), + // The spine's one provider call, behind the durable `generatedAt` claim. + defineJob({ + queue: REACTION_LINE_QUEUE, + policy: { + policy: "exclusive", + reason: + "Day-scoped per-kind keys carry the user's local date; under standard they coalesce nothing and a retried spine job could stack a second provider call behind the first.", + }, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/reaction-line")).handleReactionLine, + }), + defineJob({ + queue: WORKOUT_INSIGHT_GENERATE_QUEUE, + work: { localConcurrency: WORKOUT_INSIGHT_GENERATE_CONCURRENCY }, + load: async () => + (await import("@/lib/jobs/workout-insight-generate")) + .handleWorkoutInsightGenerate, + }), + // The nightly scores, in order after the consolidation and drain they read: + // recovery 04:45, stress 04:50, strain 04:55. + defineJob({ + queue: RECOVERY_SCORE_QUEUE, + cron: RECOVERY_SCORE_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleRecoveryScore, + }), + defineJob({ + queue: STRESS_SCORE_QUEUE, + cron: STRESS_SCORE_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleStressScore, + }), + defineJob({ + queue: STRAIN_SCORE_QUEUE, + cron: STRAIN_SCORE_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleStrainScore, + }), + // The nightly fan-out runs on a week or month boundary, in the lane and one + // pass at a time; a single account's warm for the GET runs beside it. + defineJob({ + queue: PERIOD_NARRATIVE_QUEUE, + cron: PERIOD_NARRATIVE_CRON, + scheduleOptions: { + ...insightRetryOptions, + expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS, + }, + work: { localConcurrency: 1 }, + lock: (job: Job<{ userId?: string }>) => + job.data?.userId ? `user:${job.data.userId}` : WHOLE_PASS, + lane: (job: Job<{ userId?: string }>) => !job.data?.userId, + load: async () => (await handlers()).handlePeriodNarrative, + }), + // One slot, so a burst of long conversations cannot fan out provider calls. + defineJob({ + queue: COACH_MEMORY_REFRESH_QUEUE, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleCoachMemoryRefresh, + }), + defineJob({ + queue: COACH_NUDGE_QUEUE, + cron: COACH_NUDGE_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleCoachNudge, + }), + defineJob({ + queue: COACH_REMINDER_SWEEP_QUEUE, + cron: COACH_REMINDER_SWEEP_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleCoachReminderSweep, + }), + defineJob({ + queue: COACH_PLAN_REVIEW_QUEUE, + cron: COACH_PLAN_REVIEW_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleCoachPlanReview, + }), + defineJob({ + queue: MEDICATION_LOW_STOCK_QUEUE, + cron: MEDICATION_LOW_STOCK_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleMedicationLowStock, + }), + // Every 15 minutes; the per-user local-hour gate picks each user's slot. + defineJob({ + queue: DAILY_BRIEFING_QUEUE, + cron: DAILY_BRIEFING_CRON, + work: { localConcurrency: 1 }, + load: async () => (await handlers()).handleDailyBriefing, + }), +]; + +export const STATUS_GROUP: JobGroup = { + name: "status", + jobs, + retired: RETIRED_STATUS_QUEUES, +}; diff --git a/src/lib/jobs/reminder-worker.ts b/src/lib/jobs/reminder-worker.ts index e486a92db..c882ca002 100644 --- a/src/lib/jobs/reminder-worker.ts +++ b/src/lib/jobs/reminder-worker.ts @@ -1,15 +1,12 @@ /** - * pg-boss based reminder worker — boot composition. + * pg-boss worker boot. * - * v1.18.1 — this file was a 2143-LOC monolith that declared every pg-boss - * queue name, listed them in one `allQueues` array, scheduled every cron, and - * bound every `boss.work` handler inline. It is now a thin registration table: - * `startReminderWorker()` boots pg-boss, runs the boot one-shots, then calls - * the domain registrars under `src/lib/jobs/reminder/`. Each registrar owns the - * four facts the v1.4.37 dead-queue guards pin — the queue-name constant, its - * `allQueues` membership, its `[QUEUE, CRON]` schedule tuple, and its - * `createAndWork(boss, QUEUE, …, handler)` binding — so a queue can never be - * declared without being provisioned, scheduled, and drained. + * `startReminderWorker()` starts pg-boss, registers every queue from the + * declarative registry (`./registry/`), runs the boot one-shots and the boot + * discovery. The registry declares each queue with its schedule, policy, + * concurrency and handler loader; a queue cannot be declared without being + * created, scheduled and drained. Handler modules load on their queue's first + * job, not at boot. * * Usage: Run as a standalone process or call startReminderWorker() from a * custom server setup. In dev, use: npx tsx src/lib/jobs/reminder-worker.ts @@ -20,23 +17,15 @@ import { reportJobsCutOffAtBoot } from "@/lib/jobs/job-observer"; import { beginBootStep, reportBootSteps } from "@/lib/boot/boot-steps"; import { assertSubsystemEnabled } from "@/lib/process-type"; import { DATABASE_URL, workerLog } from "./reminder/shared"; -import type { PgBoss } from "pg-boss"; /** - * Give the event loop a turn. Each registrar's module graph is evaluated in - * one synchronous stretch when it is first imported; loading the five one - * after another, with a turn in between, keeps the longest stall to the - * largest graph instead of the sum of all five, so requests and the health - * probe are answered in between. + * Give the event loop a turn around a module load, so requests and the + * health probe are answered between boot steps. */ const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve)); -/** - * Import one registrar module as its own boot step. The registrars and the - * handler graphs behind them are loaded here, after `boss.start()`, rather - * than when this module is imported. - */ -async function loadRegistrar( +/** Import one boot module as its own boot step. */ +async function loadBootModule( name: string, load: () => Promise, ): Promise { @@ -74,65 +63,25 @@ export async function startReminderWorker() { markWorkerStarted(); // Graceful shutdown: drain in-flight jobs on SIGTERM/SIGINT. - const lifecycle = await loadRegistrar( + const lifecycle = await loadBootModule( "worker_lifecycle", () => import("./reminder/worker-lifecycle"), ); lifecycle.registerShutdownHandlers(boss); - // Register every domain's queues (createQueue + schedule + boss.work). Each - // registrar returns the queue names it provisioned; the aggregate is the - // boot-level mirror of the per-registrar `allQueues` arrays the dead-queue - // guards pin. Order preserves the monolith's createQueue / schedule / work - // sequence (integration → status → rollup → reminders → maintenance). - const integration = await loadRegistrar( - "integration_sync", - () => import("./reminder/register-integration-sync"), + // Register every queue (createQueue + schedule + boss.work) from the + // declarative registry, one group per boot step. The registry holds + // declarations only; each handler module loads on its queue's first job. + const registry = await loadBootModule( + "registry", + () => import("./registry/index"), ); - const registeredQueues = await registerInStep( - "integration_sync", - boss, - integration.registerIntegrationSyncQueues, - ); - const status = await loadRegistrar( - "status", - () => import("./reminder/register-status"), - ); - registeredQueues.push( - ...(await registerInStep("status", boss, status.registerStatusQueues)), - ); - const rollup = await loadRegistrar( - "rollup", - () => import("./reminder/register-rollup"), - ); - registeredQueues.push( - ...(await registerInStep("rollup", boss, rollup.registerRollupQueues)), - ); - const reminders = await loadRegistrar( - "reminders", - () => import("./reminder/register-reminders"), - ); - registeredQueues.push( - ...(await registerInStep( - "reminders", - boss, - reminders.registerReminderQueues, - )), - ); - const maintenance = await loadRegistrar( - "maintenance", - () => import("./reminder/register-maintenance"), - ); - registeredQueues.push( - ...(await registerInStep( - "maintenance", - boss, - maintenance.registerMaintenanceQueues, - )), + const registeredQueues = await registry.registerAllJobGroups(boss, (group) => + beginBootStep(`register_${group.name}`), ); workerLog( "info", - `[reminder-worker] registered ${registeredQueues.length} queues across 5 domain registrars`, + `[reminder-worker] registered ${registeredQueues.length} queues across ${registry.JOB_GROUPS.length} groups`, ); // v1.4.34 — reconcile any `ImportJob` rows that were mid-parse when the @@ -146,26 +95,14 @@ export async function startReminderWorker() { // Fire-and-forget boot discovery for every self-converging backfill / // consolidation pass. Each is idempotent across reboots and never fails - // worker boot on a miss (errors come back through the helper's result). + // worker boot on a miss. beginBootStep("boot_discovery"); - await rollup.enqueueRollupBootDiscovery(); - await integration.enqueueIntegrationSyncBootDiscovery(); - await maintenance.enqueueMaintenanceBootDiscovery(); + await registry.runAllBootDiscovery(); await reportBootSteps({ boot_queue_count: registeredQueues.length }); return boss; } -/** Run one registrar as its own boot step. */ -async function registerInStep( - name: string, - boss: PgBoss, - register: (boss: PgBoss) => Promise, -): Promise { - beginBootStep(`register_${name}`); - return [...(await register(boss))]; -} - // Run standalone if ( process.argv[1]?.endsWith("reminder-worker.ts") || diff --git a/src/lib/jobs/reminder/climb-backfill-at-boot.ts b/src/lib/jobs/reminder/climb-backfill-at-boot.ts new file mode 100644 index 000000000..344b9c017 --- /dev/null +++ b/src/lib/jobs/reminder/climb-backfill-at-boot.ts @@ -0,0 +1,34 @@ +/** + * The one-time climb backfill for workouts stored with a track but no + * elevation (`workouts/climb-backfill.ts`), started at boot and left to run + * detached: it reads one track at a time with a pause between batches and + * resumes from its cursor on the next boot, so it never holds the boot or the + * heap. A finished pass returns at once. + */ +import { trackBackgroundTask } from "@/lib/logging/background-tasks"; + +import { workerLog } from "./shared"; + +export async function startClimbBackfill(): Promise { + trackBackgroundTask( + (async () => { + const [{ prisma }, { runWorkoutClimbBackfill }] = await Promise.all([ + import("@/lib/db"), + import("@/lib/workouts/climb-backfill"), + ]); + const result = await runWorkoutClimbBackfill({ prisma }); + if (result.scanned > 0) { + workerLog( + "info", + `[workout-climb-backfill] scanned=${result.scanned} filled=${result.filled} done=${result.done}`, + ); + } + })().catch((err) => { + workerLog( + "error", + "[workout-climb-backfill] failed; the next boot resumes from the cursor", + err, + ); + }), + ); +} diff --git a/src/lib/jobs/reminder/document-handlers.ts b/src/lib/jobs/reminder/document-handlers.ts new file mode 100644 index 000000000..fa7c2d55c --- /dev/null +++ b/src/lib/jobs/reminder/document-handlers.ts @@ -0,0 +1,233 @@ +/** + * Handlers of the document vault queues that are not a module's own export: + * the content-index backfill, the per-document index, thumbnail and summary, + * the auto-read catch-up and the thumbnail backfill. A payload without its ids + * is counted as skipped rather than failed. The registry binds them + * (`registry/maintenance.ts`). + */ +import type { Job } from "pg-boss"; + +import { + runContentIndexBackfillForUser, + type ContentIndexBackfillPayload, +} from "@/lib/jobs/document-content-index-backfill"; +import { + runDocumentIndex, + type DocumentIndexPayload, +} from "@/lib/jobs/document-index"; +import { + runDocumentSummaryJob, + type DocumentSummaryPayload, +} from "@/lib/jobs/document-summary"; +import { + runSummaryCatchUpForUser, + type SummaryCatchUpPayload, +} from "@/lib/jobs/document-summary-catchup"; +import { + runDocumentThumbnail, + type DocumentThumbnailPayload, +} from "@/lib/jobs/document-thumbnail"; +import { + runThumbnailBackfillForUser, + type ThumbnailBackfillPayload, +} from "@/lib/jobs/document-thumbnail-backfill"; +import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { recordError } from "@/lib/jobs/worker-status"; + +import { workerLog } from "./shared"; + +/** + * The on-demand "index all documents" run for one account: consent and + * budget gated, bounded and resumable. + */ +export async function handleContentIndexBackfill( + jobs: Job[], +): Promise { + let users = 0; + let stoppedEarly = false; + const shouldStop = jobBudget(jobs); + let skipped = 0; + let documentsIndexed = 0; + let documentsRetokenised = 0; + let documentsSkipped = 0; + let documentsFailed = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + skipped++; + continue; + } + try { + const { + indexed, + retokenised, + skipped: docsSkipped, + failed: docsFailed, + reason, + stoppedEarly: stopped, + } = await runContentIndexBackfillForUser(userId, shouldStop); + users++; + stoppedEarly ||= stopped === true; + documentsIndexed += indexed; + documentsRetokenised += retokenised; + documentsSkipped += docsSkipped; + documentsFailed += docsFailed; + workerLog( + "info", + `[document-content-index-backfill] user=${userId} indexed=${indexed} retokenised=${retokenised} skipped=${docsSkipped} failed=${docsFailed} reason=${reason}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[document-content-index-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + skipped, + documents_indexed: documentsIndexed, + documents_retokenised: documentsRetokenised, + documents_skipped: documentsSkipped, + documents_failed: documentsFailed, + stopped_early: stoppedEarly, + }); +} + +/** One stored document's job, by the step that runs it. */ +function perDocument

( + name: string, + run: (payload: P) => Promise, +): (jobs: Job

[]) => Promise { + return async (jobs) => { + let documents = 0; + let skipped = 0; + for (const job of jobs) { + const { userId, documentId } = job.data; + if (!userId || !documentId) { + skipped++; + continue; + } + try { + await run(job.data); + documents++; + } catch (err) { + recordError(); + workerLog( + "error", + `[${name}] user=${userId} document=${documentId} failed`, + err, + ); + throw err; + } + } + return jobDone({ jobs: jobs.length, documents, skipped }); + }; +} + +/** + * Content indexing on upload: the vision provider when configured and + * consented, the local text layer otherwise. + */ +export const handleDocumentIndex = perDocument( + "document-index", + runDocumentIndex, +); + +/** The preview thumbnail on upload: local decode and downscale, no egress. */ +export const handleDocumentThumbnail = perDocument( + "document-thumbnail", + runDocumentThumbnail, +); + +/** + * The plain-language summary on upload, only with the auto-read opt-in on + * (consent and budget gated), stored encrypted. + */ +export const handleDocumentSummary = perDocument( + "document-summary", + runDocumentSummaryJob, +); + +/** + * After a genuine off-to-on flip of the auto-read opt-in: one summary job per + * stored document the upload-time enqueue skipped. Every gate still runs + * inside the summary job. + */ +export async function handleDocumentSummaryCatchUp( + jobs: Job[], +): Promise { + let users = 0; + let skipped = 0; + let summariesEnqueued = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + skipped++; + continue; + } + try { + const { enqueued } = await runSummaryCatchUpForUser(userId); + users++; + summariesEnqueued += enqueued; + } catch (err) { + recordError(); + workerLog( + "error", + `[document-summary-catchup] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + skipped, + summaries_enqueued: summariesEnqueued, + }); +} + +/** One thumbnail job per not-yet-previewed document of the account. */ +export async function handleDocumentThumbnailBackfill( + jobs: Job[], +): Promise { + let users = 0; + let skipped = 0; + let thumbnailsEnqueued = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + skipped++; + continue; + } + try { + const { enqueued } = await runThumbnailBackfillForUser(userId); + users++; + thumbnailsEnqueued += enqueued; + workerLog( + "info", + `[document-thumbnail-backfill] user=${userId} enqueued=${enqueued}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[document-thumbnail-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + skipped, + thumbnails_enqueued: thumbnailsEnqueued, + }); +} diff --git a/src/lib/jobs/reminder/encryption-backfill-handlers.ts b/src/lib/jobs/reminder/encryption-backfill-handlers.ts new file mode 100644 index 000000000..33a8bde60 --- /dev/null +++ b/src/lib/jobs/reminder/encryption-backfill-handlers.ts @@ -0,0 +1,214 @@ +/** + * Handlers of the three converging encryption backfills: the mood and + * measurement notes, the medication notes, and the remaining free text (Coach + * titles, custom-metric notes, practitioner contacts, workout tracks). A job + * with a `userId` migrates that account; the daily tick (no `userId`) re-runs + * the boot discovery and fans out one job per account still holding a + * readable value. The registry binds them (`registry/maintenance.ts`). + */ +import type { Job } from "pg-boss"; + +import { + enqueueBootTimeFreeTextEncryptionBackfill, + runFreeTextEncryptionBackfillForUser, + type FreeTextEncryptionBackfillPayload, +} from "@/lib/jobs/free-text-encryption-backfill"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { + enqueueBootTimeMedNotesEncryptionBackfill, + runMedNotesEncryptionBackfillForUser, + type MedNotesEncryptionBackfillPayload, +} from "@/lib/jobs/med-notes-encryption-backfill"; +import { + enqueueBootTimeNoteEncryptionBackfill, + runNoteEncryptionBackfillForUser, + type NoteEncryptionBackfillPayload, +} from "@/lib/jobs/note-encryption-backfill"; +import { recordError } from "@/lib/jobs/worker-status"; + +import { workerLog } from "./shared"; + +export async function handleNoteEncryptionBackfill( + jobs: Job>[], +): Promise { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + // The discovery helper returns its error instead of throwing, so a failed + // fan-out would otherwise leave no trace on the job at all. + let discoveryFailed = 0; + let measurements = 0; + let moodEntries = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + const result = await enqueueBootTimeNoteEncryptionBackfill(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; + workerLog( + "info", + `[note-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, + ); + continue; + } + try { + const { measurementsMigrated, moodEntriesMigrated } = + await runNoteEncryptionBackfillForUser(userId); + users++; + measurements += measurementsMigrated; + moodEntries += moodEntriesMigrated; + workerLog( + "info", + `[note-encryption-backfill] user=${userId} measurements=${measurementsMigrated} moodEntries=${moodEntriesMigrated}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[note-encryption-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + measurements_migrated: measurements, + mood_entries_migrated: moodEntries, + }); +} + +export async function handleMedNotesEncryptionBackfill( + jobs: Job>[], +): Promise { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + let discoveryFailed = 0; + let sideEffects = 0; + let doseChanges = 0; + let inventoryItems = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + const result = await enqueueBootTimeMedNotesEncryptionBackfill(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; + workerLog( + "info", + `[med-notes-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, + ); + continue; + } + try { + const { + sideEffectsMigrated, + doseChangesMigrated, + inventoryItemsMigrated, + } = await runMedNotesEncryptionBackfillForUser(userId); + users++; + sideEffects += sideEffectsMigrated; + doseChanges += doseChangesMigrated; + inventoryItems += inventoryItemsMigrated; + workerLog( + "info", + `[med-notes-encryption-backfill] user=${userId} sideEffects=${sideEffectsMigrated} doseChanges=${doseChangesMigrated} inventoryItems=${inventoryItemsMigrated}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[med-notes-encryption-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + side_effects_migrated: sideEffects, + dose_changes_migrated: doseChanges, + inventory_items_migrated: inventoryItems, + }); +} + +export async function handleFreeTextEncryptionBackfill( + jobs: Job>[], +): Promise { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + let discoveryFailed = 0; + let titles = 0; + let notes = 0; + let contacts = 0; + let routes = 0; + let appointmentAddresses = 0; + let contactAudits = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + const result = await enqueueBootTimeFreeTextEncryptionBackfill(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; + workerLog( + "info", + `[free-text-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, + ); + continue; + } + try { + const { + conversationTitlesMigrated, + metricNotesMigrated, + practitionerContactsMigrated, + routeGeometriesMigrated, + appointmentAddressesCleared, + contactAuditRowsScrubbed, + } = await runFreeTextEncryptionBackfillForUser(userId); + users++; + titles += conversationTitlesMigrated; + notes += metricNotesMigrated; + contacts += practitionerContactsMigrated; + routes += routeGeometriesMigrated; + appointmentAddresses += appointmentAddressesCleared; + contactAudits += contactAuditRowsScrubbed; + workerLog( + "info", + `[free-text-encryption-backfill] user=${userId} titles=${conversationTitlesMigrated} notes=${metricNotesMigrated} contacts=${practitionerContactsMigrated} routes=${routeGeometriesMigrated} appointment_addresses=${appointmentAddressesCleared} contact_audits=${contactAuditRowsScrubbed}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[free-text-encryption-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + conversation_titles_migrated: titles, + metric_notes_migrated: notes, + practitioner_contacts_migrated: contacts, + route_geometries_migrated: routes, + appointment_addresses_cleared: appointmentAddresses, + contact_audit_rows_scrubbed: contactAudits, + }); +} diff --git a/src/lib/jobs/reminder/maintenance-handlers.ts b/src/lib/jobs/reminder/maintenance-handlers.ts new file mode 100644 index 000000000..1f1facfd7 --- /dev/null +++ b/src/lib/jobs/reminder/maintenance-handlers.ts @@ -0,0 +1,186 @@ +/** + * Handlers of the maintenance queues that are not a module's own export: the + * Apple Health and medication-intake imports, the duplicate dose-slot dedup, + * and the boss-bound environment and Dawarich passes. The registry binds them + * (`registry/maintenance.ts`) and loads this module on the first job. + */ +import type { Job, JobWithMetadata, PgBoss } from "pg-boss"; + +import { + handleAppleHealthImport, + migrateLegacyAppleHealthImport, + type AppleHealthImportPayload, +} from "@/lib/jobs/apple-health-import-worker"; +import { + handleDawarichSync, + type DawarichSyncPayload, +} from "@/lib/jobs/dawarich-sync"; +import { + handleEnvironmentAqHistory, + type EnvironmentAqHistoryPayload, +} from "@/lib/jobs/environment-air-quality-history"; +import { + handleEnvironmentFetch, + type EnvironmentFetchPayload, +} from "@/lib/jobs/environment-fetch"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { + handleMedicationIntakeImport, + type MedicationIntakeImportQueuePayload, +} from "@/lib/jobs/medication-intake-import"; +import { recordError } from "@/lib/jobs/worker-status"; +import { + dedupeUserIntakeSlots, + enqueueBootTimeIntakeSlotDedup, + type IntakeSlotDedupPayload, +} from "@/lib/medications/intake-slot-dedup"; + +import { workerLog } from "./shared"; + +/** + * The Apple Health export ingest. Each import is independent work, so a + * failed one does not abort the rest of the batch; the first failure is what + * the job reports. + */ +export async function handleAppleHealthImportJobs( + jobs: Job[], +): Promise { + let failure: JobOutcome | null = null; + for (const job of jobs) { + const outcome = await handleAppleHealthImport(job); + if (!outcome.ok && failure === null) failure = outcome; + } + return failure ?? jobDone({ jobs: jobs.length }); +} + +/** + * Drains the pre-revision-2 backlog by moving each claimed legacy job onto + * the current queue, keeping the import's status row, so the current parser + * never runs under a revision-1 mirror. + */ +export async function handleLegacyAppleHealthImportJobs( + jobs: Job[], +): Promise { + for (const job of jobs) { + await migrateLegacyAppleHealthImport(job); + } + return jobDone({ jobs: jobs.length }); +} + +/** + * Independent per-delivery medication-intake imports: run them all and report + * the first failure rather than losing it behind a later success. Bound with + * metadata, so a retry resumes from the job's creation time. + */ +export async function handleMedicationIntakeImportJobs( + jobs: JobWithMetadata[], +): Promise { + let failure: JobOutcome | null = null; + for (const job of jobs) { + const outcome = await handleMedicationIntakeImport(job); + if (!outcome.ok && failure === null) failure = outcome; + } + return failure ?? jobDone({ jobs: jobs.length }); +} + +/** + * The duplicate dose-slot cleanup. A job with a `userId` keeps one account's + * winner per slot, soft-deletes the losers and recomputes the affected + * compliance days; the daily tick (no `userId`) runs the same discovery + * fan-out as the boot pass, so slots duplicated between deploys collapse + * within a day. + */ +export async function handleIntakeSlotDedup( + jobs: Job>[], +): Promise { + let users = 0; + let discoveryEnqueued = 0; + let discoverySkipped = 0; + // The discovery helper returns its error instead of throwing, so a failed + // fan-out would otherwise leave no trace on the job at all. + let discoveryFailed = 0; + let slotsCollapsed = 0; + let rowsSoftDeleted = 0; + let rowsNormalised = 0; + let daysRecomputed = 0; + for (const job of jobs) { + const { userId } = job.data; + if (!userId) { + const result = await enqueueBootTimeIntakeSlotDedup(); + discoveryEnqueued += result.enqueued; + discoverySkipped += result.skipped; + if (result.error) discoveryFailed++; + workerLog( + "info", + `[intake-slot-dedup] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, + ); + continue; + } + try { + const summary = await dedupeUserIntakeSlots(userId); + users++; + slotsCollapsed += summary.slotsCollapsed; + rowsSoftDeleted += summary.rowsSoftDeleted; + rowsNormalised += summary.rowsNormalised; + daysRecomputed += summary.daysRecomputed; + workerLog( + "info", + `[intake-slot-dedup] user=${userId} slotsCollapsed=${summary.slotsCollapsed} rowsSoftDeleted=${summary.rowsSoftDeleted} rowsNormalised=${summary.rowsNormalised} daysRecomputed=${summary.daysRecomputed}`, + ); + } catch (err) { + recordError(); + workerLog("error", `[intake-slot-dedup] user=${userId} failed`, err); + throw err; + } + } + return jobDone({ + jobs: jobs.length, + users, + discovery_enqueued: discoveryEnqueued, + discovery_skipped: discoverySkipped, + discovery_failed: discoveryFailed, + slots_collapsed: slotsCollapsed, + rows_soft_deleted: rowsSoftDeleted, + rows_normalised: rowsNormalised, + days_recomputed: daysRecomputed, + }); +} + +/** The air-quality history backfill; the first failure is what the job reports. */ +export function environmentAqHistoryHandler( + boss: PgBoss, +): (jobs: Job[]) => Promise { + return async (jobs) => { + let failure: JobOutcome | null = null; + for (const job of jobs) { + const outcome = await handleEnvironmentAqHistory(boss, job.data); + if (!outcome.ok && failure === null) failure = outcome; + } + return failure ?? jobDone({ jobs: jobs.length }); + }; +} + +/** + * The environment fetch. A discovery tick and a per-user backfill share the + * queue and are independent, so the pass runs to the end and reports the + * first failure it saw. + */ +export function environmentFetchHandler( + boss: PgBoss, +): (jobs: Job[]) => Promise { + return async (jobs) => { + let failure: JobOutcome | null = null; + for (const job of jobs) { + const outcome = await handleEnvironmentFetch(boss, job.data); + if (!outcome.ok && failure === null) failure = outcome; + } + return failure ?? jobDone({ jobs: jobs.length }); + }; +} + +/** The Dawarich movement context, one account at a time. */ +export function dawarichSyncHandler( + boss: PgBoss, +): (jobs: Job[]) => Promise { + return (jobs) => handleDawarichSync(boss, jobs); +} diff --git a/src/lib/jobs/reminder/register-integration-sync.ts b/src/lib/jobs/reminder/register-integration-sync.ts deleted file mode 100644 index d96055632..000000000 --- a/src/lib/jobs/reminder/register-integration-sync.ts +++ /dev/null @@ -1,1051 +0,0 @@ -/** - * Integration-sync queue registrar. - * - * Owns the third-party-provider sync queues — Withings (fallback / activity / - * sleep), WHOOP (recovery / sleep / workout / cycle + backfill + OAuth-state - * cleanup), Fitbit (sync + backfill + sleep repair + OAuth-state cleanup), - * Nightscout, Polar, Oura, and the two one-shot backfills - * (sleep-timeline, lab-biomarker) — plus their boot-time discovery enqueues. - * - * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its - * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The - * queue-wiring guards (`withings-queues`, `whoop-queues`, `fitbit-queues`, - * `nightscout-queues`, `sleep-timeline-queues`) read THIS module. - */ -import { PgBoss } from "pg-boss"; -import { - WHOOP_BACKFILL_QUEUE, - WHOOP_BACKFILL_CONCURRENCY, - enqueueBootTimeWhoopBackfill, - type WhoopBackfillPayload, -} from "@/lib/jobs/whoop-backfill"; -import { - FITBIT_BACKFILL_QUEUE, - FITBIT_BACKFILL_CONCURRENCY, - enqueueBootTimeFitbitBackfill, - type FitbitBackfillPayload, -} from "@/lib/jobs/fitbit-backfill"; -import { - GOOGLE_HEALTH_BACKFILL_QUEUE, - GOOGLE_HEALTH_BACKFILL_CONCURRENCY, - enqueueBootTimeGoogleHealthBackfill, - type GoogleHealthBackfillPayload, -} from "@/lib/jobs/google-health-backfill"; -import { - GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, - GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY, - enqueueBootTimeGoogleHealthSleepRepair, - type GoogleHealthSleepRepairPayload, -} from "@/lib/jobs/google-health-sleep-repair"; -import { - FITBIT_SLEEP_REPAIR_QUEUE, - FITBIT_SLEEP_REPAIR_CONCURRENCY, - enqueueBootTimeFitbitSleepRepair, - type FitbitSleepRepairPayload, -} from "@/lib/jobs/fitbit-sleep-repair"; -import { - STRAVA_BACKFILL_QUEUE, - STRAVA_BACKFILL_CONCURRENCY, - enqueueBootTimeStravaBackfill, - type StravaBackfillPayload, -} from "@/lib/jobs/strava-backfill"; -import { - OURA_BACKFILL_QUEUE, - OURA_BACKFILL_CONCURRENCY, - enqueueBootTimeOuraBackfill, - type OuraBackfillPayload, -} from "@/lib/jobs/oura-backfill"; -import { - SLEEP_TIMELINE_BACKFILL_QUEUE, - SLEEP_TIMELINE_BACKFILL_CONCURRENCY, - enqueueBootTimeSleepTimelineBackfill, - type SleepTimelineBackfillPayload, -} from "@/lib/jobs/sleep-timeline-backfill"; -import { - LAB_BIOMARKER_BACKFILL_QUEUE, - LAB_BIOMARKER_BACKFILL_CONCURRENCY, - enqueueBootTimeLabBiomarkerBackfill, - type LabBiomarkerBackfillPayload, -} from "@/lib/jobs/lab-biomarker-backfill"; -import { - INTEGRATION_BACKFILL_ADMISSION_QUEUE, - INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, - bootStaggerSecondsFor, - enqueueIntegrationBackfillAdmission, - type IntegrationBackfillAdmissionPayload, -} from "@/lib/jobs/integration-backfill-admission"; -import { drainIntegrationBackfillAdmission } from "@/lib/jobs/integration-backfill-drain"; -import { heavyLane } from "@/lib/jobs/heavy-lane"; -import { - INTEGRATION_WATCHDOG_DAILY_CRON, - INTEGRATION_WATCHDOG_DAILY_QUEUE, - INTEGRATION_WATCHDOG_HOURLY_CRON, - INTEGRATION_WATCHDOG_HOURLY_QUEUE, - handleIntegrationWatchdogDaily, - handleIntegrationWatchdogHourly, - type IntegrationWatchdogPayload, -} from "@/lib/jobs/integration-watchdog"; -import { workerLog } from "./shared"; -import { jobDone } from "@/lib/jobs/job-outcome"; -import { - createAndSchedule, - createAndWork, - cronIsTheRetry, - type QueuePolicyTable, - type ScheduleEntry, -} from "./registrar-shared"; -import { WITHINGS_ECG_SYNC_QUEUE } from "@/lib/jobs/withings-ecg-queue"; -import { - NIGHTSCOUT_SYNC_QUEUE, - OURA_SYNC_QUEUE, - POLAR_SYNC_QUEUE, - STRAVA_SYNC_QUEUE, -} from "@/lib/jobs/integration-poll-queues"; -import { - WithingsSyncPayload, - WithingsActivitySyncPayload, - WithingsSleepSyncPayload, - WithingsEcgSyncPayload, - handleWithingsFallbackSync, - handleWithingsActivitySync, - handleWithingsSleepSync, - handleWithingsEcgSync, -} from "./withings-sync"; -import { - WhoopSyncPayload, - handleWhoopRecoverySync, - handleWhoopSleepSync, - handleWhoopWorkoutSync, - handleWhoopCycleSync, -} from "./whoop-sync"; -import { - WithingsOAuthStateCleanupPayload, - handleWithingsOAuthStateCleanup, - WhoopOAuthStateCleanupPayload, - handleWhoopOAuthStateCleanup, - OidcNativeHandoffCleanupPayload, - handleOidcNativeHandoffCleanup, -} from "./cleanup-handlers"; -import { NightscoutSyncPayload, handleNightscoutSync } from "./nightscout-sync"; -import { PolarSyncPayload, handlePolarSync } from "./polar-sync"; -import { OuraSyncPayload, handleOuraSync } from "./oura-sync"; -import { StravaSyncPayload, handleStravaSync } from "./strava-sync"; -import { - FitbitSyncPayload, - handleFitbitSync, - FitbitOAuthStateCleanupPayload, - handleFitbitOAuthStateCleanup, -} from "./fitbit-sync"; -import { - GoogleHealthSyncPayload, - handleGoogleHealthSync, - GoogleHealthOAuthStateCleanupPayload, - handleGoogleHealthOAuthStateCleanup, -} from "./google-health-sync"; - -const WITHINGS_SYNC_QUEUE = "withings-fallback-sync"; - -const WITHINGS_SYNC_CRON = "0 * * * *"; // every 60 minutes -// v1.4.25 — webhook-primary + cron-safety-net for activity and -// sleep v2. The webhook handler enqueues per-user jobs on appli=16 / 44 -// notifications; the crons below are the catch-net for the 1 % of -// notifications Withings drops. Offset 15-minute cadence per the -// research recommendation so the two queues don't lockstep against the -// existing measure cron at :00. - -const WITHINGS_ACTIVITY_QUEUE = "withings-activity-sync"; - -const WITHINGS_ACTIVITY_CRON = "0 * * * *"; // every hour at :00 - -const WITHINGS_SLEEP_QUEUE = "withings-sleep-sync"; - -const WITHINGS_SLEEP_CRON = "15 * * * *"; // every hour at :15 -// v1.32.23 — Withings ECG cron catch-net. The queue is webhook-primary -// (appli=54 → `enqueueWithingsEcgSync`), but appli 54 is not in the -// subscription set and the manual sync route measures only, so a ScanWatch-only -// account (no scale) never had its ECG strips pulled. An hourly cohort walk -// (one `v2/heart list` call per connected user, empty + cheap for non-ECG -// devices) closes the gap. :41 is a free minute (used: 0,5,8,11,13,15,17,18, -// 20,30,35,50) staggered off every other hourly sync tick. -const WITHINGS_ECG_SYNC_CRON = "41 * * * *"; // every hour at :41 - -// v1.4.47 — daily sweep for the Withings OAuth state ledger. Slots -// at :20 between the audit-log cleanup (:15) and the mood-reminder -// cleanup (:25), inside the existing 02:xx-03:xx maintenance window. -// Rows are normally consumed by the callback handler in single-use -// fashion; this sweep picks up the long tail where a user closed the -// Withings approval tab without bouncing back to the callback URL. - -const WITHINGS_OAUTH_STATE_CLEANUP_QUEUE = "withings-oauth-state-cleanup"; - -const WITHINGS_OAUTH_STATE_CLEANUP_CRON = "20 3 * * *"; -// v1.11.0 — WHOOP sync queues. Webhook-primary + cron-safety-net, mirroring -// the Withings activity/sleep crons. Recovery / sleep / workout each have a -// WHOOP webhook (`*.updated`) that enqueues the matching per-resource job; the -// crons below are the catch-net for dropped deliveries. Cycle has NO webhook, -// so its cron is the only driver. Minutes are staggered off the Withings crons -// (:00/:15) to spread DB load. - -const WHOOP_RECOVERY_SYNC_QUEUE = "whoop-recovery-sync"; - -const WHOOP_RECOVERY_SYNC_CRON = "5 * * * *"; // every hour at :05 - -const WHOOP_SLEEP_SYNC_QUEUE = "whoop-sleep-sync"; - -const WHOOP_SLEEP_SYNC_CRON = "20 * * * *"; // every hour at :20 - -const WHOOP_WORKOUT_SYNC_QUEUE = "whoop-workout-sync"; - -const WHOOP_WORKOUT_SYNC_CRON = "35 * * * *"; // every hour at :35 - -const WHOOP_CYCLE_SYNC_QUEUE = "whoop-cycle-sync"; - -const WHOOP_CYCLE_SYNC_CRON = "50 * * * *"; // every hour at :50 (poll-only) -// v1.11.0 — daily sweep for the WHOOP OAuth state ledger. Slots at 03:22, -// next to the Withings sweep (03:20), inside the maintenance window. - -const WHOOP_OAUTH_STATE_CLEANUP_QUEUE = "whoop-oauth-state-cleanup"; - -const WHOOP_OAUTH_STATE_CLEANUP_CRON = "22 3 * * *"; -// v1.12.0 — Fitbit / Google Health poll-only sync. There is no Fitbit webhook -// at launch (Pub/Sub deferred), so a single hourly cron drives the per-user -// `syncUserFitbit` driver across every connection. Minute staggered off the -// WHOOP slots (:05/:20/:35/:50) and the Withings slots (:00/:15) so the hourly -// ticks don't pile up on one boss poll. - -const FITBIT_SYNC_QUEUE = "fitbit-sync"; - -const FITBIT_SYNC_CRON = "8 * * * *"; // every hour at :08 -// v1.12.0 — daily sweep for the Fitbit OAuth state ledger. Slots at 03:24, next -// to the WHOOP sweep (03:22), inside the maintenance window. - -const FITBIT_OAUTH_STATE_CLEANUP_QUEUE = "fitbit-oauth-state-cleanup"; - -const FITBIT_OAUTH_STATE_CLEANUP_CRON = "24 3 * * *"; -// v1.26.0 — Google Health / Fitbit-Pixel-Wear-OS poll-only sync. There is no -// Google Health webhook at launch (Pub/Sub deferred), so a single hourly cron -// drives the per-user `syncUserGoogleHealth` driver across every connection. -// Minute staggered off the WHOOP (:05/:20/:35/:50), Fitbit (:08), Nightscout -// (:11), Polar (:13), and Oura (:15) slots so the hourly ticks don't pile up on -// one boss poll. - -const GOOGLE_HEALTH_SYNC_QUEUE = "google-health-sync"; - -const GOOGLE_HEALTH_SYNC_CRON = "18 * * * *"; // every hour at :18 -// v1.26.0 — daily sweep for the Google Health OAuth state ledger. Slots at -// 03:26, next to the Fitbit sweep (03:24), inside the maintenance window. - -const GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE = - "google-health-oauth-state-cleanup"; - -const GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON = "26 3 * * *"; -// v1.17.0 — Nightscout CGM poll sync. Poll-only (no webhook): one hourly tick -// pulls the recent SGV window per configured instance. :11 staggers off the -// WHOOP (:05), Fitbit (:08), and Withings sync ticks so the hourly polls don't -// pile up on one boss poll. -// -// The four poll-queue NAMES live in `@/lib/jobs/integration-poll-queues` (see -// the import above) so an OAuth callback can enqueue a connect-time first sync -// without importing this registrar and its whole handler graph. Creation, -// scheduling, and handler binding stay here. -const NIGHTSCOUT_SYNC_CRON = "11 * * * *"; // every hour at :11 -// v1.17.0 — Polar + Oura OAuth poll sync. Poll-only (one hourly tick per -// provider re-walks every connected user). :13 / :15 stagger off the other -// hourly sync ticks (WHOOP :05, Fitbit :08, Nightscout :11) so the polls don't -// pile up on one boss poll. The queues MUST be registered in `allQueues` below -// or pg-boss never provisions them and the schedule silently no-ops (the -// v1.4.37 dead-queue class). -const POLAR_SYNC_CRON = "13 * * * *"; // every hour at :13 -const OURA_SYNC_CRON = "15 * * * *"; // every hour at :15 -// v1.28.x — Strava OAuth poll sync. Poll-only (webhook deferred): one hourly -// tick re-walks every connected user. :17 staggers off the Polar (:13) / Oura -// (:15) ticks so the polls don't pile up on one boss poll. The queue MUST be -// registered in `allQueues` below or pg-boss never provisions it and the -// schedule silently no-ops (the v1.4.37 dead-queue class). -const STRAVA_SYNC_CRON = "17 * * * *"; // every hour at :17 -// v1.30.x — daily sweep for the native OIDC handoff ledger. Handoff codes are -// single-use + 90s-lived and expiry is enforced at read, so this is pure -// hygiene: it reaps rows the app never came back to exchange. Slots at 03:28, -// next to the Google Health OAuth-state sweep (03:26), inside the maintenance -// window. The queue MUST be registered in `allQueues` below or pg-boss never -// provisions it and the schedule silently no-ops (the v1.4.37 dead-queue class). -const OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE = "oidc-native-handoff-cleanup"; -const OIDC_NATIVE_HANDOFF_CLEANUP_CRON = "28 3 * * *"; - -// pg-boss v12 requires explicit queue creation before scheduling. Every queue -// MUST be registered here or pg-boss never provisions it and both the webhook -// enqueue AND the cron schedule below silently no-op (the v1.4.37 dead-queue -// class). -const allQueues = [ - WITHINGS_SYNC_QUEUE, - WITHINGS_ACTIVITY_QUEUE, - WITHINGS_SLEEP_QUEUE, - WITHINGS_ECG_SYNC_QUEUE, - WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, - // v1.11.0 — WHOOP sync queues. Webhook-primary + cron-safety-net for - // recovery / sleep / workout; cycle is poll-only (no WHOOP webhook). - WHOOP_RECOVERY_SYNC_QUEUE, - WHOOP_SLEEP_SYNC_QUEUE, - WHOOP_WORKOUT_SYNC_QUEUE, - WHOOP_CYCLE_SYNC_QUEUE, - // v1.11.0 — self-converging boot backfill for newly connected WHOOP - // accounts. Discovery enqueues one full-history sync per un-backfilled - // connection; idempotent across reboots. - WHOOP_BACKFILL_QUEUE, - // v1.11.0 — daily sweep for the WHOOP OAuth state ledger. - WHOOP_OAUTH_STATE_CLEANUP_QUEUE, - // v1.12.0 — Fitbit / Google Health poll-only sync (no webhook at launch), - // self-converging boot backfill, and the daily OAuth-state ledger sweep. - FITBIT_SYNC_QUEUE, - FITBIT_BACKFILL_QUEUE, - // One-shot Fitbit sleep duplicate repair. Discovery enqueues one bounded - // sleep re-read per connection not yet repaired; the replace-by-window write - // path collapses each night's leftovers. Idempotent across reboots. - FITBIT_SLEEP_REPAIR_QUEUE, - FITBIT_OAUTH_STATE_CLEANUP_QUEUE, - // v1.26.0 — Google Health poll-only sync (no webhook at launch), - // self-converging boot backfill, and the daily OAuth-state ledger sweep. - GOOGLE_HEALTH_SYNC_QUEUE, - GOOGLE_HEALTH_BACKFILL_QUEUE, - // v1.28.x — one-shot sleep duplicate repair. Discovery enqueues one full - // sleep re-read per connection not yet repaired; the replace-by-window write - // path collapses each night's duplicate rows. Idempotent across reboots. - GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, - GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, - // v1.17.1 — one-shot sleep-timeline backfill for WHOOP + Withings. - // Discovery enqueues one job per connection whose sleep rows predate the - // stamp/shape fix; the pass deletes the affected SLEEP_DURATION rows and - // re-syncs. Idempotent across reboots. - SLEEP_TIMELINE_BACKFILL_QUEUE, - // v1.18.1 — one-shot backfill that links legacy free-text lab readings to - // a user-scoped Biomarker catalog entry (group by `lower(analyte)`). - LAB_BIOMARKER_BACKFILL_QUEUE, - // Shared durable drain for every full-history integration job. Source queues - // retain their per-connection singleton keys; this queue supplies the one - // cross-provider execution lane. - INTEGRATION_BACKFILL_ADMISSION_QUEUE, - // v1.17.0 — Nightscout CGM poll sync. Poll-only (no webhook, no OAuth, no - // backfill queue — the hourly window walks the recent SGV set). - NIGHTSCOUT_SYNC_QUEUE, - // v1.17.0 — Polar + Oura OAuth poll sync. - POLAR_SYNC_QUEUE, - OURA_SYNC_QUEUE, - // v1.44 — Oura history backfill, sliced through the shared lane. - OURA_BACKFILL_QUEUE, - // v1.28.x — Strava OAuth poll sync + self-converging boot backfill. - STRAVA_SYNC_QUEUE, - STRAVA_BACKFILL_QUEUE, - // v1.30.x — daily sweep for the native OIDC handoff ledger. - OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, - // v1.42.4 — the integration watchdog: an hourly cheap pass and a daily - // pass that re-reads the history behind standing silence flags. - INTEGRATION_WATCHDOG_HOURLY_QUEUE, - INTEGRATION_WATCHDOG_DAILY_QUEUE, -]; - -const schedules: ScheduleEntry[] = [ - [WITHINGS_SYNC_QUEUE, WITHINGS_SYNC_CRON], - [WITHINGS_ACTIVITY_QUEUE, WITHINGS_ACTIVITY_CRON], - [WITHINGS_SLEEP_QUEUE, WITHINGS_SLEEP_CRON], - // v1.32.23 — hourly ECG catch-net (:41) so watch-only accounts pull their - // strips without a webhook or a scale. - [WITHINGS_ECG_SYNC_QUEUE, WITHINGS_ECG_SYNC_CRON], - // The five OAuth-state / handoff prunes carry `cronIsTheRetry`: their - // handlers used to warn and carry on, they now fail the job, and a DELETE - // that could not run will not run any better a second later. The daily tick - // is the retry. The sync crons above keep the default retries — a provider - // outage is transient and worth retrying inside the hour. - [ - WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, - WITHINGS_OAUTH_STATE_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.11.0 — WHOOP poll-fallback crons. Recovery/sleep/workout catch - // dropped webhooks; cycle is the sole driver (no webhook). Staggered off - // the Withings crons so the hourly ticks don't pile up on one boss poll. - [WHOOP_RECOVERY_SYNC_QUEUE, WHOOP_RECOVERY_SYNC_CRON], - [WHOOP_SLEEP_SYNC_QUEUE, WHOOP_SLEEP_SYNC_CRON], - [WHOOP_WORKOUT_SYNC_QUEUE, WHOOP_WORKOUT_SYNC_CRON], - [WHOOP_CYCLE_SYNC_QUEUE, WHOOP_CYCLE_SYNC_CRON], - // v1.11.0 — daily 03:22 Europe/Berlin prune for expired WHOOP OAuth states. - [ - WHOOP_OAUTH_STATE_CLEANUP_QUEUE, - WHOOP_OAUTH_STATE_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.12.0 — hourly Fitbit poll (:08, staggered off WHOOP/Withings) + the - // daily 03:24 Europe/Berlin prune for expired Fitbit OAuth states. - [FITBIT_SYNC_QUEUE, FITBIT_SYNC_CRON], - [ - FITBIT_OAUTH_STATE_CLEANUP_QUEUE, - FITBIT_OAUTH_STATE_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.26.0 — hourly Google Health poll (:18, staggered off WHOOP/Fitbit/ - // Nightscout/Polar/Oura) + the daily 03:26 Europe/Berlin prune for expired - // Google Health OAuth states. - [GOOGLE_HEALTH_SYNC_QUEUE, GOOGLE_HEALTH_SYNC_CRON], - [ - GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, - GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.17.0 — hourly Nightscout CGM poll (:11, staggered off the other sync - // ticks). - [NIGHTSCOUT_SYNC_QUEUE, NIGHTSCOUT_SYNC_CRON], - // v1.17.0 — hourly Polar (:13) + Oura (:15) OAuth polls. - [POLAR_SYNC_QUEUE, POLAR_SYNC_CRON], - [OURA_SYNC_QUEUE, OURA_SYNC_CRON], - // v1.28.x — hourly Strava (:17) OAuth poll. - [STRAVA_SYNC_QUEUE, STRAVA_SYNC_CRON], - // v1.42.4 — integration watchdog, hourly at :47 and daily at 04:52. The - // next tick is the retry: a missed pass only delays a flag by an hour. - [ - INTEGRATION_WATCHDOG_HOURLY_QUEUE, - INTEGRATION_WATCHDOG_HOURLY_CRON, - cronIsTheRetry, - ], - [ - INTEGRATION_WATCHDOG_DAILY_QUEUE, - INTEGRATION_WATCHDOG_DAILY_CRON, - cronIsTheRetry, - ], - // v1.30.x — daily 03:28 Europe/Berlin prune for expired native OIDC handoffs. - [ - OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, - OIDC_NATIVE_HANDOFF_CLEANUP_CRON, - cronIsTheRetry, - ], -]; - -/** - * De-duplication policy per integration-sync queue. - * - * The recurring poll queues (Withings / WHOOP / Fitbit / Google Health / - * Nightscout / Polar / Oura / Strava sync, and every OAuth-state cleanup) are - * keyless cron ticks — no `singletonKey` on any send — so a policy would only - * constrain the empty key. They are deliberately absent. - * - * The one-shot backfills and repairs below all share the same shape: one job - * per connection, keyed per user (per provider where a provider can differ), - * enqueued by a boot-discovery predicate that only stops offering the job once - * it has COMPLETED. That last property is what made the bug bite — a worker - * restarting during a heavy account's multi-hour full-history pass appended - * another identical job per restart, sustaining the boot storm the stagger - * deferrals exist to damp. `exclusive` covers queued, active, and retry-backoff, - * so the restart case is closed; and because discovery re-offers the job while - * the work is still outstanding, a suppressed send can never lose work. - */ -const queuePolicies: QueuePolicyTable = { - [WITHINGS_ECG_SYNC_QUEUE]: { - policy: "short", - reason: - "Webhook retries share one identity while queued; once work starts, a later replay remains admissible as a rescue if the original exhausts retries.", - }, - [WHOOP_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection full-history backfill; long-running, and boot discovery re-enqueues until the connection is marked backfilled.", - }, - [FITBIT_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection full-history backfill; boot-discovery driven and self-converging.", - }, - [GOOGLE_HEALTH_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection full-history backfill; boot-discovery driven and self-converging.", - }, - [STRAVA_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection full-history backfill; boot-discovery driven and self-converging.", - }, - [OURA_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection full-history backfill; boot-discovery driven and self-converging.", - }, - [GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection one-shot sleep re-read; discovery drops the connection once it is marked repaired.", - }, - [FITBIT_SLEEP_REPAIR_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection one-shot sleep re-read; discovery drops the connection once it is marked repaired.", - }, - [SLEEP_TIMELINE_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-connection one-shot timeline backfill, keyed per provider+user; discovery drops the connection once its rows carry the new shape.", - }, - [LAB_BIOMARKER_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user one-shot catalog link-up; discovery drops the user once no unlinked lab readings remain.", - }, - [INTEGRATION_BACKFILL_ADMISSION_QUEUE]: { - policy: "exclusive", - reason: - "Shared full-history drain; keyed per source job so boot rediscovery cannot duplicate queued, active, or retrying work.", - }, -}; - -/** - * Register every integration-sync queue: create, schedule, and bind handlers. - * Returns the queue names created (for the boot-level aggregate assertion). - */ -export async function registerIntegrationSyncQueues( - boss: PgBoss, -): Promise { - await createAndSchedule(boss, allQueues, schedules, queuePolicies); - // pg-boss scopes groupConcurrency to one queue name, so every provider - // reaches this shared durable queue before doing full-history work. The - // common group is enforced in PostgreSQL across worker processes/nodes; - // the heavy lane keeps an import off the heap while a backup, a restore - // or an upload is on it. - - await createAndWork( - boss, - INTEGRATION_BACKFILL_ADMISSION_QUEUE, - { - localConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, - groupConcurrency: INTEGRATION_BACKFILL_GLOBAL_CONCURRENCY, - }, - heavyLane( - INTEGRATION_BACKFILL_ADMISSION_QUEUE, - drainIntegrationBackfillAdmission, - ), - ); - - await createAndWork( - boss, - WITHINGS_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWithingsFallbackSync, - ); - await createAndWork( - boss, - WITHINGS_ACTIVITY_QUEUE, - { localConcurrency: 1 }, - handleWithingsActivitySync, - ); - await createAndWork( - boss, - WITHINGS_SLEEP_QUEUE, - { localConcurrency: 1 }, - handleWithingsSleepSync, - ); - await createAndWork( - boss, - WITHINGS_ECG_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWithingsEcgSync, - ); - await createAndWork( - boss, - WITHINGS_OAUTH_STATE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleWithingsOAuthStateCleanup, - ); - // v1.11.0 — WHOOP per-resource sync handlers. Webhook-driven per-user + - // cron full-iteration. Serial concurrency so a backfill-heavy tick never - // crowds the request pool and stays inside WHOOP's 100 req/min app cap. - await createAndWork( - boss, - WHOOP_RECOVERY_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWhoopRecoverySync, - ); - await createAndWork( - boss, - WHOOP_SLEEP_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWhoopSleepSync, - ); - await createAndWork( - boss, - WHOOP_WORKOUT_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWhoopWorkoutSync, - ); - await createAndWork( - boss, - WHOOP_CYCLE_SYNC_QUEUE, - { localConcurrency: 1 }, - handleWhoopCycleSync, - ); - // v1.11.0 — self-converging WHOOP backfill. This source queue retains the - // per-connection singleton and forwards admitted work to the shared drain. - await createAndWork( - boss, - WHOOP_BACKFILL_QUEUE, - { localConcurrency: WHOOP_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "whoop-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - await createAndWork( - boss, - WHOOP_OAUTH_STATE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleWhoopOAuthStateCleanup, - ); - // v1.30.x — daily sweep for the native OIDC handoff ledger. - await createAndWork( - boss, - OIDC_NATIVE_HANDOFF_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleOidcNativeHandoffCleanup, - ); - // v1.12.0 — Fitbit poll-sync (cron full-iteration; no webhook). Serial - // concurrency so a backfill-heavy tick never crowds the request pool. - await createAndWork( - boss, - FITBIT_SYNC_QUEUE, - { localConcurrency: 1 }, - handleFitbitSync, - ); - // v1.12.0 — self-converging Fitbit backfill. This source queue retains the - // per-connection singleton and forwards admitted work to the shared drain. - await createAndWork( - boss, - FITBIT_BACKFILL_QUEUE, - { localConcurrency: FITBIT_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "fitbit-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // One-shot Fitbit sleep duplicate repair. This source queue retains the - // per-connection singleton and forwards the watermark-safe bounded sleep - // re-read to the shared drain. - await createAndWork( - boss, - FITBIT_SLEEP_REPAIR_QUEUE, - { localConcurrency: FITBIT_SLEEP_REPAIR_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "fitbit-sleep-repair", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - await createAndWork( - boss, - FITBIT_OAUTH_STATE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleFitbitOAuthStateCleanup, - ); - // v1.26.0 — Google Health poll-sync (cron full-iteration; no webhook). Serial - // concurrency so a backfill-heavy tick never crowds the request pool. - await createAndWork( - boss, - GOOGLE_HEALTH_SYNC_QUEUE, - { localConcurrency: 1 }, - handleGoogleHealthSync, - ); - // v1.26.0 — self-converging Google Health backfill. This source queue - // retains the per-connection singleton and forwards admitted work to the - // shared drain. - await createAndWork( - boss, - GOOGLE_HEALTH_BACKFILL_QUEUE, - { localConcurrency: GOOGLE_HEALTH_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "google-health-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // v1.28.x — one-shot Google Health sleep duplicate repair. This source - // queue retains the per-connection singleton and forwards the watermark-safe - // full sleep-history re-read to the shared drain. - await createAndWork( - boss, - GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE, - { localConcurrency: GOOGLE_HEALTH_SLEEP_REPAIR_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "google-health-sleep-repair", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - await createAndWork( - boss, - GOOGLE_HEALTH_OAUTH_STATE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleGoogleHealthOAuthStateCleanup, - ); - // v1.17.1 — one-shot sleep-timeline backfill. This source queue retains the - // per-(user, provider) singleton and forwards admitted work to the shared - // drain. - await createAndWork( - boss, - SLEEP_TIMELINE_BACKFILL_QUEUE, - { localConcurrency: SLEEP_TIMELINE_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "sleep-timeline-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // v1.18.1 — one-shot lab-biomarker backfill. This source queue retains the - // per-user singleton and forwards admitted work to the shared drain. - await createAndWork( - boss, - LAB_BIOMARKER_BACKFILL_QUEUE, - { localConcurrency: LAB_BIOMARKER_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "lab-biomarker-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // v1.17.0 — Nightscout CGM poll-cohort sync. The hourly cron tick (no - // `userId`) walks every configured instance; one user's unreachable host is - // warned, not fatal. - await createAndWork( - boss, - NIGHTSCOUT_SYNC_QUEUE, - { localConcurrency: 1 }, - handleNightscoutSync, - ); - // v1.17.0 — Polar + Oura OAuth poll-cohort sync. The hourly cron tick - // (no `userId`) re-walks every connected user; one user's revoked grant is - // warned, not fatal. - await createAndWork( - boss, - POLAR_SYNC_QUEUE, - { localConcurrency: 1 }, - handlePolarSync, - ); - await createAndWork( - boss, - OURA_SYNC_QUEUE, - { localConcurrency: 1 }, - handleOuraSync, - ); - // v1.28.x — Strava OAuth poll-cohort sync (poll-only; webhook deferred). The - // hourly cron tick (no `userId`) re-walks every connected user; one user's - // revoked grant is warned, not fatal. - await createAndWork( - boss, - STRAVA_SYNC_QUEUE, - { localConcurrency: 1 }, - handleStravaSync, - ); - // v1.42.4 — integration watchdog. Single-flight per process; both queues - // also share one advisory lock, so the daily and hourly passes never run - // side by side on any replica. - await createAndWork( - boss, - INTEGRATION_WATCHDOG_HOURLY_QUEUE, - { localConcurrency: 1 }, - handleIntegrationWatchdogHourly, - ); - await createAndWork( - boss, - INTEGRATION_WATCHDOG_DAILY_QUEUE, - { localConcurrency: 1 }, - handleIntegrationWatchdogDaily, - ); - // v1.28.x — self-converging Strava backfill. This source queue retains the - // per-connection singleton and forwards admitted work to the shared drain. - await createAndWork( - boss, - STRAVA_BACKFILL_QUEUE, - { localConcurrency: STRAVA_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "strava-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // v1.44 — Oura history backfill. The source queue keeps the per-account - // singleton and forwards admitted work to the shared drain, where it runs - // in resumable slices. - await createAndWork( - boss, - OURA_BACKFILL_QUEUE, - { localConcurrency: OURA_BACKFILL_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await enqueueIntegrationBackfillAdmission(boss, { - kind: "oura-backfill", - data: job.data, - }); - } - return jobDone({ jobs: jobs.length }); - }, - ); - return allQueues; -} - -/** - * Fire-and-forget boot discovery for the self-converging integration backfills. - * Each pass is idempotent across reboots and never fails worker boot on a miss - * (errors come back through the helper's result value). - */ -export async function enqueueIntegrationSyncBootDiscovery(): Promise { - // v1.11.0 — WHOOP backfill. Finds every WHOOP connection not yet backfilled - // and enqueues one full-history sync per account. A completed backfill stamps - // `backfillCompletedAt`, dropping the connection from the discovery set. - try { - const { enqueued, skipped, error } = await enqueueBootTimeWhoopBackfill( - bootStaggerSecondsFor("whoop-backfill"), - ); - if (error) { - workerLog("error", `[whoop-backfill] boot discovery failed: ${error}`); - } else { - workerLog( - "info", - `[whoop-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[whoop-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.12.0 — Fitbit backfill. Finds every Fitbit connection not yet - // backfilled and enqueues one full-history sync per account. - try { - const { enqueued, skipped, error } = await enqueueBootTimeFitbitBackfill( - bootStaggerSecondsFor("fitbit-backfill"), - ); - if (error) { - workerLog("error", `[fitbit-backfill] boot discovery failed: ${error}`); - } else { - workerLog( - "info", - `[fitbit-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[fitbit-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.26.0 — Google Health backfill. Finds every Google Health connection not - // yet backfilled and enqueues one full-history sync per account. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeGoogleHealthBackfill( - bootStaggerSecondsFor("google-health-backfill"), - ); - if (error) { - workerLog( - "error", - `[google-health-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[google-health-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[google-health-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.28.x — one-shot Google Health sleep duplicate repair. Finds every - // connection not yet repaired and enqueues one full sleep re-read per - // account; a completed pass stamps `sleepRepairedAt`. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeGoogleHealthSleepRepair( - bootStaggerSecondsFor("google-health-sleep-repair"), - ); - if (error) { - workerLog( - "error", - `[google-health-sleep-repair] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[google-health-sleep-repair] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[google-health-sleep-repair] boot discovery threw an unexpected error", - err, - ); - } - - // v1.17.1 — one-shot sleep-timeline backfill. Finds every WHOOP + Withings - // connection whose sleep rows predate the stamp/shape fix and enqueues one - // job per (user, provider). A completed pass stamps `sleepTimelineBackfillAt`. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeSleepTimelineBackfill( - bootStaggerSecondsFor("sleep-timeline-backfill"), - ); - if (error) { - workerLog( - "error", - `[sleep-timeline-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[sleep-timeline-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[sleep-timeline-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.28.x — Strava backfill. Finds every Strava connection not yet - // backfilled and enqueues one full-history sync per account. - try { - const { enqueued, skipped, error } = await enqueueBootTimeStravaBackfill( - bootStaggerSecondsFor("strava-backfill"), - ); - if (error) { - workerLog("error", `[strava-backfill] boot discovery failed: ${error}`); - } else { - workerLog( - "info", - `[strava-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[strava-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.44 — Oura history backfill. Finds every Oura connection not yet - // backfilled (and every account still holding the retired - // equivalent-distance rows) and enqueues one sliced backfill per account. - try { - const { enqueued, skipped, error } = await enqueueBootTimeOuraBackfill( - bootStaggerSecondsFor("oura-backfill"), - ); - if (error) { - workerLog("error", `[oura-backfill] boot discovery failed: ${error}`); - } else { - workerLog( - "info", - `[oura-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[oura-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.18.1 — one-shot lab-biomarker backfill. Finds every user holding an - // un-linked live lab reading and enqueues one job each. A completed pass - // links every reading, dropping the user from the discovery set. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeLabBiomarkerBackfill( - bootStaggerSecondsFor("lab-biomarker-backfill"), - ); - if (error) { - workerLog( - "error", - `[lab-biomarker-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[lab-biomarker-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[lab-biomarker-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // One-shot Fitbit sleep duplicate repair. Finds every connection not yet - // repaired and enqueues one bounded sleep re-read per account; a completed - // pass stamps `sleepRepairedAt`. - try { - const { enqueued, skipped, error } = await enqueueBootTimeFitbitSleepRepair( - bootStaggerSecondsFor("fitbit-sleep-repair"), - ); - if (error) { - workerLog( - "error", - `[fitbit-sleep-repair] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[fitbit-sleep-repair] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[fitbit-sleep-repair] boot discovery threw an unexpected error", - err, - ); - } -} diff --git a/src/lib/jobs/reminder/register-maintenance.ts b/src/lib/jobs/reminder/register-maintenance.ts deleted file mode 100644 index b0e02c042..000000000 --- a/src/lib/jobs/reminder/register-maintenance.ts +++ /dev/null @@ -1,2197 +0,0 @@ -/** - * Maintenance / ops / cleanup queue registrar. - * - * Owns the housekeeping tier: data backup (in-DB weekly + off-host nightly), - * the restore drill, every ledger cleanup (rate-limit, idempotency, audit-log, - * mood-reminder, push-attempt, measurement-tombstone), medication inventory - * expiry, intake auto-skip, the duplicate dose-slot dedup (boot + daily-cron), - * host-metric sampling, the rec-feedback aggregator, the geo backfill, the TLS - * pin monitor, PR detection, and the Apple Health export ingest. - * - * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its - * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The - * `measurement-tombstone-cleanup`, `withings-oauth-state-cleanup`, - * `intake-slot-dedup-queue`, `tls-pin-monitor-queue`, and `geo-backfill` - * guards read THIS module. - */ -import { trackBackgroundTask } from "@/lib/logging/background-tasks"; -import { PgBoss } from "pg-boss"; -import { GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON } from "@/lib/jobs/geo-backfill"; -import { - GEOLITE2_FETCH_QUEUE, - GEOLITE2_FETCH_CRON, - enqueueGeolite2FetchBootDiscovery, - type Geolite2FetchPayload, -} from "@/lib/jobs/geolite2-fetch"; -import { - TLS_PIN_MONITOR_QUEUE, - TLS_PIN_MONITOR_CRON, -} from "@/lib/jobs/tls-pin-monitor"; -import { - PR_DETECTION_QUEUE, - PR_DETECTION_CONCURRENCY, - PR_DETECTION_FALLBACK_CRON, - PR_DETECTION_EXPIRE_SECONDS, - type PrDetectionPayload, -} from "@/lib/jobs/pr-detection"; -import { jobBudget } from "@/lib/jobs/job-budget"; -import { lockedPass, WHOLE_PASS } from "@/lib/jobs/long-pass"; -import { heavyLane } from "@/lib/jobs/heavy-lane"; -import { - OFFHOST_BACKUP_QUEUE, - OFFHOST_BACKUP_SEND_OPTIONS, -} from "@/lib/jobs/offhost-backup"; -import { - MEDICATION_INVENTORY_EXPIRE_QUEUE, - MEDICATION_INVENTORY_EXPIRE_CRON, - type MedicationInventoryExpirePayload, -} from "@/lib/jobs/medication-inventory-expire"; -import { - INTAKE_AUTO_SKIP_QUEUE, - INTAKE_AUTO_SKIP_CRON, - type IntakeAutoSkipPayload, -} from "@/lib/jobs/intake-auto-skip"; -import { - INTAKE_AUTO_TAKE_QUEUE, - INTAKE_AUTO_TAKE_CRON, - type IntakeAutoTakePayload, -} from "@/lib/jobs/intake-auto-take"; -import { - APPLE_HEALTH_IMPORT_V2_QUEUE, - APPLE_HEALTH_IMPORT_LEGACY_QUEUE, - APPLE_HEALTH_IMPORT_CONCURRENCY, - IMPORT_JOB_RECONCILE_QUEUE, - IMPORT_JOB_RECONCILE_CRON, - handleAppleHealthImport, - handleImportJobReconcileTick, - migrateLegacyAppleHealthImport, - type AppleHealthImportPayload, -} from "@/lib/jobs/apple-health-import-worker"; -import { - COMPACTION_TOMBSTONE_PURGE_QUEUE, - enqueueBootTimeCompactionTombstonePurge, - handleCompactionTombstonePurge, - type CompactionTombstonePurgePayload, -} from "@/lib/jobs/compaction-tombstone-purge"; -import { - MEASUREMENT_FOLD_REPAIR_QUEUE, - enqueueBootTimeMeasurementFoldRepair, - handleMeasurementFoldRepair, - type MeasurementFoldRepairPayload, -} from "@/lib/jobs/measurement-fold-repair"; -import { - MEASUREMENT_MAINTENANCE_CRON, - MEASUREMENT_MAINTENANCE_QUEUE, - MEASUREMENT_MAINTENANCE_SEND_OPTIONS, - handleMeasurementMaintenance, - type MeasurementMaintenancePayload, -} from "@/lib/jobs/measurement-maintenance"; -import { - PULSE_DAY_MEAN_QUEUE, - handlePulseDayMean, - type PulseDayMeanPayload, -} from "@/lib/jobs/pulse-day-mean"; -import { - HEALTH_CONNECT_IMPORT_QUEUE, - HEALTH_CONNECT_IMPORT_CONCURRENCY, - handleHealthConnectImport, - type HealthConnectImportPayload, -} from "@/lib/jobs/health-connect-import-worker"; -import { - MEDICATION_INTAKE_IMPORT_QUEUE, - MEDICATION_INTAKE_IMPORT_CONCURRENCY, - handleMedicationIntakeImport, - type MedicationIntakeImportQueuePayload, -} from "@/lib/jobs/medication-intake-import"; -import { - INTAKE_SLOT_DEDUP_QUEUE, - INTAKE_SLOT_DEDUP_CONCURRENCY, - dedupeUserIntakeSlots, - enqueueBootTimeIntakeSlotDedup, - type IntakeSlotDedupPayload, -} from "@/lib/medications/intake-slot-dedup"; -import { - handleOffhostPurge, - OFFHOST_PURGE_CRON, - OFFHOST_PURGE_QUEUE, -} from "@/lib/jobs/offhost-purge"; -import { - handleRestoreDrill, - RESTORE_DRILL_CRON, - RESTORE_DRILL_QUEUE, - RESTORE_DRILL_SEND_OPTIONS, -} from "@/lib/jobs/restore-drill"; -import { - BACKUP_RESTORE_QUEUE, - enqueueBackupRestoreSweepAtBoot, - handleBackupRestore, - type BackupRestorePayload, -} from "@/lib/jobs/backup-restore"; -import { - NOTE_ENCRYPTION_BACKFILL_QUEUE, - NOTE_ENCRYPTION_BACKFILL_CONCURRENCY, - runNoteEncryptionBackfillForUser, - enqueueBootTimeNoteEncryptionBackfill, - type NoteEncryptionBackfillPayload, -} from "@/lib/jobs/note-encryption-backfill"; -import { - MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, - MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY, - runMedNotesEncryptionBackfillForUser, - enqueueBootTimeMedNotesEncryptionBackfill, - type MedNotesEncryptionBackfillPayload, -} from "@/lib/jobs/med-notes-encryption-backfill"; -import { - FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, - FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY, - runFreeTextEncryptionBackfillForUser, - enqueueBootTimeFreeTextEncryptionBackfill, - type FreeTextEncryptionBackfillPayload, -} from "@/lib/jobs/free-text-encryption-backfill"; -import { - CONTENT_INDEX_BACKFILL_QUEUE, - CONTENT_INDEX_BACKFILL_CONCURRENCY, - runContentIndexBackfillForUser, - type ContentIndexBackfillPayload, -} from "@/lib/jobs/document-content-index-backfill"; -import { - DOCUMENT_INDEX_QUEUE, - DOCUMENT_INDEX_CONCURRENCY, - runDocumentIndex, - type DocumentIndexPayload, -} from "@/lib/jobs/document-index"; -import { - DOCUMENT_AI_RUN_QUEUE, - DOCUMENT_AI_RUN_CONCURRENCY, - handleDocumentAiRunJobs, - type DocumentAiRunPayload, -} from "@/lib/jobs/document-ai-run"; -import { - DOCUMENT_AI_RUN_REAPER_QUEUE, - DOCUMENT_AI_RUN_REAPER_CRON, - handleDocumentAiRunReaper, - type DocumentAiRunReaperPayload, -} from "@/lib/jobs/document-ai-run-reaper"; -import { - DOCUMENT_THUMBNAIL_QUEUE, - DOCUMENT_THUMBNAIL_CONCURRENCY, - runDocumentThumbnail, - type DocumentThumbnailPayload, -} from "@/lib/jobs/document-thumbnail"; -import { - DOCUMENT_SUMMARY_QUEUE, - DOCUMENT_SUMMARY_CONCURRENCY, - runDocumentSummaryJob, - type DocumentSummaryPayload, -} from "@/lib/jobs/document-summary"; -import { - DOCUMENT_SUMMARY_CATCHUP_QUEUE, - DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY, - runSummaryCatchUpForUser, - type SummaryCatchUpPayload, -} from "@/lib/jobs/document-summary-catchup"; -import { - DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, - DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY, - runThumbnailBackfillForUser, - enqueueBootTimeThumbnailBackfill, - type ThumbnailBackfillPayload, -} from "@/lib/jobs/document-thumbnail-backfill"; -import { - ENCRYPTION_KEY_ROTATE_QUEUE, - ENCRYPTION_KEY_ROTATE_CONCURRENCY, - handleEncryptionKeyRotate, - type EncryptionKeyRotatePayload, -} from "@/lib/jobs/encryption-key-rotate"; -import { - ENVIRONMENT_FETCH_QUEUE, - ENVIRONMENT_FETCH_CRON, - ENVIRONMENT_FETCH_CONCURRENCY, - handleEnvironmentFetch, - type EnvironmentFetchPayload, -} from "@/lib/jobs/environment-fetch"; -import { - DAWARICH_SYNC_CRON, - DAWARICH_SYNC_QUEUE, - handleDawarichSync, - type DawarichSyncPayload, -} from "@/lib/jobs/dawarich-sync"; -import { - ENVIRONMENT_AQ_HISTORY_CRON, - ENVIRONMENT_AQ_HISTORY_QUEUE, - enqueueBootTimeAirQualityHistory, - handleEnvironmentAqHistory, - type EnvironmentAqHistoryPayload, -} from "@/lib/jobs/environment-air-quality-history"; -import { recordError } from "@/lib/jobs/worker-status"; -import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; -import { - DATA_BACKUP_QUEUE, - DATA_BACKUP_SEND_OPTIONS, -} from "@/lib/jobs/data-backup-policy"; -import { workerLog, BOOT_BACKFILL_STAGGER_SECONDS } from "./shared"; -import { - createAndSchedule, - createAndWork, - cronIsTheRetry, - type QueuePolicyTable, - type ScheduleEntry, -} from "./registrar-shared"; -import { - DataBackupPayload, - OffhostBackupPayload, - handleOffhostBackup, - handleDataBackup, -} from "./backup-handlers"; -import { - RateLimitCleanupPayload, - StepUpElevationCleanupPayload, - IdempotencyCleanupPayload, - AuditLogCleanupPayload, - MoodReminderCleanupPayload, - handleMoodReminderCleanup, - PushAttemptCleanupPayload, - handlePushAttemptCleanup, - ArrivalReactionCleanupPayload, - handleArrivalReactionCleanup, - WorkoutInsightClaimCleanupPayload, - handleWorkoutInsightClaimCleanup, - MeasurementTombstoneCleanupPayload, - handleMeasurementTombstoneCleanup, - handleRateLimitCleanup, - handleIdempotencyCleanup, - handleAuditLogCleanup, - handleStepUpElevationCleanup, - CoachMessageCleanupPayload, - handleCoachMessageCleanup, - McpTokenCleanupPayload, - handleMcpTokenCleanup, -} from "./cleanup-handlers"; -import { - HostMetricSamplePayload, - FeedbackAggregatorPayload, - GeoBackfillPayload, - TlsPinMonitorPayload, - handleHostMetricSample, - handleFeedbackAggregator, - handleGeoBackfill, - handleGeolite2Fetch, - handleTlsPinMonitor, - handlePrDetection, -} from "./ops-handlers"; -import { - handleMedicationInventoryExpire, - handleIntakeAutoSkip, - handleIntakeAutoTake, -} from "./medication-maintenance"; -import { - DOCUMENT_PURGE_QUEUE, - DOCUMENT_PURGE_CRON, - handleDocumentPurge, - type DocumentPurgePayload, -} from "@/lib/jobs/document-purge"; -import { - SOFT_DELETE_PURGE_QUEUE, - SOFT_DELETE_PURGE_CRON, - handleSoftDeletePurge, - type SoftDeletePurgePayload, -} from "@/lib/jobs/soft-delete-purge"; -import { - AUTH_LEDGER_CLEANUP_QUEUE, - AUTH_LEDGER_CLEANUP_CRON, - handleAuthLedgerCleanup, - type AuthLedgerCleanupPayload, -} from "@/lib/jobs/auth-ledger-cleanup"; -import { - DOCUMENT_SUMMARY_REAPER_QUEUE, - DOCUMENT_SUMMARY_REAPER_CRON, - handleDocumentSummaryReaper, - type DocumentSummaryReaperPayload, -} from "@/lib/jobs/document-summary-reaper"; -import { - CYCLE_PREDICTION_REFRESH_QUEUE, - CYCLE_PREDICTION_REFRESH_CRON, - handleCyclePredictionRefresh, - type CyclePredictionRefreshPayload, -} from "@/lib/jobs/cycle-prediction-refresh"; -import { - MOOD_PROGNOSIS_REFRESH_QUEUE, - MOOD_PROGNOSIS_REFRESH_CRON, - handleMoodPrognosisRefresh, - type MoodPrognosisRefreshPayload, -} from "@/lib/jobs/mood-prognosis-refresh"; -import { - ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, - ACHIEVEMENT_UNLOCK_SWEEP_CRON, - handleAchievementUnlockSweep, - type AchievementUnlockSweepPayload, -} from "@/lib/jobs/achievement-unlock-sweep"; - -const DATA_BACKUP_CRON = "0 3 * * 0"; // weekly Sunday at 03:00 - -const RATE_LIMIT_CLEANUP_QUEUE = "rate-limit-cleanup"; - -const RATE_LIMIT_CLEANUP_CRON = "*/5 * * * *"; // every 5 minutes - -const IDEMPOTENCY_CLEANUP_QUEUE = "idempotency-cleanup"; - -const IDEMPOTENCY_CLEANUP_CRON = "0 3 * * *"; // daily at 03:00 (Europe/Berlin) - -const STEP_UP_ELEVATION_CLEANUP_QUEUE = "step-up-elevation-cleanup"; - -// Every 15 minutes. An elevation lives five, so nothing lingers long; the -// cadence only has to be short enough that the table stays small. -const STEP_UP_ELEVATION_CLEANUP_CRON = "*/15 * * * *"; - -const AUDIT_LOG_CLEANUP_QUEUE = "audit-log-cleanup"; - -const AUDIT_LOG_CLEANUP_CRON = "15 3 * * *"; // daily at 03:15 (Europe/Berlin) -// v1.15.19 — daily duplicate dose-slot dedup discovery tick. The boot-time -// pass only ran on worker restart, so a cross-source duplicate slot created -// between deploys (a pending REMINDER row plus a standalone API/WEB row on -// the same instant) survived until the next reboot. The cron payload omits -// `userId`; the handler treats that as the discovery tick and fans out one -// per-user job (singletonKey-coalesced) exactly like the boot pass. Slots at -// 03:28 between the mood-reminder cleanup (03:25) and the inventory expire -// (03:30), inside the maintenance window. - -const INTAKE_SLOT_DEDUP_CRON = "28 3 * * *"; - -// 02:30 Europe/Berlin — runs after audit-log/idempotency cleanups so old -// rows are gone before they're snapshotted, but before the existing -// in-DB DATA_BACKUP at 03:00 (Sundays only) so the off-host copy is -// always at-or-ahead of the local one. - -const OFFHOST_BACKUP_CRON = "30 2 * * *"; - -const HOST_METRIC_QUEUE = "host-metric-sample"; -// v1.42 — every five minutes. A per-minute sample filled pg-boss's job -// table with one row a minute for a chart nobody reads at that resolution. -// The admin chart's polling interval (`host-metrics-chart.tsx`) has to match. - -const HOST_METRIC_CRON = "*/5 * * * *"; -// v1.4.16 — daily rec-feedback aggregator. 04:00 Europe/Berlin -// runs the slot AFTER all the cleanup jobs (rate-limit, idempotency, -// audit-log) so the previous-day's noise is gone before we aggregate. - -const FEEDBACK_AGGREGATOR_QUEUE = "feedback-aggregator"; - -const FEEDBACK_AGGREGATOR_CRON = "0 4 * * *"; -// v0.5.4 ios-coord — daily mood-reminder dispatch-ledger retention sweep. -// Rows older than 90 days are behavioural footprints of mood-log gaps; we -// keep them long enough to debug a duplicate-push report (~one billing -// cycle) but no longer. Slots between the audit-log cleanup (03:15) and the -// drain (03:45). - -const MOOD_REMINDER_CLEANUP_QUEUE = "mood-reminder-cleanup"; - -const MOOD_REMINDER_CLEANUP_CRON = "25 3 * * *"; - -const PUSH_ATTEMPT_CLEANUP_QUEUE = "push-attempt-cleanup"; - -const PUSH_ATTEMPT_CLEANUP_CRON = "35 3 * * *"; - -// v1.31.0 — daily prune for the data-arrival spine's reaction markers. -// Slots at 04:10, after the 03:xx retention block and the 04:00 MCP-token -// prune, so the maintenance window stays in a readable order. -const ARRIVAL_REACTION_CLEANUP_QUEUE = "arrival-reaction-cleanup"; - -const ARRIVAL_REACTION_CLEANUP_CRON = "10 4 * * *"; - -const MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE = "measurement-tombstone-cleanup"; - -const MEASUREMENT_TOMBSTONE_CLEANUP_CRON = "40 3 * * *"; -// v1.18.7 — daily prune for the encrypted Coach conversation history. -// coach_messages is append-only and grows forever; rows older than the -// configurable retention window (default 365 days) are hard-deleted. -// Slots at 03:50 Europe/Berlin after the measurement-tombstone cleanup -// (03:40) and the cumulative drain (03:45), staying inside the existing -// 03:xx maintenance window. - -const COACH_MESSAGE_CLEANUP_QUEUE = "coach-message-cleanup"; - -const COACH_MESSAGE_CLEANUP_CRON = "50 3 * * *"; -// v1.40 — daily prune of the workout-insight claim ledger. Claims outlive a -// deleted workout so the delete cannot free a daily slot; rows older than a -// week are never read by the cap again. - -const WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE = "workout-insight-claim-cleanup"; - -const WORKOUT_INSIGHT_CLAIM_CLEANUP_CRON = "52 3 * * *"; -// v1.23 — converging backfill that migrates the free-text health-note columns -// (mood + measurement) from plaintext to AES-256-GCM at rest. Boot discovery -// enqueues one per-user job; a daily 03:55 Europe/Berlin discovery tick (empty -// payload) re-fans for any rows that landed between worker reboots, so the -// migration converges without a restart. Idempotent + fail-closed per row. - -const NOTE_ENCRYPTION_BACKFILL_CRON = "55 3 * * *"; -// v1.24 — daily prune of expired/revoked MCP connector access tokens (every -// code exchange + hourly refresh mints a 60-minute row) and long-revoked OAuth -// connection anchors. Slots at 04:00 Europe/Berlin after the note-encryption -// backfill discovery (03:55) so the two don't pile up on the same boss poll. - -const MCP_TOKEN_CLEANUP_QUEUE = "mcp-token-cleanup"; - -const MCP_TOKEN_CLEANUP_CRON = "0 4 * * *"; -// v1.25 — converging backfill that migrates the three medication free-text -// note columns (side-effect note, dose-change note, inventory-item note) from -// plaintext to AES-256-GCM at rest. Boot discovery enqueues one per-user job; -// a daily 04:05 Europe/Berlin discovery tick (empty payload) re-fans for any -// rows that landed between worker reboots. Idempotent + fail-closed per row. -// Slots after the MCP-token prune (04:00) so the two don't pile up on the same -// boss poll. - -const MED_NOTES_ENCRYPTION_BACKFILL_CRON = "5 4 * * *"; -// v1.39.3 — converging backfill that moves the Coach conversation title and the -// custom-metric reading note to AES-256-GCM at rest. Boot discovery (staggered -// past the startup storm) enqueues one per-user job; a daily 04:07 -// Europe/Berlin discovery tick re-fans for rows written by a previous-release -// process after the upgrade. Two minutes after the medication-note tick so the -// two do not share a poll. v1.39.4 adds the practitioner phone + address and -// the workout GPS track to the same pass. -const FREE_TEXT_ENCRYPTION_BACKFILL_CRON = "7 4 * * *"; - -const allQueues = [ - // v1.25 — nightly environmental-context fetch. A daily discovery tick - // fans out one per-user job per opted-in account with a home location; the - // queue also serves the on-demand backfill from the Environment settings - // surface. Without this entry pg-boss never provisions it and both the cron - // and the backfill button silently no-op. - ENVIRONMENT_FETCH_QUEUE, - // v1.42 — the air-quality history backfill: per account, from boot and - // nightly discovery, the home being set and air quality switched on. - ENVIRONMENT_AQ_HISTORY_QUEUE, - // v1.43 — the Dawarich movement context: nightly discovery, per-account - // runs, and the connect / settings backfills. - DAWARICH_SYNC_QUEUE, - DATA_BACKUP_QUEUE, - RATE_LIMIT_CLEANUP_QUEUE, - IDEMPOTENCY_CLEANUP_QUEUE, - AUDIT_LOG_CLEANUP_QUEUE, - STEP_UP_ELEVATION_CLEANUP_QUEUE, - OFFHOST_BACKUP_QUEUE, - RESTORE_DRILL_QUEUE, - // v1.39.3 — deleting an account or wiping its data takes its off-host - // copies out of the bucket. Without this entry the requests would be - // written and never worked. - OFFHOST_PURGE_QUEUE, - // v1.39.1 — an admin's restore of a stored backup, run off the request, and - // the boot sweep that re-queues one a stopped worker left running. Without - // this entry every restore request would queue a job nobody works. - BACKUP_RESTORE_QUEUE, - HOST_METRIC_QUEUE, - FEEDBACK_AGGREGATOR_QUEUE, - GEO_BACKFILL_QUEUE, - // v1.37.7 (issue #659) — runtime GeoLite2 fetch. A self-hoster who sets - // MAXMIND_LICENSE_KEY on the published image gets the offline geo tier - // without a rebuild or a manual mount: boot discovery fetches when no - // database is present, a monthly cron keeps a keyed host current. Without - // this entry pg-boss never provisions the queue and both silently no-op. - GEOLITE2_FETCH_QUEUE, - // v1.12.2 ios-coord — TLS leaf SPKI-change monitor. The iOS client - // pins the served leaf certificate; this queue probes it every 6 h and - // alarms when the served SPKI leaves the operator's known-good set. - TLS_PIN_MONITOR_QUEUE, - PR_DETECTION_QUEUE, - MEDICATION_INVENTORY_EXPIRE_QUEUE, - // v1.4.46 — hourly auto-skip pass for stale unmarked intakes. Without - // this entry the schedule silently no-ops and pending rows older than - // 24 h pile up unflipped. - INTAKE_AUTO_SKIP_QUEUE, - // v1.42.2 — records the doses of a medication with auto-mark-taken on. - // Without this entry the schedule silently no-ops and those doses stay - // open. - INTAKE_AUTO_TAKE_QUEUE, - APPLE_HEALTH_IMPORT_V2_QUEUE, - APPLE_HEALTH_IMPORT_LEGACY_QUEUE, - // v1.32.1 (issue #588) — periodic sweep for ImportJob rows orphaned by a - // worker crash/restart mid-run. Without this entry the 15-minute cron - // below silently no-ops and a stuck "unpacking"/"parsing"/"upserting" row - // that survives past the next worker boot is never revisited. - IMPORT_JOB_RECONCILE_QUEUE, - MEDICATION_INTAKE_IMPORT_QUEUE, - // v1.42 (#972) — Android Health Connect export import, the Apple Health - // import's sibling. Without this entry every upload would queue a job - // nobody works. - HEALTH_CONNECT_IMPORT_QUEUE, - // v1.42 — backlog purge of compaction tombstones, queued once at boot. - // Without this entry the boot enqueue silently no-ops. - COMPACTION_TOMBSTONE_PURGE_QUEUE, - // v1.42 — one-time repair of the means older releases folded from part of - // their window; the purge above waits for it. Without this entry the boot - // enqueue silently no-ops and the purge never runs. - MEASUREMENT_FOLD_REPAIR_QUEUE, - // v1.42 — VACUUM / REINDEX of `measurements`: the admin route enqueues the - // operator's run, and a nightly check vacuums when dead rows pile up. - // Without this entry both silently no-op. - MEASUREMENT_MAINTENANCE_QUEUE, - // v1.43 — the pulse day-mean dry run and rebuild, admin-triggered only. - PULSE_DAY_MEAN_QUEUE, - // v1.8.2 — one-time duplicate dose-slot cleanup. Boot discovery enqueues - // one job per user holding two live intake rows that snap to the same - // canonical slot (the pre-fix REMINDER-pending + API-taken pair). Also - // carries a daily discovery cron (03:28). Without this entry pg-boss never - // provisions it and both the boot enqueue and the cron silently no-op. - INTAKE_SLOT_DEDUP_QUEUE, - // v0.5.4 ios-coord — mood-reminder dispatch-ledger retention sweep. - MOOD_REMINDER_CLEANUP_QUEUE, - // v1.4.49 — push-attempt ledger cleanup. The daily schedule below would - // silently no-op without this entry. - PUSH_ATTEMPT_CLEANUP_QUEUE, - // v1.31.0 — data-arrival reaction-marker prune. Without this entry the - // daily schedule silently no-ops and the markers accumulate forever. - ARRIVAL_REACTION_CLEANUP_QUEUE, - // v1.35.3 — nightly refresh of the cycle forecast rows the reminder cron - // scans. Without this entry the schedule silently no-ops and the forecast - // is only as fresh as the last time somebody opened the calendar. - CYCLE_PREDICTION_REFRESH_QUEUE, - // v1.38 — nightly rebuild of the mood forecast rows. Without this entry the - // schedule silently no-ops: nothing else writes them, so the surface would - // answer "not enough data yet" forever on an account that has plenty. - MOOD_PROGNOSIS_REFRESH_QUEUE, - // v1.35.3 — nightly pin for newly unlocked achievements. Without this entry - // the schedule silently no-ops and no unlock date is ever made durable. - ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, - // v1.7.0 — soft-deleted measurement tombstone prune. Without this entry - // the daily schedule silently no-ops and pruned-past-retention tombstones - // pile up forever. - MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, - // v1.18.7 — Coach conversation-history retention prune. Without this entry - // the daily schedule silently no-ops and the encrypted coach_messages - // table grows unbounded. - COACH_MESSAGE_CLEANUP_QUEUE, - // v1.40 — workout-insight claim ledger retention prune. Without this entry - // the daily schedule silently no-ops and the claim table keeps every row - // whose workout was deleted. - WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE, - // v1.23 — free-text health-note encryption backfill. Boot discovery + a - // daily discovery cron enqueue one per-user job per account still holding a - // plaintext note; without this entry pg-boss never provisions the queue and - // both the boot enqueue and the cron silently no-op. - NOTE_ENCRYPTION_BACKFILL_QUEUE, - // v1.23 — admin-triggered encryption-key rotation. On-demand only (no - // cron): the admin panel enqueues a singleton run that re-encrypts the - // corpus to the active key. Without this entry pg-boss never provisions the - // queue and the admin trigger silently no-ops. - ENCRYPTION_KEY_ROTATE_QUEUE, - // v1.24 — expired/revoked MCP access-token + connection prune. Without this - // entry the daily schedule silently no-ops and the api_tokens table grows - // unbounded with dead 60-minute connector rows. - MCP_TOKEN_CLEANUP_QUEUE, - // v1.25 — medication free-text note encryption backfill. Boot discovery + a - // daily discovery cron enqueue one per-user job per account still holding a - // plaintext medication note; without this entry pg-boss never provisions the - // queue and both the boot enqueue and the cron silently no-op. - MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, - // v1.39.3 — Coach title + custom-metric note encryption backfill. Without - // this entry pg-boss never provisions the queue and both the boot enqueue - // and the cron silently no-op. - FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, - // Document vault — daily physical purge for tombstones past the 30-day - // undo grace (returns the encrypted blob's TOAST space). Without this entry - // the daily schedule silently no-ops and "deleted" documents hold backup - // weight forever. - DOCUMENT_PURGE_QUEUE, - // Nightly purge of expired tombstones for every soft-delete model the - // retention register hands to it (`soft-delete-retention.ts`). - SOFT_DELETE_PURGE_QUEUE, - // Nightly removal of expired sessions, tokens, challenges and trusted - // devices. - AUTH_LEDGER_CLEANUP_QUEUE, - // Document vault — hourly reaper for summary rows stuck on PENDING (a - // job dead past its retry budget). Without this entry the schedule - // silently no-ops and a dead "wird generiert" claim never converges to - // the honest UNAVAILABLE state. - DOCUMENT_SUMMARY_REAPER_QUEUE, - // Document vault P2 — on-demand content-search index backfill. Fired by the - // "index all documents" action (no cron): indexes a user's not-yet-indexed - // documents via one provider transcription each, consent + budget gated. - // Without this entry pg-boss never provisions the queue and the trigger - // silently no-ops. - CONTENT_INDEX_BACKFILL_QUEUE, - // Document AI — automatic per-document content indexing, enqueued on upload. - // Provider-first (vision) when configured + consented, else local text-layer - // extraction. Without this entry pg-boss never provisions the queue and every - // upload enqueue silently no-ops. - DOCUMENT_INDEX_QUEUE, - // v1.40 — background document AI runs ("Read with AI", lab scans): the - // request answers 202 and this queue does the read. Without this entry - // pg-boss never provisions the queue and every run waits until the reaper - // fails it as worker-unavailable. - DOCUMENT_AI_RUN_QUEUE, - // v1.40 — five-minute reaper for those runs: fails a run no worker took or - // one past its deadline, deletes a finished one after an hour. - DOCUMENT_AI_RUN_REAPER_QUEUE, - // Document vault — automatic per-document preview thumbnail, enqueued on - // upload. Pure local compute (canvas/pdfjs downscale), no egress. Without - // this entry pg-boss never provisions the queue and every upload enqueue - // silently no-ops. - DOCUMENT_THUMBNAIL_QUEUE, - // Document vault — boot-time preview-thumbnail backfill. Discovers accounts - // holding thumbnailable documents without a preview and fans out per-document - // thumbnail jobs. Without this entry pg-boss never provisions the queue and - // the boot discovery silently no-ops. - DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, - // Document AI — automatic per-document plain-language summary, enqueued on - // upload. Provider (vision) only, gated on the `documentsAutoAiRead` opt-in + - // egress consent + budget; persists the summary encrypted. Without this entry - // pg-boss never provisions the queue and every upload enqueue silently no-ops. - DOCUMENT_SUMMARY_QUEUE, - // v1.30.31 — auto-read catch-up. Scheduled on a genuine OFF→ON flip of the - // `documentsAutoAiRead` opt-in; fans out summary jobs for documents that were - // uploaded while the flag was still OFF. Without this entry pg-boss never - // provisions the queue and the catch-up enqueue silently no-ops. - DOCUMENT_SUMMARY_CATCHUP_QUEUE, -]; - -// `cronIsTheRetry` (retryLimit 0) marks the passes whose handler now returns -// `jobFailed` where it used to warn and carry on, AND whose failure repeats: -// a bulk DELETE or UPDATE that timed out will time out again on an immediate -// retry, so pg-boss's default two would run the same doomed statement three -// times a night. The queues without it are the ones whose failure is -// transient — the TLS probe, the off-host upload, the per-user environment -// fetch — and those still want the default retries. The restore drill sends -// `retryLimit: 0` in its own options for the other reason: it pages each -// failed account itself, and a retry would page the same accounts again. -const schedules: ScheduleEntry[] = [ - // v1.43.7 — nightly 03:40 check of `measurements`: a plain VACUUM when the - // dead rows pass the threshold, never a rebuild. Same singleton key as the - // operator's run, so the two collapse. - [ - MEASUREMENT_MAINTENANCE_QUEUE, - MEASUREMENT_MAINTENANCE_CRON, - { ...MEASUREMENT_MAINTENANCE_SEND_OPTIONS }, - ], - // v1.25 — daily 02:10 Europe/Berlin discovery tick (empty payload) - // that fans out one per-user environment fetch per opted-in account. - [ENVIRONMENT_FETCH_QUEUE, ENVIRONMENT_FETCH_CRON], - // v1.42 — daily 02:40 discovery for the air-quality history backfill, after - // the night's environment fetch has taken its share of the budget. - [ENVIRONMENT_AQ_HISTORY_QUEUE, ENVIRONMENT_AQ_HISTORY_CRON], - // v1.43 — daily 03:20 discovery for the Dawarich movement context. A - // failed run is an outage at the instance; the default retries apply. - [DAWARICH_SYNC_QUEUE, DAWARICH_SYNC_CRON], - // The send options are the point of this tuple: without them the weekly - // snapshot inherits pg-boss's 15-minute expiration and is killed mid-run on - // any sizeable record. See `data-backup-policy.ts` for why the window rides - // the schedule rather than the queue. - [DATA_BACKUP_QUEUE, DATA_BACKUP_CRON, DATA_BACKUP_SEND_OPTIONS], - [RATE_LIMIT_CLEANUP_QUEUE, RATE_LIMIT_CLEANUP_CRON, cronIsTheRetry], - [IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON, cronIsTheRetry], - [AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON], - [ - STEP_UP_ELEVATION_CLEANUP_QUEUE, - STEP_UP_ELEVATION_CLEANUP_CRON, - cronIsTheRetry, - ], - [OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON, OFFHOST_BACKUP_SEND_OPTIONS], - [RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON, RESTORE_DRILL_SEND_OPTIONS], - // Nightly backstop for the purge the deletion kicks straight away: a - // request the bucket refused, or one written while the queue was down. - [OFFHOST_PURGE_QUEUE, OFFHOST_PURGE_CRON, cronIsTheRetry], - [HOST_METRIC_QUEUE, HOST_METRIC_CRON, cronIsTheRetry], - [FEEDBACK_AGGREGATOR_QUEUE, FEEDBACK_AGGREGATOR_CRON, cronIsTheRetry], - // v1.4.37 — hourly geo backfill. The helper is idempotent + capped - // at 5 000 rows per pass; running it at :40 every hour catches the - // long tail of audit rows that landed with the offline MMDB - // missing or the online provider unreachable. - [GEO_BACKFILL_QUEUE, GEO_BACKFILL_CRON, cronIsTheRetry], - // v1.37.7 (issue #659) — monthly runtime GeoLite2 refresh (8th, 04:00 - // Europe/Berlin, past MaxMind's first-Tuesday reissue). Deliberately NOT - // cronIsTheRetry: a month between ticks is far too long to be the retry, so - // the queue keeps pg-boss's default retries for a transient MaxMind hiccup. - [GEOLITE2_FETCH_QUEUE, GEOLITE2_FETCH_CRON], - // v1.12.2 ios-coord — every-6-hour TLS leaf SPKI probe (:07 off the - // hourly sync crons). Surfaces a pinned-leaf rotation well inside the - // ≥11-day re-pin window the iOS release owner needs. - [TLS_PIN_MONITOR_QUEUE, TLS_PIN_MONITOR_CRON], - // Fallback rescan every 30 minutes — protects against ingest paths - // that ship measurements without enqueueing a per-user job. The - // cron payload deliberately omits a `userId` so the handler iterates - // every user; per-user push-suppression cannot apply on the cron - // path (the silent flag is set by the ingest hooks). - [ - PR_DETECTION_QUEUE, - PR_DETECTION_FALLBACK_CRON, - { expireInSeconds: PR_DETECTION_EXPIRE_SECONDS }, - ], - // v1.4.25 — daily expire-stale pass for the per-pen inventory - // entities. Flips IN_USE rows whose 30-day clock has blown to - // EXPIRED at 03:30 Europe/Berlin (in the existing 02:xx–03:xx - // maintenance window, right after idempotency-cleanup). - [ - MEDICATION_INVENTORY_EXPIRE_QUEUE, - MEDICATION_INVENTORY_EXPIRE_CRON, - cronIsTheRetry, - ], - // v1.4.46 — hourly auto-skip for medication intakes the user never - // marked. Cron `5 * * * *` slots off the top-of-the-hour - // reminder-check (:00) so the hourly ticks don't pile up on the same - // boss poll. - [INTAKE_AUTO_SKIP_QUEUE, INTAKE_AUTO_SKIP_CRON, cronIsTheRetry], - // v1.42.2 — every 15 minutes, off the reminder tick and the auto-miss - // pass, so a dose of an auto-mark-taken medication reads taken within - // minutes of its time. - [INTAKE_AUTO_TAKE_QUEUE, INTAKE_AUTO_TAKE_CRON, cronIsTheRetry], - // v0.5.4 ios-coord — daily mood-reminder dispatch-ledger retention sweep. - [MOOD_REMINDER_CLEANUP_QUEUE, MOOD_REMINDER_CLEANUP_CRON, cronIsTheRetry], - // v1.15.19 — daily 03:28 Europe/Berlin duplicate dose-slot dedup - // discovery. The empty cron payload (no `userId`) is the handler's - // signal to run the discovery fan-out instead of a per-user pass, so - // cross-source duplicate slots created between deploys collapse within - // a day instead of waiting for the next worker reboot. - [INTAKE_SLOT_DEDUP_QUEUE, INTAKE_SLOT_DEDUP_CRON], - // v1.4.49 — daily 03:35 Europe/Berlin prune for push_attempts. - [PUSH_ATTEMPT_CLEANUP_QUEUE, PUSH_ATTEMPT_CLEANUP_CRON, cronIsTheRetry], - // v1.31.0 — daily 04:10 Europe/Berlin prune for arrival_reactions. - [ - ARRIVAL_REACTION_CLEANUP_QUEUE, - ARRIVAL_REACTION_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.7.0 — daily 03:40 Europe/Berlin prune for expired measurement - // tombstones. - [MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, MEASUREMENT_TOMBSTONE_CLEANUP_CRON], - // v1.18.7 — daily 03:50 Europe/Berlin prune for stale Coach history. - [COACH_MESSAGE_CLEANUP_QUEUE, COACH_MESSAGE_CLEANUP_CRON, cronIsTheRetry], - // v1.40 — daily 03:52 Europe/Berlin prune for the workout-insight claims. - [ - WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE, - WORKOUT_INSIGHT_CLAIM_CLEANUP_CRON, - cronIsTheRetry, - ], - // v1.23 — daily 03:55 Europe/Berlin note-encryption backfill discovery. - // The empty cron payload (no `userId`) is the handler's signal to fan out - // one per-user job per account still holding a plaintext note. - [NOTE_ENCRYPTION_BACKFILL_QUEUE, NOTE_ENCRYPTION_BACKFILL_CRON], - // v1.24 — daily 04:00 Europe/Berlin prune for dead MCP connector tokens. - [MCP_TOKEN_CLEANUP_QUEUE, MCP_TOKEN_CLEANUP_CRON, cronIsTheRetry], - // v1.35.3 — daily 04:20 Europe/Berlin refresh of the cycle forecast rows - // the reminder cron scans. Tomorrow's tick is the retry: the forecast moves - // on a day scale and the reminder windows are days wide. - [ - CYCLE_PREDICTION_REFRESH_QUEUE, - CYCLE_PREDICTION_REFRESH_CRON, - cronIsTheRetry, - ], - // v1.38 — daily 04:35 Europe/Berlin rebuild of the mood forecast rows. - // Tomorrow's tick is the retry: the fit moves on a day scale and a night - // late writes the same rows from the same days. - [MOOD_PROGNOSIS_REFRESH_QUEUE, MOOD_PROGNOSIS_REFRESH_CRON, cronIsTheRetry], - // v1.35.3 — daily 04:25 Europe/Berlin pin for newly unlocked achievements. - // Tomorrow's tick is the retry: the date written is the computed completion - // date, so a night late writes the identical row. - [ - ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, - ACHIEVEMENT_UNLOCK_SWEEP_CRON, - cronIsTheRetry, - ], - // v1.25 — daily 04:05 Europe/Berlin medication-note encryption backfill - // discovery. The empty cron payload (no `userId`) is the handler's signal to - // fan out one per-user job per account still holding a plaintext medication - // note. - [MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, MED_NOTES_ENCRYPTION_BACKFILL_CRON], - // v1.39.3 — daily 04:07 Europe/Berlin discovery for the Coach title + - // custom-metric note backfill. Empty payload = fan out per user. - [FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, FREE_TEXT_ENCRYPTION_BACKFILL_CRON], - // Document vault — daily 04:10 Europe/Berlin purge for tombstoned - // documents past the 30-day undo grace. - [DOCUMENT_PURGE_QUEUE, DOCUMENT_PURGE_CRON, cronIsTheRetry], - // Daily 03:42 Europe/Berlin, after the sync-domain tombstone cleanup. The - // next night is the retry: the purge picks up whatever a failed run left. - [SOFT_DELETE_PURGE_QUEUE, SOFT_DELETE_PURGE_CRON, cronIsTheRetry], - // Daily 03:35 Europe/Berlin. Same: the next night deletes what is left. - [AUTH_LEDGER_CLEANUP_QUEUE, AUTH_LEDGER_CLEANUP_CRON, cronIsTheRetry], - // Document vault — hourly heal of summary rows stuck on PENDING. The next - // tick is the retry: the updateMany is idempotent and the TTL predicate - // re-selects anything a failed tick left behind. - [DOCUMENT_SUMMARY_REAPER_QUEUE, DOCUMENT_SUMMARY_REAPER_CRON, cronIsTheRetry], - // v1.40 — five-minute pass over background document AI runs. The next tick - // is the retry: every write is conditional on the state it leaves. - [DOCUMENT_AI_RUN_REAPER_QUEUE, DOCUMENT_AI_RUN_REAPER_CRON, cronIsTheRetry], - // v1.32.1 (issue #588) — every-15-minute orphan-ImportJob sweep. Re-runs - // the same reconcile the boot path uses, so a stuck "unpacking" row - // whose worker crashed/restarted without the boot-time pass catching it - // (heartbeat not yet stale, pg-boss job not yet expired) still converges - // to a visible `failed` state within two ticks instead of staying stuck - // forever on an otherwise-healthy worker. - [IMPORT_JOB_RECONCILE_QUEUE, IMPORT_JOB_RECONCILE_CRON, cronIsTheRetry], -]; - -/** - * De-duplication policy per maintenance queue. - * - * Every retention / cleanup cron here (rate-limit, idempotency, audit-log, - * push-attempt, tombstone, coach-message, MCP-token, document-purge, …) is a - * keyless tick with no `singletonKey` on any send, so a policy would only - * constrain the empty key. Those are deliberately absent from this table. - * - * Two deliberate omissions worth naming, because both look like candidates: - * - * - ENVIRONMENT_FETCH_QUEUE is LEFT ALONE. It is genuinely ambiguous: the - * lookback refresh sends a per-user key, but an explicit-range backfill - * sends with NO options at all, deliberately, so that "it always runs". - * Any policy would collapse every keyless explicit-range backfill onto the - * shared empty key, so two different requested date ranges would silently - * become one. That is exactly the class of silent work-dropping a policy is - * supposed to prevent, so the queue keeps `standard`. The settings - * backfill (v1.42) is held per account by a keyed time slot - * (`singletonSeconds`), which pg-boss enforces under any policy; the - * travel-period refresh stays keyless so two periods never merge. - * - APPLE_HEALTH_IMPORT_V2_QUEUE, APPLE_HEALTH_IMPORT_LEGACY_QUEUE, - * DATA_BACKUP_QUEUE and PR_DETECTION_QUEUE send keylessly by design (each - * import / backup / detection run is a distinct unit of work). A policy - * would coalesce independent runs. - */ -const queuePolicies: QueuePolicyTable = { - // Per-user, boot- and cron-discovery driven, self-converging one-shots. The - // discovery predicate re-offers the job while the work is outstanding, so - // suppressing a duplicate of a queued OR active pass cannot lose work. - [INTAKE_SLOT_DEDUP_QUEUE]: { - policy: "exclusive", - reason: - "Per-user duplicate-slot cleanup; discovery drops the user once no colliding intake pair remains.", - }, - [NOTE_ENCRYPTION_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user note-encryption backfill; discovery drops the user once no plaintext note remains.", - }, - [MED_NOTES_ENCRYPTION_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user medication-note encryption backfill; discovery drops the user once no plaintext note remains.", - }, - [FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user encryption backfill (Coach title, custom-metric note, practitioner phone + address, workout GPS track); discovery drops the user once no readable value remains.", - }, - [DOCUMENT_THUMBNAIL_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user thumbnail backfill; discovery drops the user once every thumbnailable document has a preview.", - }, - [CONTENT_INDEX_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user index-all run, triggered from the documents surface. Idempotent (it indexes only not-yet-indexed documents), so a second press while one runs is correctly a no-op.", - }, - [ENCRYPTION_KEY_ROTATE_QUEUE]: { - policy: "exclusive", - reason: - "Fixed singleton key, admin-triggered. Two concurrent corpus rotations must never overlap; the route already reports a suppressed send back as alreadyQueued.", - }, - // v1.42 — VACUUM / REINDEX of `measurements`, operator or nightly. - [MEASUREMENT_MAINTENANCE_QUEUE]: { - policy: "exclusive", - reason: - "Fixed singleton key on the admin send and the nightly tick alike. Two maintenance passes would vacuum or rebuild side by side; the route reports a suppressed send back as enqueued: false.", - }, - // v1.43 — the pulse day-mean dry run and rebuild. - [PULSE_DAY_MEAN_QUEUE]: { - policy: "exclusive", - reason: - "Fixed singleton key, admin-triggered. A dry run beside a rebuild would compare against rollups being rewritten; the route answers 409 busy instead.", - }, - // v1.42 — compaction-tombstone backlog purge. `short`, not `exclusive`: - // a run that stops early sends its own follow-up while it is still active, - // and `exclusive` would swallow that send. A boot's send while a run is - // queued collapses into it; the delete predicate is idempotent, so a boot - // that lands beside an active run costs a duplicate scan, never a wrong row. - [COMPACTION_TOMBSTONE_PURGE_QUEUE]: { - policy: "short", - reason: - "Fixed singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", - }, - // v1.42 — per-account fold repair. `short` for the same reason: a run the - // budget stopped sends its own follow-up for the account while it is still - // active. A boot's send for a queued account collapses into it; a duplicate - // beside an active run rewrites nothing, since only differing means are - // written. - // v1.42 — the air-quality history backfill. `short` for the same reason: a - // run with work left sends its own follow-up while it is still active. A - // send for an account whose run is queued collapses into it; one beside an - // active run only re-reads what is done, since done days are never fetched. - [ENVIRONMENT_AQ_HISTORY_QUEUE]: { - policy: "short", - reason: - "Per-account singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", - }, - [MEASUREMENT_FOLD_REPAIR_QUEUE]: { - policy: "short", - reason: - "Per-account singleton key; collapse queued duplicates only, so the active run can queue its own follow-up.", - }, - - // Per-document, enqueued on upload. `short`, NOT `exclusive`: each handler - // re-reads the document when it starts, so collapsing sends that arrive while - // an identical job is still queued is safe. `exclusive` would additionally - // suppress a re-process requested after the current job had already read the - // old bytes, leaving a stale summary/thumbnail/index behind with no discovery - // pass to re-offer it — these queues have no cron to re-converge. - [DOCUMENT_INDEX_QUEUE]: { - policy: "short", - reason: - "Per-document, no re-converging discovery pass. Collapse queued duplicates only, so a re-index after a content change is never dropped.", - }, - [DOCUMENT_THUMBNAIL_QUEUE]: { - policy: "short", - reason: - "Per-document, no re-converging discovery pass. Collapse queued duplicates only.", - }, - [DOCUMENT_SUMMARY_QUEUE]: { - policy: "short", - reason: - "Per-document, no re-converging discovery pass. Collapse queued duplicates only.", - }, - [DOCUMENT_SUMMARY_CATCHUP_QUEUE]: { - policy: "short", - reason: - "Per-user, keyed on the opt-in flip. Collapse a double toggle into one pass; the pass re-reads its candidate set when it starts.", - }, -}; - -/** - * Register every maintenance / ops / cleanup queue. Returns the queue names - * created (for the boot-level aggregate assertion). - */ -export async function registerMaintenanceQueues( - boss: PgBoss, -): Promise { - await createAndSchedule(boss, allQueues, schedules, queuePolicies); - - await createAndWork( - boss, - DATA_BACKUP_QUEUE, - { localConcurrency: 1 }, - lockedPass( - DATA_BACKUP_QUEUE, - () => WHOLE_PASS, - heavyLane(DATA_BACKUP_QUEUE, handleDataBackup), - ), - ); - await createAndWork( - boss, - RATE_LIMIT_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleRateLimitCleanup, - ); - await createAndWork( - boss, - IDEMPOTENCY_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleIdempotencyCleanup, - ); - await createAndWork( - boss, - AUDIT_LOG_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleAuditLogCleanup, - ); - await createAndWork( - boss, - STEP_UP_ELEVATION_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleStepUpElevationCleanup, - ); - // With metadata: the handler resumes a retry from the job's creation time. - await createAndWork( - boss, - OFFHOST_BACKUP_QUEUE, - { localConcurrency: 1, includeMetadata: true }, - lockedPass( - OFFHOST_BACKUP_QUEUE, - () => WHOLE_PASS, - heavyLane(OFFHOST_BACKUP_QUEUE, handleOffhostBackup), - ), - ); - // Locked: a drill that outlives a restart or a manual run never reads the - // bucket beside another one. - await createAndWork( - boss, - RESTORE_DRILL_QUEUE, - { localConcurrency: 1 }, - lockedPass( - RESTORE_DRILL_QUEUE, - () => WHOLE_PASS, - heavyLane(RESTORE_DRILL_QUEUE, handleRestoreDrill), - ), - ); - await createAndWork( - boss, - OFFHOST_PURGE_QUEUE, - { localConcurrency: 1 }, - handleOffhostPurge, - ); - // One restore at a time per worker: each holds a transaction over a whole - // account and a batch of readings in memory, and two at once on a small host - // is how a restore that fits alone would run out of heap. - await createAndWork( - boss, - BACKUP_RESTORE_QUEUE, - { localConcurrency: 1 }, - heavyLane(BACKUP_RESTORE_QUEUE, handleBackupRestore), - ); - // A restore the previous process was running is picked up again once its - // heartbeat has gone stale; see `sweepInterruptedRestores`. - await enqueueBackupRestoreSweepAtBoot(boss); - await createAndWork( - boss, - HOST_METRIC_QUEUE, - { localConcurrency: 1 }, - handleHostMetricSample, - ); - await createAndWork( - boss, - FEEDBACK_AGGREGATOR_QUEUE, - { localConcurrency: 1 }, - handleFeedbackAggregator, - ); - await createAndWork( - boss, - GEO_BACKFILL_QUEUE, - { localConcurrency: 1 }, - handleGeoBackfill, - ); - // v1.37.7 (issue #659) — runtime GeoLite2 fetch. Single-flight: one download - // at a time, and two ticks racing the same atomic place is wasted bandwidth. - await createAndWork( - boss, - GEOLITE2_FETCH_QUEUE, - { localConcurrency: 1 }, - handleGeolite2Fetch, - ); - // v1.12.2 ios-coord — TLS leaf SPKI-change monitor. Single-flight: one - // short outbound TLS handshake per tick, no benefit to overlapping ticks. - await createAndWork( - boss, - TLS_PIN_MONITOR_QUEUE, - { localConcurrency: 1 }, - handleTlsPinMonitor, - ); - await createAndWork( - boss, - MOOD_REMINDER_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleMoodReminderCleanup, - ); - // v1.4.49 — daily prune of the push-attempt ledger. Single-flight - // matches every other cleanup queue; two ticks racing on the same - // DELETE statement is wasted work and the second tick's payload - // would be a no-op anyway. - await createAndWork( - boss, - PUSH_ATTEMPT_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handlePushAttemptCleanup, - ); - // v1.31.0 — daily prune of the data-arrival reaction markers. Single-flight - // like every other cleanup queue: two ticks racing the same DELETE is - // wasted work. - await createAndWork( - boss, - ARRIVAL_REACTION_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleArrivalReactionCleanup, - ); - // v1.35.3 — nightly cycle-forecast refresh. Single-flight: two ticks - // recomputing the same cohort is wasted work and the second upsert is a - // no-op against the debounce. - await createAndWork( - boss, - CYCLE_PREDICTION_REFRESH_QUEUE, - { localConcurrency: 1 }, - handleCyclePredictionRefresh, - ); - // v1.38 — nightly mood-forecast rebuild. Single-flight: two ticks would fit - // the same windows twice and the second pass would write the identical rows. - await createAndWork( - boss, - MOOD_PROGNOSIS_REFRESH_QUEUE, - { localConcurrency: 1 }, - handleMoodPrognosisRefresh, - ); - // v1.35.3 — nightly achievement-unlock pin. Single-flight: the underlying - // createMany is idempotent on the (userId, achievementId) unique, so a - // second concurrent tick would only re-walk the same accounts. - await createAndWork( - boss, - ACHIEVEMENT_UNLOCK_SWEEP_QUEUE, - { localConcurrency: 1 }, - handleAchievementUnlockSweep, - ); - // v1.7.0 — daily prune of expired measurement tombstones. Single-flight - // like every other cleanup queue. - await createAndWork( - boss, - MEASUREMENT_TOMBSTONE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleMeasurementTombstoneCleanup, - ); - // v1.18.7 — daily prune of the encrypted Coach conversation history. - // Single-flight like every other cleanup queue; two ticks racing on the - // same DELETE is wasted work and the second is a no-op. - await createAndWork( - boss, - COACH_MESSAGE_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleCoachMessageCleanup, - ); - // v1.40 — daily prune of the workout-insight claim ledger. Single-flight - // like every other cleanup queue. - await createAndWork( - boss, - WORKOUT_INSIGHT_CLAIM_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleWorkoutInsightClaimCleanup, - ); - // MCP Phase 3 (M2) — daily prune of dead MCP connector access tokens + - // long-revoked connection anchors. Single-flight like every other cleanup. - await createAndWork( - boss, - MCP_TOKEN_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleMcpTokenCleanup, - ); - // Document vault — daily purge of tombstoned documents past the 30-day - // undo grace. Single-flight like every other cleanup queue; the underlying - // deleteMany is idempotent so a duplicate tick is a no-op. - await createAndWork( - boss, - DOCUMENT_PURGE_QUEUE, - { localConcurrency: 1 }, - handleDocumentPurge, - ); - // Single-flight like every other cleanup queue; both deletes are bounded - // and idempotent, so a duplicate tick finds nothing. - await createAndWork( - boss, - SOFT_DELETE_PURGE_QUEUE, - { localConcurrency: 1 }, - handleSoftDeletePurge, - ); - await createAndWork( - boss, - AUTH_LEDGER_CLEANUP_QUEUE, - { localConcurrency: 1 }, - handleAuthLedgerCleanup, - ); - // v1.40 — background document AI runs. Each job is one run, under the lock - // `lockedPass` takes per run id, so a redelivery never reads beside the - // first delivery; the claim inside is conditional as well. - await createAndWork( - boss, - DOCUMENT_AI_RUN_QUEUE, - { localConcurrency: DOCUMENT_AI_RUN_CONCURRENCY }, - lockedPass( - DOCUMENT_AI_RUN_QUEUE, - (job) => `run:${job.data.runId ?? ""}`, - handleDocumentAiRunJobs, - ), - ); - // v1.40 — the five-minute run reaper. Single-flight; every write in it is - // conditional, so a duplicate tick is a no-op. - await createAndWork( - boss, - DOCUMENT_AI_RUN_REAPER_QUEUE, - { localConcurrency: 1 }, - handleDocumentAiRunReaper, - ); - // Document vault — hourly stale-PENDING summary reaper. Single-flight; - // the underlying updateMany is idempotent so a duplicate tick is a no-op. - await createAndWork( - boss, - DOCUMENT_SUMMARY_REAPER_QUEUE, - { localConcurrency: 1 }, - handleDocumentSummaryReaper, - ); - await createAndWork( - boss, - PR_DETECTION_QUEUE, - { localConcurrency: PR_DETECTION_CONCURRENCY }, - // An ingest job is one account; the cron pass is every account, one run - // at a time. - lockedPass( - PR_DETECTION_QUEUE, - (job) => { - const userId = (job.data as PrDetectionPayload | undefined)?.userId; - return userId ? `user:${userId}` : WHOLE_PASS; - }, - handlePrDetection, - ), - ); - await createAndWork( - boss, - MEDICATION_INVENTORY_EXPIRE_QUEUE, - { localConcurrency: 1 }, - handleMedicationInventoryExpire, - ); - // v1.4.46 — hourly auto-skip for stale unmarked intakes. - // Single-flight: two ticks racing against the same row pile is wasted - // work, and the underlying `updateMany` is the canonical idempotent - // shape anyway. - await createAndWork( - boss, - INTAKE_AUTO_SKIP_QUEUE, - { localConcurrency: 1 }, - handleIntakeAutoSkip, - ); - // v1.42.2 — auto-mark-taken pass. Single-flight: two passes racing over - // the same slots would only collide on the conditional convert. - await createAndWork( - boss, - INTAKE_AUTO_TAKE_QUEUE, - { localConcurrency: 1 }, - handleIntakeAutoTake, - ); - // v1.4.34 — Apple Health export.zip ingest worker. localConcurrency - // caps at 1 because the parse loop is CPU-bound and a concurrent - // second 1 GB import would race the first for RSS. - await createAndWork( - boss, - APPLE_HEALTH_IMPORT_V2_QUEUE, - { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, - heavyLane(APPLE_HEALTH_IMPORT_V2_QUEUE, async (jobs) => { - // pg-boss v12 work callbacks always receive an array (batched - // worker mode); for our concurrency-1 + batchSize-1 case we just - // process each job sequentially. - // - // Each import is independent work, so a failed one does not abort the - // rest of the batch; the first failure is what the job reports. - let failure: JobOutcome | null = null; - for (const job of jobs) { - const outcome = await handleAppleHealthImport(job); - if (!outcome.ok && failure === null) failure = outcome; - } - return failure ?? jobDone({ jobs: jobs.length }); - }), - ); - // Drain pre-revision-2 backlog by moving each claimed legacy job onto the - // isolated v2 queue. This preserves the ImportJob status id without letting - // the current parser execute under a revision-1 mirror. - await createAndWork( - boss, - APPLE_HEALTH_IMPORT_LEGACY_QUEUE, - { localConcurrency: APPLE_HEALTH_IMPORT_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - await migrateLegacyAppleHealthImport(job); - } - return jobDone({ jobs: jobs.length }); - }, - ); - // v1.42 (#972) — Health Connect import. One at a time per worker, like the - // Apple Health import: each run reads a whole export and consumes its - // staged upload. - await createAndWork( - boss, - HEALTH_CONNECT_IMPORT_QUEUE, - { localConcurrency: HEALTH_CONNECT_IMPORT_CONCURRENCY }, - heavyLane(HEALTH_CONNECT_IMPORT_QUEUE, handleHealthConnectImport), - ); - // v1.42 — compaction-tombstone backlog purge. Serial: each batch holds - // an account against a concurrent restore. - await createAndWork( - boss, - COMPACTION_TOMBSTONE_PURGE_QUEUE, - { localConcurrency: 1 }, - handleCompactionTombstonePurge, - ); - // v1.42 — fold repair. Serial: each day holds the account against a - // concurrent restore, and the purge waits for it. - await createAndWork( - boss, - MEASUREMENT_FOLD_REPAIR_QUEUE, - { localConcurrency: 1 }, - handleMeasurementFoldRepair, - ); - // v1.42 — measurement table maintenance, operator-triggered only. - await createAndWork( - boss, - MEASUREMENT_MAINTENANCE_QUEUE, - { localConcurrency: 1 }, - handleMeasurementMaintenance, - ); - // v1.43 — pulse day-mean dry run and rebuild, operator-triggered only. - await createAndWork( - boss, - PULSE_DAY_MEAN_QUEUE, - { localConcurrency: 1 }, - handlePulseDayMean, - ); - // v1.32.1 (issue #588) — periodic orphan-ImportJob sweep. Single-flight: - // the underlying `updateMany` is idempotent and two ticks racing the same - // read-then-write pass would just be wasted work. - await createAndWork( - boss, - IMPORT_JOB_RECONCILE_QUEUE, - { localConcurrency: 1 }, - handleImportJobReconcileTick, - ); - await createAndWork( - boss, - MEDICATION_INTAKE_IMPORT_QUEUE, - { - localConcurrency: MEDICATION_INTAKE_IMPORT_CONCURRENCY, - includeMetadata: true, - }, - async (jobs) => { - // Independent per-delivery imports: run them all, report the first - // failure rather than losing it behind a later success. - let failure: JobOutcome | null = null; - for (const job of jobs) { - const outcome = await handleMedicationIntakeImport(job); - if (!outcome.ok && failure === null) failure = outcome; - } - return failure ?? jobDone({ jobs: jobs.length }); - }, - ); - - // v1.8.2 — duplicate dose-slot cleanup worker. The boot enqueue helper - // below sends one job per user holding two live intake rows that snap - // to the same canonical slot; this handler keeps the winner, soft- - // deletes the losers, normalises the winner's scheduledFor, and - // recomputes the affected compliance rollups. Serial concurrency so the - // pass never crowds the request pool. - // - // v1.15.19 — the queue also carries a daily cron tick (03:28, scheduled - // above) whose payload omits `userId`. That tick runs the SAME discovery - // fan-out the boot pass uses, so duplicate slots created between deploys - // collapse within a day instead of waiting for the next worker reboot. - await createAndWork>( - boss, - INTAKE_SLOT_DEDUP_QUEUE, - { localConcurrency: INTAKE_SLOT_DEDUP_CONCURRENCY }, - async (jobs) => { - let users = 0; - let discoveryEnqueued = 0; - let discoverySkipped = 0; - // The discovery helper returns its error instead of throwing, so a - // failed fan-out would otherwise leave no trace on the job at all. - let discoveryFailed = 0; - let slotsCollapsed = 0; - let rowsSoftDeleted = 0; - let rowsNormalised = 0; - let daysRecomputed = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - // Daily discovery tick — fan out one per-user job per account - // still holding duplicate-slot candidates (singletonKey-coalesced, - // identical to the boot pass). - const result = await enqueueBootTimeIntakeSlotDedup(); - discoveryEnqueued += result.enqueued; - discoverySkipped += result.skipped; - if (result.error) discoveryFailed++; - workerLog( - "info", - `[intake-slot-dedup] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, - ); - continue; - } - try { - const summary = await dedupeUserIntakeSlots(userId); - users++; - slotsCollapsed += summary.slotsCollapsed; - rowsSoftDeleted += summary.rowsSoftDeleted; - rowsNormalised += summary.rowsNormalised; - daysRecomputed += summary.daysRecomputed; - workerLog( - "info", - `[intake-slot-dedup] user=${userId} slotsCollapsed=${summary.slotsCollapsed} rowsSoftDeleted=${summary.rowsSoftDeleted} rowsNormalised=${summary.rowsNormalised} daysRecomputed=${summary.daysRecomputed}`, - ); - } catch (err) { - recordError(); - workerLog("error", `[intake-slot-dedup] user=${userId} failed`, err); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - discovery_enqueued: discoveryEnqueued, - discovery_skipped: discoverySkipped, - discovery_failed: discoveryFailed, - slots_collapsed: slotsCollapsed, - rows_soft_deleted: rowsSoftDeleted, - rows_normalised: rowsNormalised, - days_recomputed: daysRecomputed, - }); - }, - ); - - // v1.23 — note-encryption backfill worker. The boot enqueue helper (and the - // daily 03:55 discovery tick) send one job per user still holding a plaintext - // note; this handler migrates that user's rows to the encrypted columns. An - // empty-userId payload is the daily discovery tick and re-fans the per-user - // jobs (singletonKey-coalesced, identical to the boot pass). Serial - // concurrency so the migration never crowds the request pool. - await createAndWork>( - boss, - NOTE_ENCRYPTION_BACKFILL_QUEUE, - { localConcurrency: NOTE_ENCRYPTION_BACKFILL_CONCURRENCY }, - async (jobs) => { - let users = 0; - let discoveryEnqueued = 0; - let discoverySkipped = 0; - // The discovery helper returns its error instead of throwing, so a - // failed fan-out would otherwise leave no trace on the job at all. - let discoveryFailed = 0; - let measurements = 0; - let moodEntries = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - const result = await enqueueBootTimeNoteEncryptionBackfill(); - discoveryEnqueued += result.enqueued; - discoverySkipped += result.skipped; - if (result.error) discoveryFailed++; - workerLog( - "info", - `[note-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, - ); - continue; - } - try { - const { measurementsMigrated, moodEntriesMigrated } = - await runNoteEncryptionBackfillForUser(userId); - users++; - measurements += measurementsMigrated; - moodEntries += moodEntriesMigrated; - workerLog( - "info", - `[note-encryption-backfill] user=${userId} measurements=${measurementsMigrated} moodEntries=${moodEntriesMigrated}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[note-encryption-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - discovery_enqueued: discoveryEnqueued, - discovery_skipped: discoverySkipped, - discovery_failed: discoveryFailed, - measurements_migrated: measurements, - mood_entries_migrated: moodEntries, - }); - }, - ); - - // v1.23 — admin-triggered encryption-key rotation (on-demand, singleton). - await createAndWork( - boss, - ENCRYPTION_KEY_ROTATE_QUEUE, - { localConcurrency: ENCRYPTION_KEY_ROTATE_CONCURRENCY }, - lockedPass( - ENCRYPTION_KEY_ROTATE_QUEUE, - () => WHOLE_PASS, - handleEncryptionKeyRotate, - ), - ); - - // v1.25 — medication-note encryption backfill worker. The boot enqueue - // helper (and the daily 04:05 discovery tick) send one job per user still - // holding a plaintext medication note; this handler migrates that user's - // rows across all three tables to the encrypted columns. An empty-userId - // payload is the daily discovery tick and re-fans the per-user jobs - // (singletonKey-coalesced, identical to the boot pass). Serial concurrency - // so the migration never crowds the request pool. - await createAndWork>( - boss, - MED_NOTES_ENCRYPTION_BACKFILL_QUEUE, - { localConcurrency: MED_NOTES_ENCRYPTION_BACKFILL_CONCURRENCY }, - async (jobs) => { - let users = 0; - let discoveryEnqueued = 0; - let discoverySkipped = 0; - // The discovery helper returns its error instead of throwing, so a - // failed fan-out would otherwise leave no trace on the job at all. - let discoveryFailed = 0; - let sideEffects = 0; - let doseChanges = 0; - let inventoryItems = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - const result = await enqueueBootTimeMedNotesEncryptionBackfill(); - discoveryEnqueued += result.enqueued; - discoverySkipped += result.skipped; - if (result.error) discoveryFailed++; - workerLog( - "info", - `[med-notes-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, - ); - continue; - } - try { - const { - sideEffectsMigrated, - doseChangesMigrated, - inventoryItemsMigrated, - } = await runMedNotesEncryptionBackfillForUser(userId); - users++; - sideEffects += sideEffectsMigrated; - doseChanges += doseChangesMigrated; - inventoryItems += inventoryItemsMigrated; - workerLog( - "info", - `[med-notes-encryption-backfill] user=${userId} sideEffects=${sideEffectsMigrated} doseChanges=${doseChangesMigrated} inventoryItems=${inventoryItemsMigrated}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[med-notes-encryption-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - discovery_enqueued: discoveryEnqueued, - discovery_skipped: discoverySkipped, - discovery_failed: discoveryFailed, - side_effects_migrated: sideEffects, - dose_changes_migrated: doseChanges, - inventory_items_migrated: inventoryItems, - }); - }, - ); - - // v1.39.3 — Coach title + custom-metric note encryption backfill worker. - // Same shape as the medication-note worker above: an empty-userId payload is - // the daily discovery tick, a userId payload migrates that account's rows. - await createAndWork>( - boss, - FREE_TEXT_ENCRYPTION_BACKFILL_QUEUE, - { localConcurrency: FREE_TEXT_ENCRYPTION_BACKFILL_CONCURRENCY }, - async (jobs) => { - let users = 0; - let discoveryEnqueued = 0; - let discoverySkipped = 0; - let discoveryFailed = 0; - let titles = 0; - let notes = 0; - let contacts = 0; - let routes = 0; - let appointmentAddresses = 0; - let contactAudits = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - const result = await enqueueBootTimeFreeTextEncryptionBackfill(); - discoveryEnqueued += result.enqueued; - discoverySkipped += result.skipped; - if (result.error) discoveryFailed++; - workerLog( - "info", - `[free-text-encryption-backfill] daily discovery enqueued=${result.enqueued} skipped=${result.skipped}${result.error ? ` error=${result.error}` : ""}`, - ); - continue; - } - try { - const { - conversationTitlesMigrated, - metricNotesMigrated, - practitionerContactsMigrated, - routeGeometriesMigrated, - appointmentAddressesCleared, - contactAuditRowsScrubbed, - } = await runFreeTextEncryptionBackfillForUser(userId); - users++; - titles += conversationTitlesMigrated; - notes += metricNotesMigrated; - contacts += practitionerContactsMigrated; - routes += routeGeometriesMigrated; - appointmentAddresses += appointmentAddressesCleared; - contactAudits += contactAuditRowsScrubbed; - workerLog( - "info", - `[free-text-encryption-backfill] user=${userId} titles=${conversationTitlesMigrated} notes=${metricNotesMigrated} contacts=${practitionerContactsMigrated} routes=${routeGeometriesMigrated} appointment_addresses=${appointmentAddressesCleared} contact_audits=${contactAuditRowsScrubbed}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[free-text-encryption-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - discovery_enqueued: discoveryEnqueued, - discovery_skipped: discoverySkipped, - discovery_failed: discoveryFailed, - conversation_titles_migrated: titles, - metric_notes_migrated: notes, - practitioner_contacts_migrated: contacts, - route_geometries_migrated: routes, - appointment_addresses_cleared: appointmentAddresses, - contact_audit_rows_scrubbed: contactAudits, - }); - }, - ); - - // Document vault P2 — on-demand content-search index backfill worker. The - // trigger endpoint sends one per-user job; this handler indexes that user's - // not-yet-indexed documents (consent + budget gated, bounded + resumable). - // Serial concurrency so the provider calls never crowd the request pool. - await createAndWork( - boss, - CONTENT_INDEX_BACKFILL_QUEUE, - { localConcurrency: CONTENT_INDEX_BACKFILL_CONCURRENCY }, - lockedPass( - CONTENT_INDEX_BACKFILL_QUEUE, - (job) => `user:${job.data.userId ?? ""}`, - async (jobs) => { - let users = 0; - let stoppedEarly = false; - const shouldStop = jobBudget(jobs); - let skipped = 0; - let documentsIndexed = 0; - let documentsRetokenised = 0; - let documentsSkipped = 0; - let documentsFailed = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - skipped++; - continue; - } - try { - const { - indexed, - retokenised, - skipped: docsSkipped, - failed: docsFailed, - reason, - stoppedEarly: stopped, - } = await runContentIndexBackfillForUser(userId, shouldStop); - users++; - stoppedEarly ||= stopped === true; - documentsIndexed += indexed; - documentsRetokenised += retokenised; - documentsSkipped += docsSkipped; - documentsFailed += docsFailed; - workerLog( - "info", - `[document-content-index-backfill] user=${userId} indexed=${indexed} retokenised=${retokenised} skipped=${docsSkipped} failed=${docsFailed} reason=${reason}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[document-content-index-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - skipped, - documents_indexed: documentsIndexed, - documents_retokenised: documentsRetokenised, - documents_skipped: documentsSkipped, - documents_failed: documentsFailed, - stopped_early: stoppedEarly, - }); - }, - ), - ); - - // Document AI — automatic per-document content indexing. Enqueued on upload - // (one job per stored document); provider-first (vision) when configured + - // consented, else local text-layer extraction. Serial concurrency so the - // provider calls + PDF parse never crowd the request pool. - await createAndWork( - boss, - DOCUMENT_INDEX_QUEUE, - { localConcurrency: DOCUMENT_INDEX_CONCURRENCY }, - async (jobs) => { - let documents = 0; - let skipped = 0; - for (const job of jobs) { - const { userId, documentId } = job.data; - if (!userId || !documentId) { - skipped++; - continue; - } - try { - await runDocumentIndex(job.data); - documents++; - } catch (err) { - recordError(); - workerLog( - "error", - `[document-index] user=${userId} document=${documentId} failed`, - err, - ); - throw err; - } - } - return jobDone({ jobs: jobs.length, documents, skipped }); - }, - ); - - // Document vault — automatic per-document preview thumbnail. Enqueued on - // upload (one job per stored document); decodes + downscales the original to - // a small encrypted JPEG. Pure local compute; serial concurrency so the - // canvas/pdfjs decode never crowds the request pool. - await createAndWork( - boss, - DOCUMENT_THUMBNAIL_QUEUE, - { localConcurrency: DOCUMENT_THUMBNAIL_CONCURRENCY }, - async (jobs) => { - let documents = 0; - let skipped = 0; - for (const job of jobs) { - const { userId, documentId } = job.data; - if (!userId || !documentId) { - skipped++; - continue; - } - try { - await runDocumentThumbnail(job.data); - documents++; - } catch (err) { - recordError(); - workerLog( - "error", - `[document-thumbnail] user=${userId} document=${documentId} failed`, - err, - ); - throw err; - } - } - return jobDone({ jobs: jobs.length, documents, skipped }); - }, - ); - - // Document AI — automatic per-document plain-language summary. Enqueued on - // upload (one job per stored document); runs the vision provider ONLY when - // the `documentsAutoAiRead` opt-in is ON (egress consent + budget gated) and - // persists the summary encrypted. Serial concurrency so the provider call - // never crowds the request pool. - await createAndWork( - boss, - DOCUMENT_SUMMARY_QUEUE, - { localConcurrency: DOCUMENT_SUMMARY_CONCURRENCY }, - async (jobs) => { - let documents = 0; - let skipped = 0; - for (const job of jobs) { - const { userId, documentId } = job.data; - if (!userId || !documentId) { - skipped++; - continue; - } - try { - await runDocumentSummaryJob(job.data); - documents++; - } catch (err) { - recordError(); - workerLog( - "error", - `[document-summary] user=${userId} document=${documentId} failed`, - err, - ); - throw err; - } - } - return jobDone({ jobs: jobs.length, documents, skipped }); - }, - ); - - // Document AI — auto-read catch-up worker. A genuine OFF→ON flip of the - // `documentsAutoAiRead` opt-in sends one per-user job; this handler fans out - // per-document summary jobs for that user's already-stored, un-summarised - // documents (the upload-time enqueue no-opped while the flag was OFF). It - // only enqueues — the opt-in, egress consent and budget gates all still run - // per document inside the summary job. Serial concurrency. - await createAndWork( - boss, - DOCUMENT_SUMMARY_CATCHUP_QUEUE, - { localConcurrency: DOCUMENT_SUMMARY_CATCHUP_CONCURRENCY }, - async (jobs) => { - let users = 0; - let skipped = 0; - let summariesEnqueued = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - skipped++; - continue; - } - try { - const { enqueued } = await runSummaryCatchUpForUser(userId); - users++; - summariesEnqueued += enqueued; - } catch (err) { - recordError(); - workerLog( - "error", - `[document-summary-catchup] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - skipped, - summaries_enqueued: summariesEnqueued, - }); - }, - ); - - // Document vault — boot-time preview-thumbnail backfill worker. The boot - // discovery sends one per-user job; this handler fans out per-document - // thumbnail jobs for that user's not-yet-thumbnailed documents. Serial - // concurrency; cheap discovery + enqueue only. - await createAndWork( - boss, - DOCUMENT_THUMBNAIL_BACKFILL_QUEUE, - { localConcurrency: DOCUMENT_THUMBNAIL_BACKFILL_CONCURRENCY }, - async (jobs) => { - let users = 0; - let skipped = 0; - let thumbnailsEnqueued = 0; - for (const job of jobs) { - const { userId } = job.data; - if (!userId) { - skipped++; - continue; - } - try { - const { enqueued } = await runThumbnailBackfillForUser(userId); - users++; - thumbnailsEnqueued += enqueued; - workerLog( - "info", - `[document-thumbnail-backfill] user=${userId} enqueued=${enqueued}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[document-thumbnail-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - jobs: jobs.length, - users, - skipped, - thumbnails_enqueued: thumbnailsEnqueued, - }); - }, - ); - - // v1.42 — the air-quality history backfill. Serial: every run draws on the - // same instance-wide Open-Meteo budget as the nightly fetch. - await createAndWork( - boss, - ENVIRONMENT_AQ_HISTORY_QUEUE, - { localConcurrency: 1 }, - async (jobs) => { - let failure: JobOutcome | null = null; - for (const job of jobs) { - const outcome = await handleEnvironmentAqHistory(boss, job.data); - if (!outcome.ok && failure === null) failure = outcome; - } - return failure ?? jobDone({ jobs: jobs.length }); - }, - ); - - // v1.25 — nightly environment fetch. The daily discovery tick (empty - // payload) fans out one per-user job per opted-in account; the queue also - // serves the on-demand backfill payloads from the settings surface. Serial - // concurrency so the staggered outbound fetches never crowd the request pool. - await createAndWork( - boss, - ENVIRONMENT_FETCH_QUEUE, - { localConcurrency: ENVIRONMENT_FETCH_CONCURRENCY }, - async (jobs) => { - // A discovery tick and a per-user backfill share this queue and are - // independent of each other, so the pass runs to the end and reports - // the first failure it saw. - let failure: JobOutcome | null = null; - for (const job of jobs) { - const outcome = await handleEnvironmentFetch(boss, job.data); - if (!outcome.ok && failure === null) failure = outcome; - } - return failure ?? jobDone({ jobs: jobs.length }); - }, - ); - - // v1.43 — Dawarich movement context. Serial: one account at a time, so a - // slow instance never crowds the request pool. - await createAndWork( - boss, - DAWARICH_SYNC_QUEUE, - { localConcurrency: 1 }, - (jobs) => handleDawarichSync(boss, jobs), - ); - - return allQueues; -} - -/** - * Fire-and-forget boot discovery for the duplicate dose-slot cleanup. Finds - * every user holding two live intake rows within the drift window on the same - * medication and enqueues one dedup job per account. Idempotent across reboots: - * collapsed losers are soft-deleted so the `deleted_at IS NULL` discovery - * predicate drops them. Errors are returned through the helper's result value — - * the worker boot never fails because of a dedup miss. - */ -export async function enqueueMaintenanceBootDiscovery(): Promise { - try { - const { enqueued, skipped, error } = await enqueueBootTimeIntakeSlotDedup(); - if (error) { - workerLog("error", `[intake-slot-dedup] boot discovery failed: ${error}`); - } else { - workerLog( - "info", - `[intake-slot-dedup] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[intake-slot-dedup] boot discovery threw an unexpected error", - err, - ); - } - - // v1.23 — note-encryption backfill. Finds every user still holding a - // plaintext mood/measurement note (no ciphertext yet) and enqueues one - // migration job per account. Idempotent across reboots: a migrated row nulls - // its plaintext column and drops off the discovery predicate. - try { - // Stage 5 — last of the staggered boot backfills. The heavy full-history - // loads each `localConcurrency: 1` used to drain in parallel onto one heavy - // tenant at boot; threading an increasing `startAfter` delay spreads them - // across a window. The daily cron tick (above) passes no offset and stays - // immediate. - const { enqueued, skipped, error } = - await enqueueBootTimeNoteEncryptionBackfill( - BOOT_BACKFILL_STAGGER_SECONDS * 5, - ); - if (error) { - workerLog( - "error", - `[note-encryption-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[note-encryption-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[note-encryption-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.25 — medication free-text note encryption backfill. Finds every user - // still holding a plaintext side-effect / dose-change / inventory-item note - // (no ciphertext yet) and enqueues one migration job per account. Idempotent - // across reboots: a migrated row nulls its plaintext column and drops off the - // discovery predicate. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeMedNotesEncryptionBackfill(); - if (error) { - workerLog( - "error", - `[med-notes-encryption-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[med-notes-encryption-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[med-notes-encryption-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.39.3 — Coach title + custom-metric note encryption backfill. Staggered - // one step past the note-encryption backfill so it starts after the boot - // storm rather than on the first poll. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeFreeTextEncryptionBackfill( - BOOT_BACKFILL_STAGGER_SECONDS * 6, - ); - if (error) { - workerLog( - "error", - `[free-text-encryption-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[free-text-encryption-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[free-text-encryption-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // Document vault — preview-thumbnail backfill. Finds every account holding a - // thumbnailable document (image or PDF) without a preview and enqueues one - // per-user backfill pass. Idempotent across reboots: a rendered thumbnail - // drops its document off the discovery predicate. - try { - const { enqueued } = await enqueueBootTimeThumbnailBackfill(); - workerLog( - "info", - `[document-thumbnail-backfill] boot discovery: enqueued=${enqueued}`, - ); - } catch (err) { - workerLog( - "error", - "[document-thumbnail-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.42 — the air-quality history backfill after an update, staggered past - // the boot storm. Accounts whose history is through are not offered. - try { - const { enqueued, skipped, error } = await enqueueBootTimeAirQualityHistory( - BOOT_BACKFILL_STAGGER_SECONDS * 7, - ); - if (error) { - workerLog( - "error", - `[environment-aq-history] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[environment-aq-history] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[environment-aq-history] boot discovery threw an unexpected error", - err, - ); - } - - // v1.42 — the fold repair, once per account; the purge below waits for it. - try { - const { enqueued, skipped, error } = - await enqueueBootTimeMeasurementFoldRepair(); - if (error) { - workerLog( - "error", - `[measurement-fold-repair] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[measurement-fold-repair] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[measurement-fold-repair] boot discovery threw an unexpected error", - err, - ); - } - - // v1.42 — one compaction-tombstone purge run per boot. The run is - // self-limiting and idempotent, so a boot with no backlog costs one query. - // It sends nothing until the fold repair has finished an account. - try { - const { enqueued } = await enqueueBootTimeCompactionTombstonePurge(); - workerLog( - "info", - `[compaction-tombstone-purge] boot discovery: enqueued=${enqueued}`, - ); - } catch (err) { - workerLog( - "error", - "[compaction-tombstone-purge] boot discovery threw an unexpected error", - err, - ); - } - - // v1.37.7 (issue #659) — runtime GeoLite2 fetch. Only enqueues when a - // MAXMIND_LICENSE_KEY is set, egress is not disabled, and the offline tier - // is not already present — so a self-hoster who sets the key on the - // published image gets the databases fetched on the next worker boot without - // a rebuild or a manual mount. - try { - const { enqueued } = await enqueueGeolite2FetchBootDiscovery(); - workerLog("info", `[geolite2-fetch] boot discovery: enqueued=${enqueued}`); - } catch (err) { - workerLog( - "error", - "[geolite2-fetch] boot discovery threw an unexpected error", - err, - ); - } - - // v1.44 — one-time climb backfill for workouts stored with a track but no - // elevation (`climb-backfill.ts`). Detached: it reads one track at a time - // with a pause between batches and resumes from its cursor on the next - // boot, so it never holds the boot or the heap. A finished pass returns at - // once. - trackBackgroundTask( - (async () => { - const [{ prisma }, { runWorkoutClimbBackfill }] = await Promise.all([ - import("@/lib/db"), - import("@/lib/workouts/climb-backfill"), - ]); - const result = await runWorkoutClimbBackfill({ prisma }); - if (result.scanned > 0) { - workerLog( - "info", - `[workout-climb-backfill] scanned=${result.scanned} filled=${result.filled} done=${result.done}`, - ); - } - })().catch((err) => { - workerLog( - "error", - "[workout-climb-backfill] failed; the next boot resumes from the cursor", - err, - ); - }), - ); -} diff --git a/src/lib/jobs/reminder/register-reminders.ts b/src/lib/jobs/reminder/register-reminders.ts deleted file mode 100644 index 581a81358..000000000 --- a/src/lib/jobs/reminder/register-reminders.ts +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Reminder-dispatch queue registrar. - * - * Owns the user-facing reminder ticks: the medication overdue check, the daily - * mood reminder, the cycle reminder (period-soon + period-start-confirm), the - * Vorsorge (measurement) reminder, and the eventful reminder-satisfy queue. - * - * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its - * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The - * `cycle-reminder-queue` and `measurement-reminder-queue` guards read THIS - * module. - * - * Retry policy: every queue here keeps the pg-boss default. These ticks are - * fan-out passes whose per-user failures never fail the job, so a job-level - * failure means the pass itself could not run — a transient the default two - * retries are the right answer to. The satisfy queue carries its own - * `retryLimit` at send time (see `reminder-satisfy.ts`). - */ -import { PgBoss } from "pg-boss"; -import { - REMINDER_SATISFY_QUEUE, - REMINDER_SATISFY_CONCURRENCY, - type ReminderSatisfyPayload, -} from "@/lib/jobs/reminder-satisfy"; -import { - createAndSchedule, - createAndWork, - type ScheduleEntry, -} from "./registrar-shared"; -import { - ReminderCheckPayload, - handleReminderCheck, -} from "./medication-reminder-check"; -import { - MoodReminderPayload, - CycleReminderPayload, - MeasurementReminderPayload, - handleMoodReminderCheck, - handleCycleReminderCheck, - handleMeasurementReminderCheck, - handleReminderSatisfy, -} from "./mood-cycle-checks"; - -const QUEUE_NAME = "medication-reminder-check"; - -const CHECK_INTERVAL_CRON = "*/15 * * * *"; // every 15 minutes -// v0.5.4 ios-coord — daily mood-reminder cron. -// -// Runs every 15 minutes so the handler can pick up users whose local -// time has just crossed 22:00 across any IANA timezone without having -// to schedule one cron entry per zone. The handler short-circuits when -// the user's local hour isn't 22, so the 15-min cadence translates to -// ~4 ticks-per-hour × 1 actual-dispatch-window-per-user = at most one -// push per user per day. Idempotency is enforced by the -// `MoodReminderDispatch` ledger inside the handler. - -const MOOD_REMINDER_QUEUE = "mood-reminder-check"; - -const MOOD_REMINDER_CRON = "*/15 * * * *"; -// v1.15 — daily cycle reminder cron (period-soon + period-start-confirm). -// -// Runs every 15 minutes for the same reason as the mood reminder: the -// handler short-circuits unless the candidate user's local time is the -// cycle-reminder hour (09:00), so the 15-min cadence picks up every IANA -// timezone crossing that hour without one cron entry per zone. At most one -// push per event per user per local day — the `push_attempts` ledger is the -// idempotency anchor inside the handler. - -const CYCLE_REMINDER_QUEUE = "cycle-reminder-check"; - -const CYCLE_REMINDER_CRON = "*/15 * * * *"; - -// v1.17.1 — every-15-min tick for Vorsorge (measurement) reminders. -// Same cadence + short-circuit shape as the mood / cycle reminders: the -// handler only fires a reminder whose `nextDueAt` is past AND whose local -// time is the reminder's `notifyHour`, so the 15-min cadence picks up -// every IANA timezone crossing that hour without one cron entry per zone. -// Dedup is the reminder's own `nextDueAt` advance — no ledger table. - -const MEASUREMENT_REMINDER_QUEUE = "measurement-reminder-check"; - -const MEASUREMENT_REMINDER_CRON = "*/15 * * * *"; - -const allQueues = [ - QUEUE_NAME, - // v0.5.4 ios-coord — mood-reminder cron tick. Same pg-boss v12 - // createQueue contract as the drain queue; without this entry the - // every-15-min schedule silently no-ops and the dispatcher never fires. - MOOD_REMINDER_QUEUE, - // v1.15 — cycle-reminder cron tick (period-soon + period-start-confirm). - // Without this entry the every-15-min schedule silently no-ops and the - // cycle dispatcher never fires (the v1.4.37 dead-queue class). - CYCLE_REMINDER_QUEUE, - // v1.17.1 — Vorsorge (measurement) reminder cron tick. Without this entry - // pg-boss never provisions the queue and the every-15-min schedule - // silently no-ops (the v1.4.37 dead-queue class). - MEASUREMENT_REMINDER_QUEUE, - // v1.18.1 — eventful Vorsorge satisfaction. No cron of its own (the - // 15-min measurement-reminder check is the safety-net); enqueued by the - // ingest paths. Must still be registered here or the worker binding - // below never provisions the queue (v1.4.37 dead-queue lesson). - REMINDER_SATISFY_QUEUE, -]; - -const schedules: ScheduleEntry[] = [ - [QUEUE_NAME, CHECK_INTERVAL_CRON], - // v0.5.4 ios-coord — every-15-min tick for the daily mood reminder. - // The handler short-circuits unless the candidate user's local - // time is the 22:00 hour, so the cron costs ~one user-row scan - // per tick for the entire opted-in cohort. - [MOOD_REMINDER_QUEUE, MOOD_REMINDER_CRON], - // v1.15 — every-15-min tick for the daily cycle reminder. The handler - // short-circuits unless the candidate user's local time is the 09:00 - // hour, so the cron costs ~one prediction-row scan per tick for the - // opted-in cohort. - [CYCLE_REMINDER_QUEUE, CYCLE_REMINDER_CRON], - // v1.17.1 — every-15-min tick for the Vorsorge (measurement) reminder. - // The handler short-circuits unless a reminder is past-due AND the - // user's local time matches the reminder's notifyHour, so the cron - // costs ~one reminder-row scan per tick for the active cohort. - [MEASUREMENT_REMINDER_QUEUE, MEASUREMENT_REMINDER_CRON], -]; - -/** - * Register every reminder-dispatch queue. Returns the queue names created (for - * the boot-level aggregate assertion). - */ -export async function registerReminderQueues( - boss: PgBoss, -): Promise { - await createAndSchedule(boss, allQueues, schedules); - - await createAndWork( - boss, - QUEUE_NAME, - { localConcurrency: 1 }, - handleReminderCheck, - ); - // v0.5.4 ios-coord — single-flight worker. localConcurrency=1 keeps - // two reminder ticks from interleaving against the same user row; - // the dedup ledger would still save us, but skipping the race here - // avoids spurious P2002 errors in the wide-event log. - await createAndWork( - boss, - MOOD_REMINDER_QUEUE, - { localConcurrency: 1 }, - handleMoodReminderCheck, - ); - // v1.15 — single-flight cycle-reminder worker. localConcurrency=1 keeps - // two ticks from racing the fire-and-forget `push_attempts` ledger that - // anchors the per-day idempotency, exactly like the mood-reminder worker. - await createAndWork( - boss, - CYCLE_REMINDER_QUEUE, - { localConcurrency: 1 }, - handleCycleReminderCheck, - ); - // v1.17.1 — single-flight Vorsorge (measurement) reminder worker. - // localConcurrency=1 keeps two ticks from racing the `nextDueAt` - // advance that anchors the per-cycle idempotency, exactly like the - // mood / cycle reminder workers. - await createAndWork( - boss, - MEASUREMENT_REMINDER_QUEUE, - { localConcurrency: 1 }, - handleMeasurementReminderCheck, - ); - // v1.18.1 — eventful Vorsorge satisfaction. Resolves a user's reminders - // against their just-landed measurement / lab. Read-heavy on the user's - // own data; the same small concurrency budget as PR detection. - await createAndWork( - boss, - REMINDER_SATISFY_QUEUE, - { localConcurrency: REMINDER_SATISFY_CONCURRENCY }, - handleReminderSatisfy, - ); - - return allQueues; -} diff --git a/src/lib/jobs/reminder/register-rollup.ts b/src/lib/jobs/reminder/register-rollup.ts deleted file mode 100644 index f4ed00269..000000000 --- a/src/lib/jobs/reminder/register-rollup.ts +++ /dev/null @@ -1,1156 +0,0 @@ -/** - * Rollup / consolidation queue registrar. - * - * Owns the pre-aggregation + consolidation tier: measurement-rollup recompute - * + boot-backfill, mood-rollup recompute + boot-backfill, medication-compliance - * boot-backfill, legacy step consolidation, daily-mean consolidation, dense - * intra-day retention, and the nightly per-sample cumulative drain (which also - * folds the daily-mean + dense-retention passes onto the same tick). Includes - * the boot-time discovery enqueues for the self-converging backfills. - * - * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its - * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The - * `drain-cumulative-queue`, `mean-consolidation-queue`, and - * `stress-strain-retention-queue` (dense-retention facet) guards read THIS - * module. - */ -import { PgBoss } from "pg-boss"; -import { reportWorkerError } from "@/lib/jobs/report-worker-error"; -import { recordError } from "@/lib/jobs/worker-status"; -import { - ROLLUP_FULL_BACKFILL_QUEUE, - ROLLUP_FULL_BACKFILL_CONCURRENCY, - ROLLUP_RECOMPUTE_QUEUE, - ROLLUP_RECOMPUTE_CONCURRENCY, - enqueueBootTimeRollupBackfill, - recomputeUserRollups, - type RollupFullBackfillPayload, - type RollupRecomputePayload, -} from "@/lib/rollups/measurement-rollups"; -import { - MOOD_ROLLUP_FULL_BACKFILL_QUEUE, - MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY, - MOOD_ROLLUP_RECOMPUTE_QUEUE, - MOOD_ROLLUP_RECOMPUTE_CONCURRENCY, - enqueueBootTimeMoodRollupBackfill, - recomputeUserMoodRollups, - type MoodRollupFullBackfillPayload, - type MoodRollupRecomputePayload, -} from "@/lib/rollups/mood-rollups"; -import { - SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE, - SUPPLEMENT_NUTRIENT_RECOMPUTE_CONCURRENCY, - runSupplementNutrientRecompute, - type SupplementNutrientRecomputePayload, -} from "@/lib/medications/ingredients/recompute-job"; -import { - drainPerSampleCumulative, - DRAIN_CUMULATIVE_CUTOFF_HOURS, -} from "@/lib/measurements/drain-per-sample-cumulative"; -import { - consolidateDailyMean, - MEAN_CONSOLIDATION_CUTOFF_HOURS, -} from "@/lib/measurements/consolidate-daily-mean"; -import { - STEP_CONSOLIDATION_QUEUE, - STEP_CONSOLIDATION_CONCURRENCY, - runStepConsolidationForUser, - enqueueBootTimeStepConsolidation, - type StepConsolidationPayload, -} from "@/lib/jobs/step-consolidation"; -import { - STEP_CONSOLIDATION_REPAIR_QUEUE, - STEP_CONSOLIDATION_REPAIR_CONCURRENCY, - runStepConsolidationRepairForUser, - enqueueBootTimeStepConsolidationRepair, - type StepConsolidationRepairPayload, -} from "@/lib/jobs/step-consolidation-repair"; -import { - CUMULATIVE_PR_REDERIVE_QUEUE, - CUMULATIVE_PR_REDERIVE_CONCURRENCY, - runCumulativePrRederivationForUser, - enqueueBootTimeCumulativePrRederivation, - type CumulativePrRederivePayload, -} from "@/lib/personal-records/cumulative-pr-rederivation"; -import { - MEAN_CONSOLIDATION_QUEUE, - MEAN_CONSOLIDATION_CONCURRENCY, - runMeanConsolidationForUser, - enqueueBootTimeMeanConsolidation, - type MeanConsolidationPayload, -} from "@/lib/jobs/mean-consolidation"; -import { - DENSE_INTRADAY_RETENTION_QUEUE, - DENSE_INTRADAY_RETENTION_CONCURRENCY, - runDenseIntradayRetentionForUser, - enqueueBootTimeDenseIntradayRetention, - DENSE_INTRADAY_RETENTION_ENABLED, - type DenseIntradayRetentionPayload, -} from "@/lib/jobs/dense-intraday-retention"; -import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention"; -import { - DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, - DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY, - runDenseIntradayHourlyRebuildForUser, - enqueueBootTimeDenseIntradayHourlyRebuild, - type DenseIntradayHourlyRebuildPayload, -} from "@/lib/jobs/dense-intraday-hourly-rebuild"; -import { - getWorkerPrisma, - workerLog, - BOOT_BACKFILL_STAGGER_SECONDS, -} from "./shared"; -import { heavyLane } from "@/lib/jobs/heavy-lane"; -import { lockedPass } from "@/lib/jobs/long-pass"; -import { - createAndSchedule, - createAndWork, - cronIsTheRetry, - retireQueues, - type QueuePolicyTable, - type ScheduleEntry, -} from "./registrar-shared"; -import { jobDone, jobFailed } from "@/lib/jobs/job-outcome"; -import { jobBudget } from "@/lib/jobs/job-budget"; -import { - DRAIN_CONTINUATION_DELAY_SECONDS, - nextDrainContinuation, -} from "@/lib/jobs/drain-continuation"; -// v1.4.37 — nightly drain of per-sample APPLE_HEALTH cumulative rows. -// Collapses each user × cumulative-type × calendar-day bucket into one -// `stats:…` row so the list view stops painting hundreds of step chunks per -// day. 03:45 Europe/Berlin slots in between the 03:15 audit-log cleanup and -// the 04:00 feedback aggregator. The 36-hour grace window keeps today + the -// trailing watch-sync window intact for real-time visibility; only -// completed-and-stable days fall to the drain. - -const DRAIN_CUMULATIVE_QUEUE = "drain-per-sample-cumulative"; - -const DRAIN_CUMULATIVE_CRON = "45 3 * * *"; - -interface DrainCumulativePayload { - triggeredAt?: string; - /** - * How many times this tick has already handed itself on. Absent on the - * cron's own job. See `nextDrainContinuation`. - */ - continuation?: number; -} - -const allQueues = [ - ROLLUP_RECOMPUTE_QUEUE, - ROLLUP_FULL_BACKFILL_QUEUE, - // v1.4.39 — per-bucket WEEK/MONTH/YEAR fold queue for the - // mood rollup tier. The DAY pass runs inline in the write hook; - // these enqueue paths cover the cross-granularity buckets the - // worker materialises off the request path. - MOOD_ROLLUP_RECOMPUTE_QUEUE, - // v1.4.39 — boot-time fold queue for the mood rollup tier. - MOOD_ROLLUP_FULL_BACKFILL_QUEUE, - // v1.5.6 — boot-time legacy step consolidation. Discovery enqueues - // one job per user still holding live pre-v1.5.0 granular step rows. - STEP_CONSOLIDATION_QUEUE, - // v1.28.37 — one-shot repair for the provider step rows the pre-fix - // consolidation swept. Boot-discovery driven, self-converging. - STEP_CONSOLIDATION_REPAIR_QUEUE, - // v1.30.3 — one-shot repair for the cumulative-type PersonalRecord rows - // the pre-fix multi-source SUM inflated. Boot-discovery driven, - // self-converging (QA F4). - CUMULATIVE_PR_REDERIVE_QUEUE, - // v1.7.0 — daily-mean consolidation for high-frequency spot HealthKit - // metrics (walking speed/step length, respiratory rate, audio exposure). - MEAN_CONSOLIDATION_QUEUE, - // v1.10.0 — computed scores (WX-E). Dense intra-day retention drain of - // daytime HRV / HR samples (per-user backfill queue, boot-discovery - // driven like mean-consolidation; the steady-state nightly walk folds - // onto the drain-cumulative tick). - DENSE_INTRADAY_RETENTION_QUEUE, - // v1.28.31 — one-shot hourly history rebuild for pre-hourly folded - // dense-tier days. Boot-discovery driven and self-converging (a rebuilt - // day's retired daily row drops it from the discovery pairing). - DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, - // v1.4.37 — explicit createQueue is required before the nightly - // schedule below registers (pg-boss v12 contract). Without this entry the - // drain schedule silently no-ops and the per-sample APPLE_HEALTH rows - // never collapse. - DRAIN_CUMULATIVE_QUEUE, - // v1.43 — one account's supplement doses recomputed into the day's - // micronutrients after a change that spans more than one day. - SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE, -]; - -const schedules: ScheduleEntry[] = [ - // v1.4.37 — nightly fold of per-sample APPLE_HEALTH cumulative - // rows into one row per day per type. Slots between the - // audit-log cleanup (03:15) and the feedback aggregator (04:00). - [DRAIN_CUMULATIVE_QUEUE, DRAIN_CUMULATIVE_CRON, cronIsTheRetry], -]; - -/** - * De-duplication policy per rollup queue. Until this table existed every queue - * here ran under pg-boss's `standard` policy, under which no unique index - * constrains `singleton_key` — so the `singletonKey` each enqueue site passes - * was inert and de-duplicated nothing. - * - * The split below is deliberate and the two halves are NOT interchangeable. - */ -const queuePolicies: QueuePolicyTable = { - // `short`, not `exclusive`. A recompute reads the bucket's live rows when it - // STARTS, so collapsing sends that pile up while an identical job is still - // queued is safe — the queued job will observe every write that landed in the - // meantime. This is what kills the storm: a sync posting a few hundred batch - // requests fanned out three jobs per type-day per request, thousands of jobs - // re-deriving byte-identical rows. Under `short` they collapse to one queued - // job per bucket. `exclusive` would be wrong here: it would also suppress a - // send issued after the recompute had already started reading, stranding the - // newer measurement in a stale bucket until the nightly pass. - [ROLLUP_RECOMPUTE_QUEUE]: { - policy: "short", - reason: - "Burst coalescing for the write-path fan-out. The handler re-reads the bucket at run time, so collapsing queued duplicates cannot strand a write; exclusive could.", - }, - [MOOD_ROLLUP_RECOMPUTE_QUEUE]: { - policy: "short", - reason: - "Same shape as the measurement recompute: re-reads the bucket at run time, so only queued duplicates may be collapsed.", - }, - [SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE]: { - policy: "short", - reason: - "Per-account recompute of supplement nutrient rows; the pass reads the live doses and composition when it starts, so a queued duplicate may be collapsed, while a send during a running pass must still queue the next one.", - }, - - // The rest are per-user, self-converging boot/discovery backfills. A second - // concurrent run is pure duplicated work, and the discovery predicate - // re-enqueues on the next boot or cron tick while work remains outstanding — - // so `exclusive` (queued OR active OR awaiting retry) can never lose work it - // does not re-offer. `short` would be too weak: these passes run for minutes - // to hours, and the observed failure was precisely a worker restarting - // mid-pass and appending another identical full-history job per restart. - [ROLLUP_FULL_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user full-history backfill; long-running. Boot discovery re-enqueues while coverage is still missing, so suppressing a duplicate of an ACTIVE pass is safe.", - }, - [MOOD_ROLLUP_FULL_BACKFILL_QUEUE]: { - policy: "exclusive", - reason: - "Per-user mood backfill; boot-discovery driven and self-converging on rollup coverage.", - }, - [STEP_CONSOLIDATION_QUEUE]: { - policy: "exclusive", - reason: - "Per-user legacy step consolidation; discovery drops the user once no live granular rows remain.", - }, - [STEP_CONSOLIDATION_REPAIR_QUEUE]: { - policy: "exclusive", - reason: - "Per-user one-shot repair; discovery drops the user once the affected provider rows are repaired.", - }, - [CUMULATIVE_PR_REDERIVE_QUEUE]: { - policy: "exclusive", - reason: - "Per-user one-shot re-derivation; discovery drops the user once the inflated records are rewritten.", - }, - [MEAN_CONSOLIDATION_QUEUE]: { - policy: "exclusive", - reason: - "Per-user daily-mean consolidation; discovery drops the user once the spot rows are folded.", - }, - [DENSE_INTRADAY_RETENTION_QUEUE]: { - policy: "exclusive", - reason: - "Per-user dense-tier drain; the multi-hour pass on a heavy account is exactly the case where a restart used to append a duplicate full-history job.", - }, - [DENSE_INTRADAY_HOURLY_REBUILD_QUEUE]: { - policy: "exclusive", - reason: - "Per-user one-shot hourly rebuild; discovery drops the day once its retired daily row is gone.", - }, - - // DRAIN_CUMULATIVE_QUEUE is deliberately absent: it is a keyless nightly cron - // tick with no singletonKey on any send, so a policy would only constrain the - // empty key and buy nothing. -}; - -/** - * The boot backfill of the per-day medication-compliance rollup, whose table - * had no reader left and was dropped. Its queue, and any job still waiting in - * it, is removed on an instance that still has it. - */ -const RETIRED_ROLLUP_QUEUES = ["medication-compliance-full-backfill"] as const; - -/** - * Register every rollup / consolidation queue. Returns the queue names created - * (for the boot-level aggregate assertion). - */ -export async function registerRollupQueues( - boss: PgBoss, -): Promise { - await createAndSchedule(boss, allQueues, schedules, queuePolicies); - await retireQueues(boss, RETIRED_ROLLUP_QUEUES); - - // v1.5.0 — persistent measurement rollup worker. Folds the - // WEEK / MONTH / YEAR buckets that the write-path hooks enqueue; - // the DAY bucket is already recomputed synchronously by the hook - // itself. Concurrency-2 keeps two recomputes in flight without - // crowding the dashboard request pool. - await createAndWork( - boss, - ROLLUP_RECOMPUTE_QUEUE, - { localConcurrency: ROLLUP_RECOMPUTE_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - const payload = job.data; - await recomputeUserRollups(payload.userId, { - types: [payload.type], - granularities: [payload.granularity], - from: new Date(payload.from), - to: new Date(payload.to), - }); - } - // No per-bucket isolation here on purpose: a recompute that throws - // fails the job, so reaching this line means every bucket in the batch - // was folded. - return jobDone({ buckets_recomputed: jobs.length }); - }, - ); - - // v1.4.35.1 — boot-time full-fold worker. The boot enqueue helper - // below sends one job per uncovered user; this handler runs the - // full `recomputeUserRollups` against the default 5-year window - // across every granularity. Serial concurrency so the populator - // never crowds the dashboard request pool. - await createAndWork( - boss, - ROLLUP_FULL_BACKFILL_QUEUE, - { localConcurrency: ROLLUP_FULL_BACKFILL_CONCURRENCY }, - // The lock sits outside the lane: a retry pg-boss starts after the expiry - // while the first fold still runs finds the account's lock and settles, - // instead of waiting in the lane and folding the account again. - lockedPass( - ROLLUP_FULL_BACKFILL_QUEUE, - (job) => job.data.userId, - heavyLane(ROLLUP_FULL_BACKFILL_QUEUE, async (jobs) => { - const shouldStop = jobBudget(jobs); - let rowsUpsertedTotal = 0; - for (const job of jobs) { - const { userId } = job.data; - const { rowsUpserted, durationMs, stopped } = - await recomputeUserRollups(userId, { shouldStop }); - if (stopped) { - // Each granularity committed on its own; the retry folds again. - throw new Error( - `rollup-full-backfill for user ${userId} stopped on its job budget; the retry folds the rest`, - ); - } - rowsUpsertedTotal += rowsUpserted; - workerLog( - "info", - `[rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, - ); - } - return jobDone({ - users: jobs.length, - rows_upserted: rowsUpsertedTotal, - }); - }), - ), - ); - - // v1.5.6 — legacy step consolidation worker. The boot enqueue helper - // below sends one job per user still holding live pre-v1.5.0 granular - // step rows; this handler collapses them into one daily-total row per - // calendar day and soft-deletes the originals. Serial concurrency so - // the populator never crowds the dashboard request pool. - await createAndWork( - boss, - STEP_CONSOLIDATION_QUEUE, - { localConcurrency: STEP_CONSOLIDATION_CONCURRENCY }, - async (jobs) => { - let daysTotal = 0; - let legacyRowsTotal = 0; - for (const job of jobs) { - const { userId } = job.data; - const { daysConsolidated, legacyRowsSoftDeleted } = - await runStepConsolidationForUser(userId); - daysTotal += daysConsolidated; - legacyRowsTotal += legacyRowsSoftDeleted; - workerLog( - "info", - `[step-consolidation] user=${userId} days=${daysConsolidated} legacyRowsSoftDeleted=${legacyRowsSoftDeleted}`, - ); - } - return jobDone({ - users: jobs.length, - days_consolidated: daysTotal, - legacy_rows_soft_deleted: legacyRowsTotal, - }); - }, - ); - - // v1.28.37 — step-consolidation repair worker. Resurrects the - // GOOGLE_HEALTH / FITBIT daily-total step rows the pre-fix consolidation - // swept and removes the shadow MANUAL totals it minted. Serial - // concurrency so the repair never crowds the dashboard request pool. - await createAndWork( - boss, - STEP_CONSOLIDATION_REPAIR_QUEUE, - { localConcurrency: STEP_CONSOLIDATION_REPAIR_CONCURRENCY }, - async (jobs) => { - let rowsResurrected = 0; - let wedgeSkipped = 0; - let manualMintsRemoved = 0; - let daysRecomputed = 0; - let failures = 0; - for (const job of jobs) { - const { userId } = job.data; - const summary = await runStepConsolidationRepairForUser(userId); - rowsResurrected += summary.rowsResurrected; - wedgeSkipped += summary.wedgeSkipped; - manualMintsRemoved += summary.manualMintsRemoved; - daysRecomputed += summary.daysRecomputed; - // The repair isolates its own per-day failures and reports them as a - // count, so they ride out as a fact rather than failing the job. - failures += summary.failures; - workerLog( - "info", - `[step-consolidation-repair] user=${userId} resurrected=${summary.rowsResurrected} wedgeSkipped=${summary.wedgeSkipped} manualMintsRemoved=${summary.manualMintsRemoved} daysRecomputed=${summary.daysRecomputed} failures=${summary.failures}`, - ); - } - return jobDone({ - users: jobs.length, - rows_resurrected: rowsResurrected, - wedge_skipped: wedgeSkipped, - manual_mints_removed: manualMintsRemoved, - days_recomputed: daysRecomputed, - failures, - }); - }, - ); - - // v1.43 — supplement nutrient recompute. One account per job, serial. - await createAndWork( - boss, - SUPPLEMENT_NUTRIENT_RECOMPUTE_QUEUE, - { localConcurrency: SUPPLEMENT_NUTRIENT_RECOMPUTE_CONCURRENCY }, - async (jobs) => { - let rowsWritten = 0; - for (const job of jobs) { - const result = await runSupplementNutrientRecompute(job.data); - rowsWritten += result.rowsWritten; - } - return jobDone({ users: jobs.length, rows_written: rowsWritten }); - }, - ); - - // v1.30.3 — cumulative-PersonalRecord rederivation worker (QA F4). The - // boot enqueue helper below sends one job per user still holding a - // suspect pre-fix inflated cumulative-type record; this handler deletes - // it and re-runs detection silently so the honest re-derived best takes - // its place. Serial concurrency so the repair never crowds the - // dashboard request pool. - await createAndWork( - boss, - CUMULATIVE_PR_REDERIVE_QUEUE, - { localConcurrency: CUMULATIVE_PR_REDERIVE_CONCURRENCY }, - async (jobs) => { - let rowsDeleted = 0; - let rowsReinserted = 0; - for (const job of jobs) { - const { userId } = job.data; - const summary = await runCumulativePrRederivationForUser(userId); - rowsDeleted += summary.rowsDeleted; - rowsReinserted += summary.rowsReinserted; - workerLog( - "info", - `[cumulative-pr-rederive] user=${userId} rowsDeleted=${summary.rowsDeleted} rowsReinserted=${summary.rowsReinserted}`, - ); - } - return jobDone({ - users: jobs.length, - rows_deleted: rowsDeleted, - rows_reinserted: rowsReinserted, - }); - }, - ); - - // v1.7.0 — daily-mean consolidation worker. The boot enqueue helper - // below sends one job per user holding live per-sample high-frequency - // mean-type rows; this handler collapses each completed day to its - // mean and soft-deletes the originals. Serial concurrency so the - // populator never crowds the dashboard request pool. - await createAndWork( - boss, - MEAN_CONSOLIDATION_QUEUE, - { localConcurrency: MEAN_CONSOLIDATION_CONCURRENCY }, - async (jobs) => { - const shouldStop = jobBudget(jobs); - let daysTotal = 0; - let perSampleRowsTotal = 0; - let stoppedEarly = false; - for (const job of jobs) { - const { userId } = job.data; - const summary = await runMeanConsolidationForUser(userId, shouldStop); - const { daysConsolidated, perSampleRowsSoftDeleted } = summary; - stoppedEarly ||= summary.stoppedEarly; - daysTotal += daysConsolidated; - perSampleRowsTotal += perSampleRowsSoftDeleted; - workerLog( - "info", - `[mean-consolidation] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted}`, - ); - } - // Out of budget is a completed run, not a failed one: the folded days - // are committed, and the nightly tick carries on from the first day - // this run did not reach. - return jobDone({ - users: jobs.length, - days_consolidated: daysTotal, - per_sample_rows_soft_deleted: perSampleRowsTotal, - stopped_early: stoppedEarly, - }); - }, - ); - - // v1.10.0 WX-E — dense intra-day retention per-user backfill worker. The - // boot enqueue helper below sends one job per user holding live per-sample - // dense-tier (HRV / HR / SpO2) rows older than the retention window; this - // handler folds those out-of-window samples to hourly means (v1.28.31) and - // soft-deletes the originals, keeping the in-window intra-day shape intact - // for the Stress engine. Serial concurrency so the backfill never crowds - // the request pool. - await createAndWork( - boss, - DENSE_INTRADAY_RETENTION_QUEUE, - { localConcurrency: DENSE_INTRADAY_RETENTION_CONCURRENCY }, - async (jobs) => { - const shouldStop = jobBudget(jobs); - let daysTotal = 0; - let perSampleRowsTotal = 0; - let derivedRestingRowsTotal = 0; - let stoppedEarly = false; - for (const job of jobs) { - const { userId } = job.data; - try { - const summary = await runDenseIntradayRetentionForUser( - userId, - shouldStop, - ); - const { - daysConsolidated, - perSampleRowsSoftDeleted, - derivedRestingRowsUpserted, - } = summary; - stoppedEarly ||= summary.stoppedEarly; - daysTotal += daysConsolidated; - perSampleRowsTotal += perSampleRowsSoftDeleted; - derivedRestingRowsTotal += derivedRestingRowsUpserted; - workerLog( - "info", - `[dense-intraday-retention] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted} derivedRestingRowsUpserted=${derivedRestingRowsUpserted}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[dense-intraday-retention] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - users: jobs.length, - days_consolidated: daysTotal, - per_sample_rows_soft_deleted: perSampleRowsTotal, - derived_resting_rows_upserted: derivedRestingRowsTotal, - // Out of budget completes the job: the folded days are committed and - // the nightly tick, which hands itself on while work remains, - // continues from the first day this run did not reach. - stopped_early: stoppedEarly, - }); - }, - ); - - // v1.28.31 — one-shot hourly history rebuild worker. The boot enqueue - // helper below sends one job per user still holding a pre-hourly daily - // `stats:` row paired with tombstoned raw rows; this handler reconstructs - // hourly means from the tombstones and retires the daily row atomically. - // Self-converging: a rebuilt day drops out of the discovery pairing, so - // the pass runs once per install. Serial concurrency so the rebuild never - // crowds the request pool. - await createAndWork( - boss, - DENSE_INTRADAY_HOURLY_REBUILD_QUEUE, - { localConcurrency: DENSE_INTRADAY_HOURLY_REBUILD_CONCURRENCY }, - async (jobs) => { - const shouldStop = jobBudget(jobs); - let stoppedEarly = false; - let daysRebuiltTotal = 0; - let hourlyRowsTotal = 0; - let dailyRowsRetiredTotal = 0; - let daysSkippedTotal = 0; - for (const job of jobs) { - const { userId } = job.data; - try { - const summary = await runDenseIntradayHourlyRebuildForUser( - userId, - shouldStop, - ); - const { - daysRebuilt, - hourlyRowsUpserted, - dailyRowsRetired, - daysSkippedNoTombstones, - } = summary; - stoppedEarly ||= summary.stoppedEarly; - daysRebuiltTotal += daysRebuilt; - hourlyRowsTotal += hourlyRowsUpserted; - dailyRowsRetiredTotal += dailyRowsRetired; - daysSkippedTotal += daysSkippedNoTombstones; - workerLog( - "info", - `[dense-intraday-hourly-rebuild] user=${userId} daysRebuilt=${daysRebuilt} hourlyRowsUpserted=${hourlyRowsUpserted} dailyRowsRetired=${dailyRowsRetired} daysSkippedNoTombstones=${daysSkippedNoTombstones}`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[dense-intraday-hourly-rebuild] user=${userId} failed`, - err, - ); - throw err; - } - } - const did = { - users: jobs.length, - stopped_early: stoppedEarly, - days_rebuilt: daysRebuiltTotal, - hourly_rows_upserted: hourlyRowsTotal, - daily_rows_retired: dailyRowsRetiredTotal, - days_skipped_no_tombstones: daysSkippedTotal, - }; - // Out of budget: fail so pg-boss retries (the boot send allows three - // retries), and the retry starts at the first day not yet rebuilt. The - // queue is `exclusive`, so the job cannot hand itself on while active. - if (stoppedEarly) { - return jobFailed( - "stopped before every day was rebuilt; the retry resumes", - undefined, - did, - ); - } - return jobDone(did); - }, - ); - - // v1.4.39 — mood-rollup per-bucket worker. Folds the - // WEEK / MONTH / YEAR buckets that the mood-entry write hooks - // enqueue; the DAY bucket runs synchronously in the hook itself. - // No current read path consumes these buckets — they exist so a - // future cross-granularity reader can ship without a backfill - // step. Concurrency-2 mirrors the measurement-rollup worker. - await createAndWork( - boss, - MOOD_ROLLUP_RECOMPUTE_QUEUE, - { localConcurrency: MOOD_ROLLUP_RECOMPUTE_CONCURRENCY }, - async (jobs) => { - for (const job of jobs) { - const payload = job.data; - await recomputeUserMoodRollups(payload.userId, { - granularities: [payload.granularity], - from: new Date(payload.from), - to: new Date(payload.to), - }); - } - return jobDone({ buckets_recomputed: jobs.length }); - }, - ); - - // v1.4.39 — mood-rollup boot-time fold worker. The boot - // discovery helper below sends one job per user with mood entries - // but zero rollup rows; this handler folds the full 5-year window - // across every granularity. Concurrency-1 so the populator never - // crowds the request pool. - await createAndWork( - boss, - MOOD_ROLLUP_FULL_BACKFILL_QUEUE, - { localConcurrency: MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY }, - async (jobs) => { - let rowsUpsertedTotal = 0; - for (const job of jobs) { - const { userId } = job.data; - try { - // v1.32.12 — delete-then-refold. Empties the user's rollup - // rows before refolding so the boot backfill is self-cleaning - // against any row minted under an older keying (e.g. a - // rolling-deploy old-container write during the swap window). - // The brief empty window serves correct data via the live - // `entry.date` fallback. - const { rowsUpserted, durationMs } = await recomputeUserMoodRollups( - userId, - { replace: true }, - ); - rowsUpsertedTotal += rowsUpserted; - workerLog( - "info", - `[mood-rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, - ); - } catch (err) { - recordError(); - workerLog( - "error", - `[mood-rollup-full-backfill] user=${userId} failed`, - err, - ); - throw err; - } - } - return jobDone({ - users: jobs.length, - rows_upserted: rowsUpsertedTotal, - }); - }, - ); - - // v1.4.37 — nightly drain worker. Walks every user × cumulative - // type and folds per-sample APPLE_HEALTH rows older than the cutoff - // into one `stats:…` row per calendar day. Idempotent — a second run - // collapses zero buckets once every day is in the `stats:` shape. - // Concurrency-1 so the drain never crowds the dashboard request pool - // and a long backfill on the maintainer's account (300 k+ measurement rows) - // stays a single sequential walk. - await createAndWork( - boss, - DRAIN_CUMULATIVE_QUEUE, - { localConcurrency: 1 }, - async (jobs) => { - // One budget shared by all three passes, run in turn. - const shouldStop = jobBudget(jobs); - let stoppedEarly = false; - // The three drains below are independent passes sharing one nightly - // tick, so one failing pass must not stop the other two from running. - // The first failure is held back and reported once the tick is over — - // that keeps the existing order and still fails the job, which is what - // the swallowed errors never did. - let firstFailure: { reason: string; cause: unknown } | undefined; - let passesFailed = 0; - let bucketsCollapsed = 0; - let perSampleRowsDeleted = 0; - let meanDaysConsolidated = 0; - let meanRowsSoftDeleted = 0; - let denseDaysConsolidated = 0; - let denseRowsSoftDeleted = 0; - let denseSkipped = false; - - for (const job of jobs) { - try { - const summary = await drainPerSampleCumulative(getWorkerPrisma(), { - dryRun: false, - cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS, - shouldStop, - log: (line) => workerLog("info", line), - }); - stoppedEarly ||= summary.stoppedEarly; - bucketsCollapsed += summary.totals.bucketsCollapsed; - perSampleRowsDeleted += summary.totals.perSampleRowsDeleted; - workerLog( - "info", - `[drain-cumulative] triggeredAt=${job.data.triggeredAt} usersScanned=${summary.totals.usersScanned} bucketsCollapsed=${summary.totals.bucketsCollapsed} perSampleRowsDeleted=${summary.totals.perSampleRowsDeleted} dailyRowsUpserted=${summary.totals.dailyRowsUpserted}`, - ); - } catch (err) { - recordError(); - await reportWorkerError("drain-cumulative", err); - passesFailed++; - firstFailure ??= { reason: "cumulative drain failed", cause: err }; - } - - // v1.8.5 — fold the daily-MEAN drain onto the same nightly tick as - // the cumulative (SUM) drain. The mean drain was previously - // boot-discovery only, so between worker reboots new high-frequency - // spot samples (walking speed, respiratory rate, gait/mobility, - // audio exposure) accumulated raw. Running both passes on one - // concurrency-1 cron keeps the maintenance window a single - // sequential walk and never crowds the request pool. The global - // (no `userId`) signature drains every user; the 36-hour grace - // cutoff keeps today's in-flight watch syncs raw. Boot discovery - // stays as the back-fill for accounts that accumulated raw rows - // before this cron shipped. - try { - const meanSummary = await consolidateDailyMean(getWorkerPrisma(), { - dryRun: false, - cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS, - shouldStop, - log: (line) => workerLog("info", line), - }); - stoppedEarly ||= meanSummary.stoppedEarly; - meanDaysConsolidated += meanSummary.totals.daysConsolidated; - meanRowsSoftDeleted += meanSummary.totals.perSampleRowsSoftDeleted; - workerLog( - "info", - `[mean-consolidation] triggeredAt=${job.data.triggeredAt} usersScanned=${meanSummary.totals.usersScanned} daysConsolidated=${meanSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${meanSummary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${meanSummary.totals.dailyRowsUpserted} daysFailed=${meanSummary.totals.daysFailed}`, - ); - } catch (err) { - recordError(); - await reportWorkerError("mean-consolidation", err, { - tick: "nightly", - }); - passesFailed++; - firstFailure ??= { reason: "mean consolidation failed", cause: err }; - } - - // v1.10.0 WX-E — fold the dense intra-day retention drain onto the - // same nightly tick. Unlike the daily-mean drain, this scopes to the - // dense-tier types (HEART_RATE_VARIABILITY, PULSE) and keeps the last - // DENSE_INTRADAY_RETENTION_DAYS of raw per-sample rows so the Stress - // engine still sees the intra-day SDNN shape; only out-of-window - // samples fold to a daily mean. These two types are NEVER in the - // destructive HIGH_FREQUENCY_MEAN_TYPES allowlist — the drain - // exemption the intra-day shape depends on. The global (no `userId`) - // signature drains every user. Boot discovery (below) back-fills - // accounts that accumulated out-of-window raw rows before this - // shipped. - try { - if (!DENSE_INTRADAY_RETENTION_ENABLED) { - denseSkipped = true; - workerLog( - "info", - "[dense-intraday-retention] disabled via DENSE_INTRADAY_RETENTION_ENABLED — skipping the nightly walk (operator kill-switch)", - ); - } else { - const denseSummary = await runDenseIntradayRetention( - getWorkerPrisma(), - { - dryRun: false, - shouldStop, - log: (line) => workerLog("info", line), - }, - ); - stoppedEarly ||= denseSummary.stoppedEarly; - denseDaysConsolidated += denseSummary.totals.daysConsolidated; - denseRowsSoftDeleted += - denseSummary.totals.perSampleRowsSoftDeleted; - workerLog( - "info", - `[dense-intraday-retention] triggeredAt=${job.data.triggeredAt} usersScanned=${denseSummary.totals.usersScanned} daysConsolidated=${denseSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${denseSummary.totals.perSampleRowsSoftDeleted} hourlyRowsUpserted=${denseSummary.totals.hourlyRowsUpserted} dailyRowsRetired=${denseSummary.totals.dailyRowsRetired}`, - ); - } - } catch (err) { - recordError(); - await reportWorkerError("dense-intraday-retention", err, { - tick: "nightly", - }); - passesFailed++; - firstFailure ??= { reason: "dense retention failed", cause: err }; - } - } - - // Hand the remaining work to a follow-up job when this one ran out of - // budget (bounded; see `nextDrainContinuation`). - const continuation = Math.max( - 0, - ...jobs.map((job) => job.data.continuation ?? 0), - ); - const next = nextDrainContinuation({ - stoppedEarly, - daysFolded: - bucketsCollapsed + meanDaysConsolidated + denseDaysConsolidated, - continuation, - }); - let continued = false; - if (next !== null) { - try { - const followUp: DrainCumulativePayload = { - triggeredAt: new Date().toISOString(), - continuation: next, - }; - continued = - (await boss.send(DRAIN_CUMULATIVE_QUEUE, followUp, { - ...cronIsTheRetry, - startAfter: DRAIN_CONTINUATION_DELAY_SECONDS, - })) !== null; - } catch (err) { - // The next nightly tick picks the work up regardless; a lost - // follow-up only delays it. - workerLog("error", "[drain-cumulative] follow-up send failed", err); - } - } - - const did = { - stopped_early: stoppedEarly, - continuation, - continued, - passes_failed: passesFailed, - buckets_collapsed: bucketsCollapsed, - per_sample_rows_deleted: perSampleRowsDeleted, - mean_days_consolidated: meanDaysConsolidated, - mean_rows_soft_deleted: meanRowsSoftDeleted, - dense_days_consolidated: denseDaysConsolidated, - dense_rows_soft_deleted: denseRowsSoftDeleted, - dense_skipped: denseSkipped, - }; - - if (firstFailure) { - return jobFailed(firstFailure.reason, firstFailure.cause, did); - } - return jobDone(did); - }, - ); - - return allQueues; -} - -/** - * Fire-and-forget boot discovery for the self-converging rollup / consolidation - * backfills. Each pass is idempotent across reboots and never fails worker boot - * on a miss (errors come back through the helper's result value). - * - * Each backfill is `localConcurrency: 1` on its own, but every one of them used - * to drain from the first pg-boss poll at boot — on a heavy tenant that meant - * several full-history loads contending for the connection pool at once (a boot - * storm / crash-loop risk). Each type now gets an increasing - * `BOOT_BACKFILL_STAGGER_SECONDS` multiple as a `startAfter` delay so their - * loads spread across a window instead of all landing on the same poll. Dense - * intra-day retention keeps its own larger boot defer (the established P2028 - * pool-exhaustion fix) and stays the furthest-out stage. - */ -export async function enqueueRollupBootDiscovery(): Promise { - // v1.4.35.1 — measurement-rollup backfill. Finds every user with - // measurements but no rollup coverage and enqueues a full-fold per account. - // The discovery query only matches accounts with zero rollup rows, so once - // a fold completes the user drops off the list. - try { - // Stage 0 — first off the line; no stagger delay. - const { enqueued, skipped, error } = await enqueueBootTimeRollupBackfill( - BOOT_BACKFILL_STAGGER_SECONDS * 0, - ); - if (error) { - workerLog( - "error", - `[rollup-full-backfill] boot discovery failed: ${error}`, - ); - } else { - // v1.4.38.7 — log the discovery result on every boot, including - // the silent `enqueued=0 skipped=0` case. Without this, an - // operator chasing "analytics is slow" cannot tell whether the - // discovery query ran successfully (and found nothing to fold) - // vs. silently no-op'd. The line is one row per worker boot, so - // the log cost is negligible. - workerLog( - "info", - `[rollup-full-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[rollup-full-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.5.6 — legacy step consolidation. Finds every user still holding live - // pre-v1.5.0 granular step rows and enqueues one consolidation job per - // account. Consolidated legacy rows are soft-deleted, so the - // `deleted_at IS NULL` discovery predicate drops them. - try { - // Stage 1. - const { enqueued, skipped, error } = await enqueueBootTimeStepConsolidation( - BOOT_BACKFILL_STAGGER_SECONDS * 1, - ); - if (error) { - workerLog( - "error", - `[step-consolidation] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[step-consolidation] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[step-consolidation] boot discovery threw an unexpected error", - err, - ); - } - - // v1.7.0 — daily-mean consolidation. Finds every user holding live - // per-sample high-frequency mean-type rows and enqueues one job per account. - // Consolidated rows are soft-deleted, so the discovery predicate drops them. - try { - // Stage 2. - const { enqueued, skipped, error } = await enqueueBootTimeMeanConsolidation( - BOOT_BACKFILL_STAGGER_SECONDS * 2, - ); - if (error) { - workerLog( - "error", - `[mean-consolidation] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[mean-consolidation] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[mean-consolidation] boot discovery threw an unexpected error", - err, - ); - } - - // v1.10.0 WX-E — dense intra-day retention drain. Finds every user holding - // live per-sample dense-tier (HRV / HR) rows OLDER than the retention window - // and enqueues one job per account. Folded rows are soft-deleted, so the - // discovery predicate drops them. - try { - // Furthest-out stage — the helper self-defers past the boot window via its - // own larger constant (the P2028 pool-exhaustion fix), so no stagger - // argument is threaded here. - const { enqueued, skipped, error } = - await enqueueBootTimeDenseIntradayRetention(); - if (error) { - workerLog( - "error", - `[dense-intraday-retention] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[dense-intraday-retention] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[dense-intraday-retention] boot discovery threw an unexpected error", - err, - ); - } - - // v1.28.31 — one-shot hourly history rebuild. Finds every user still - // holding a pre-hourly daily dense-tier `stats:` row paired with tombstoned - // raw rows and enqueues one rebuild job per account. Rebuilt days retire - // their daily row, so the pairing predicate converges to zero across - // boots — the retired row is the durable once-per-install marker. - try { - // Self-defers past both the boot window and the retention drain's own - // 600 s stage via its larger constant; no stagger argument threaded. - const { enqueued, skipped, error } = - await enqueueBootTimeDenseIntradayHourlyRebuild(); - if (error) { - workerLog( - "error", - `[dense-intraday-hourly-rebuild] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[dense-intraday-hourly-rebuild] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[dense-intraday-hourly-rebuild] boot discovery threw an unexpected error", - err, - ); - } - - // v1.4.39 — mood rollup tier. Mirrors the v1.4.35.1 - // measurement-rollup pattern: one job per user with mood entries but no - // rollup coverage. Idempotent across reboots and singleton-keyed inside - // pg-boss so a fast restart while a backfill is queued doesn't double up. - try { - // Stage 3. - const { enqueued, skipped, error } = - await enqueueBootTimeMoodRollupBackfill( - BOOT_BACKFILL_STAGGER_SECONDS * 3, - ); - if (error) { - workerLog( - "error", - `[mood-rollup-full-backfill] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[mood-rollup-full-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[mood-rollup-full-backfill] boot discovery threw an unexpected error", - err, - ); - } - - // v1.28.37 — step-consolidation repair. Finds every account still holding - // a resurrectable tombstoned GOOGLE_HEALTH / FITBIT daily-total step row - // (swept by the pre-fix consolidation) and enqueues one repair job per - // account. Self-converging: a resurrected row goes live and drops out of - // the discovery predicate. - try { - // Stage 5. - const { enqueued, skipped, error } = - await enqueueBootTimeStepConsolidationRepair( - BOOT_BACKFILL_STAGGER_SECONDS * 5, - ); - if (error) { - workerLog( - "error", - `[step-consolidation-repair] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[step-consolidation-repair] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[step-consolidation-repair] boot discovery threw an unexpected error", - err, - ); - } - - // v1.30.3 — cumulative-PersonalRecord rederivation. Finds every - // account still holding a suspect measurement-driven cumulative-type - // record created before the source-collapse fix landed and enqueues one - // repair job per account. Self-converging: a deleted-and-redetected - // record's `createdAt` sits after the fix cutoff, so the account drops - // out of the discovery predicate for good. - try { - // Stage 6. - const { enqueued, skipped, error } = - await enqueueBootTimeCumulativePrRederivation( - BOOT_BACKFILL_STAGGER_SECONDS * 6, - ); - if (error) { - workerLog( - "error", - `[cumulative-pr-rederive] boot discovery failed: ${error}`, - ); - } else { - workerLog( - "info", - `[cumulative-pr-rederive] boot discovery: enqueued=${enqueued} skipped=${skipped}`, - ); - } - } catch (err) { - workerLog( - "error", - "[cumulative-pr-rederive] boot discovery threw an unexpected error", - err, - ); - } -} diff --git a/src/lib/jobs/reminder/register-status.ts b/src/lib/jobs/reminder/register-status.ts deleted file mode 100644 index 81f241b10..000000000 --- a/src/lib/jobs/reminder/register-status.ts +++ /dev/null @@ -1,878 +0,0 @@ -/** - * Status / insight / computed-score queue registrar. - * - * Owns the nightly status batch (every per-metric status card, one provider - * call per account), the comprehensive-insight - * pre-generation, the on-demand per-metric status generation, the nightly - * computed scores (recovery / stress / strain), the proactive Coach nudge, the - * Coach memory refresh, the period-narrative warm, and the medication - * low-stock pass. - * - * v1.4.37 dead-queue contract: every queue name appears in `allQueues`, its - * cron (where it has one) appears as a `[QUEUE, CRON]` tuple in `schedules`, - * and a `createAndWork(boss, QUEUE, …, handler)` binding drains it. The - * status-wiring guards (`status-cron-candidates`, `insight-pregenerate`, - * `insight-status-generate`, `recovery-score-queue`, - * `stress-strain-retention-queue`, `period-narrative-warm`, - * `coach-memory-refresh-queue`) read THIS module. - */ -import { PgBoss } from "pg-boss"; -import { reportWorkerError } from "@/lib/jobs/report-worker-error"; -import { recordError } from "@/lib/jobs/worker-status"; -import { - INSIGHT_PREGENERATE_QUEUE, - INSIGHT_PREGENERATE_CRON, - type InsightPregeneratePayload, -} from "@/lib/jobs/insight-pregenerate"; -import { - MORNING_DIGEST_REFRESH_QUEUE, - runMorningDigestRefresh, - type MorningDigestRefreshPayload, -} from "@/lib/jobs/morning-digest-refresh"; -import { DATA_ARRIVAL_QUEUE, handleDataArrival } from "@/lib/jobs/data-arrival"; -import { - REACTION_LINE_QUEUE, - handleReactionLine, -} from "@/lib/jobs/reaction-line"; -import type { ReactionLineJob } from "@/lib/arrivals/reaction-line-shared"; -import { - handleWorkoutInsightGenerate, - WORKOUT_INSIGHT_GENERATE_CONCURRENCY, - WORKOUT_INSIGHT_GENERATE_QUEUE, - type WorkoutInsightGeneratePayload, -} from "@/lib/jobs/workout-insight-generate"; -import type { DataArrival } from "@/lib/arrivals/types"; -import { - DAILY_BRIEFING_QUEUE, - DAILY_BRIEFING_CRON, - runDailyBriefingTick, -} from "@/lib/jobs/daily-briefing"; -import { maybeDispatchDailyBriefing } from "@/lib/daily/daily-briefing-push"; -import { - RECOVERY_SCORE_QUEUE, - RECOVERY_SCORE_CRON, - runRecoveryScore, -} from "@/lib/jobs/recovery-score"; -import { - COACH_NUDGE_QUEUE, - COACH_NUDGE_CRON, - runCoachNudgeTick, -} from "@/lib/jobs/coach-nudge"; -import { - COACH_REMINDER_SWEEP_QUEUE, - COACH_REMINDER_SWEEP_CRON, - runCoachReminderSweep, -} from "@/lib/jobs/coach-reminder-sweep"; -import { - COACH_PLAN_REVIEW_QUEUE, - COACH_PLAN_REVIEW_CRON, - runCoachPlanReviewTick, -} from "@/lib/jobs/coach-plan-review"; -import { - MEDICATION_LOW_STOCK_QUEUE, - MEDICATION_LOW_STOCK_CRON, - runMedicationLowStockTick, -} from "@/lib/jobs/medication-low-stock"; -import { - STRESS_SCORE_QUEUE, - STRESS_SCORE_CRON, - runStressScore, -} from "@/lib/jobs/stress-score"; -import { - STRAIN_SCORE_QUEUE, - STRAIN_SCORE_CRON, - runStrainScore, -} from "@/lib/jobs/strain-score"; -import { - PERIOD_NARRATIVE_QUEUE, - PERIOD_NARRATIVE_CRON, - runPeriodNarrativeWarm, - warmOneNarrative, - type PeriodNarrativePayload, -} from "@/lib/jobs/period-narrative-warm"; -import { - COACH_MEMORY_REFRESH_QUEUE, - type CoachMemoryRefreshPayload, -} from "@/lib/ai/coach/coach-memory-shared"; -import { runCoachMemoryRefresh } from "@/lib/ai/coach/coach-memory-refresh-worker"; -import { - INSIGHT_STATUS_GENERATE_QUEUE, - INSIGHT_STATUS_GENERATE_CONCURRENCY, - type InsightStatusGeneratePayload, -} from "@/lib/jobs/insight-status-generate"; -import { withBackgroundEvent } from "@/lib/logging/background"; -import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; -import { jobBudget } from "@/lib/jobs/job-budget"; -import { lockedPass, WHOLE_PASS } from "@/lib/jobs/long-pass"; -import { heavyLane } from "@/lib/jobs/heavy-lane"; -import { INSIGHT_PREGENERATE_EXPIRE_SECONDS } from "@/lib/jobs/insight-pregenerate-shared"; -import { PERIOD_NARRATIVE_EXPIRE_SECONDS } from "@/lib/jobs/period-narrative-shared"; -import { getWorkerPrisma, workerLog } from "./shared"; -import { - createAndSchedule, - createAndWork, - insightRetryOptions, - nightlyInsightPassOptions, - retireQueues, - type QueuePolicyTable, - type ScheduleEntry, -} from "./registrar-shared"; -import { - GeneralStatusPayload, - handleGeneralStatusGenerate, - handleInsightPregenerateJob, - handleInsightStatusGenerate, -} from "./insights-handlers"; - -const GENERAL_STATUS_QUEUE = "insights-general-status"; - -const GENERAL_STATUS_CRON = "0 2 * * *"; // daily at 02:00 - -/** - * The six per-metric status crons that followed the 02:00 batch (02:05 to - * 02:30). The batch builds every card and falls each metric it could not - * cover back to that card's own path, so they only re-read cards the batch - * had just written; a user who joins later is warmed by the visit or the next - * night. Their schedules and queues are removed on an instance that still has - * them. - */ -const RETIRED_STATUS_QUEUES = [ - "insights-blood-pressure-status", - "insights-weight-status", - "insights-pulse-status", - "insights-bmi-status", - "insights-mood-status", - "insights-medication-compliance-status", -] as const; - -const allQueues = [ - GENERAL_STATUS_QUEUE, - // v1.7.0 — nightly comprehensive-insight pre-generation so the daily - // briefing is warm before the user opens /insights or the dashboard - // snapshot. Without this entry the 04:30 schedule silently no-ops. - INSIGHT_PREGENERATE_QUEUE, - // v1.8.3 — on-demand per-metric status generation. The read-only status - // route enqueues here on a cold card; without this entry pg-boss never - // provisions the queue and the enqueue silently drops. No cron schedule — - // a send-only queue driven by navigation. - INSIGHT_STATUS_GENERATE_QUEUE, - // Event-driven morning digest refresh, enqueued by the sleep-arrival - // trigger when last night's completed sleep lands. No cron schedule — a - // send-only queue driven by sleep ingest; without this entry pg-boss never - // provisions the queue and every enqueue silently drops. - MORNING_DIGEST_REFRESH_QUEUE, - // v1.31.0 — the data-arrival spine. Every salient ingest emits here; the - // worker claims the day's reaction marker and fans out to the reaction - // surfaces. No cron schedule — a send-only queue driven by ingest; without - // this entry pg-boss never provisions the queue and every emit silently - // drops (the v1.4.37 dead-queue class). - DATA_ARRIVAL_QUEUE, - // v1.31.0 — the day's single arrival reaction line. Send-only, enqueued by - // the spine worker on the pass that claimed the day's marker. Without this - // entry pg-boss never provisions the queue and every enqueue silently drops. - REACTION_LINE_QUEUE, - // v1.31.0 — the per-workout Activity Insight, dispatched by the spine when a - // workout lands. No cron schedule — nothing warms it and no read path - // triggers it, so this entry is the only thing that provisions the queue. - WORKOUT_INSIGHT_GENERATE_QUEUE, - // v1.10.0 — computed scores (WX-C / WX-E). Nightly Recovery / Stress / - // Strain compute + store. The queue MUST be registered here or pg-boss - // never provisions it and the nightly schedule silently never fires. - RECOVERY_SCORE_QUEUE, - STRESS_SCORE_QUEUE, - STRAIN_SCORE_QUEUE, - // v1.11.0 — nightly period-narrative warm + single-user warm enqueued by - // the read-only narrative GET. - PERIOD_NARRATIVE_QUEUE, - // v1.11.1 — combined Coach memory-refresh (rolling conversation summary + - // durable fact extraction), enqueued fire-and-forget from a long chat turn. - COACH_MEMORY_REFRESH_QUEUE, - // v1.15.20 — proactive Coach nudge. Without this entry the daily 05:15 - // schedule silently no-ops and no nudge ever fires. - COACH_NUDGE_QUEUE, - // v1.22 — daily Coach-reminder sweep. Without this entry the 05:20 - // schedule silently no-ops and no "remind me" reminder ever surfaces. - COACH_REMINDER_SWEEP_QUEUE, - // v1.22 — daily n-of-1 experiment review worker. Without this entry - // the 05:25 schedule silently no-ops and no experiment read-back is written. - COACH_PLAN_REVIEW_QUEUE, - // v1.16.11 — daily medication low-stock pass. Without this entry the 09:00 - // schedule silently no-ops and no low-stock alert ever fires. - MEDICATION_LOW_STOCK_QUEUE, - // Daily-briefing fallback cron. The primary morning push rides the - // sleep-arrival finalisation hook below; this queue is the fixed local-morning - // slot for the no-sleep-arrived case. Without this entry the every-15-min - // schedule silently no-ops and the fallback push never fires. - DAILY_BRIEFING_QUEUE, -]; - -const schedules: ScheduleEntry[] = [ - [GENERAL_STATUS_QUEUE, GENERAL_STATUS_CRON, nightlyInsightPassOptions], - // v1.7.0 — nightly 04:30 Europe/Berlin comprehensive-insight - // pre-generation. Budget-gated per user inside the handler. - [ - INSIGHT_PREGENERATE_QUEUE, - INSIGHT_PREGENERATE_CRON, - { - ...insightRetryOptions, - expireInSeconds: INSIGHT_PREGENERATE_EXPIRE_SECONDS, - }, - ], - // v1.10.0 — computed scores (WX-C). Nightly 04:45 Europe/Berlin - // Recovery-score compute + store, after the rollup-feeding consolidation - // + drain so the signals it reads are already folded. - [RECOVERY_SCORE_QUEUE, RECOVERY_SCORE_CRON], - // v1.10.0 — computed scores (WX-E). Nightly 04:50 Europe/Berlin - // Stress-score compute + store, after the dense intra-day retention - // drain so the HRV inputs it reads are settled. - [STRESS_SCORE_QUEUE, STRESS_SCORE_CRON], - // v1.10.0 — computed scores (WX-E). Nightly 04:55 Europe/Berlin - // Strain-score compute + store, after the recovery + stress passes so - // the nightly score writes stay ordered. - [STRAIN_SCORE_QUEUE, STRAIN_SCORE_CRON], - // v1.11.0 — nightly 05:05 Europe/Berlin period-narrative warm. The - // handler only fans out on a week (Mon) / month (1st) boundary; every - // other night is a cheap no-op. Budget-gated per user inside the runner. - [ - PERIOD_NARRATIVE_QUEUE, - PERIOD_NARRATIVE_CRON, - { - ...insightRetryOptions, - expireInSeconds: PERIOD_NARRATIVE_EXPIRE_SECONDS, - }, - ], - // v1.15.20 — daily 05:15 Europe/Berlin proactive Coach nudge, after - // the 04:45–04:55 score crons so the recovery-score trigger reads - // settled rows. Deterministic triggers only — no AI call on this path. - [COACH_NUDGE_QUEUE, COACH_NUDGE_CRON], - // v1.22 — daily 05:20 Europe/Berlin Coach-reminder sweep, just after - // the nudge tick. Flips overdue reminders to `due` + mints plan-review - // reminders from passed CoachPlan.reviewDate. In-app surface only; no push. - [COACH_REMINDER_SWEEP_QUEUE, COACH_REMINDER_SWEEP_CRON], - // v1.22 — daily 05:25 Europe/Berlin n-of-1 experiment review, just - // after the reminder sweep mints the plan-review reminders this worker reads. - // Writes the grounded before/after read-back; no provider call, no push. - [COACH_PLAN_REVIEW_QUEUE, COACH_PLAN_REVIEW_CRON], - // v1.16.11 — medication low-stock pass at 09:00 Europe/Berlin: a - // supply alert is an errand prompt, so it fires at a time the user - // can act on it. Once daily; the per-medication stamp keeps it at - // one push per threshold crossing. - [MEDICATION_LOW_STOCK_QUEUE, MEDICATION_LOW_STOCK_CRON], - // Daily-briefing fallback slot. Ticks every 15 min; the per-user - // local-hour gate inside the tick selects each user's fixed morning slot, so - // one UTC cron serves every timezone. The `push_attempts` cap suppresses it - // when the sleep-arrival finalisation push already fired this morning. - [DAILY_BRIEFING_QUEUE, DAILY_BRIEFING_CRON], -]; - -/** - * De-duplication policy per status queue. - * - * Most queues here are keyless nightly cron ticks — no `singletonKey` on any - * send — so a policy would only constrain the empty key and is deliberately - * omitted. The LLM-bound warm queues (insight pre-generate, per-metric status - * generate, period narrative, Coach memory refresh) already pass an explicit - * `singletonSeconds` on every send, which populates `singleton_on` and is - * therefore constrained by pg-boss's `job_i4` index REGARDLESS of queue policy. - * Those already de-duplicate today; changing their policy would layer a second, - * differently-shaped constraint over a working one for no gain, so they are - * left alone. - * - * That leaves exactly two queues here whose keys would otherwise be inert. - */ -const queuePolicies: QueuePolicyTable = { - // v1.31.0 — the arrival spine's day-scoped keys - // (`arrival:::`) carry the SAME requirement as the - // morning refresh below, for the same reason: a user's local date cannot be - // expressed as a wall-clock `singletonSeconds` window, so the date lives in - // the key and the policy has to honour it. Under `standard` the key - // coalesces nothing at all, and a device posting a batch per minute would - // queue a job per batch instead of one per day. - // - // `exclusive` rather than `short` because the handler is self-converging: - // it claims a durable unique row, so a second run while the first is active - // is pure duplicated work, and the next ingest re-emits anyway. - // - // The `workout` kind is the deliberate exception — it is keyed per workout - // id and DOES pass `singletonSeconds`, so it is constrained by pg-boss's - // `job_i4` index regardless of this policy. Both shapes coexist on one - // queue safely: they populate different columns. - // The reaction line's key is `reaction-line:::` — the - // same day-scoped shape, and for the same reason: a wall-clock window cannot - // express a user's local date. `exclusive` is what makes the key real, and it - // is the queue-level restatement of the durable claim the worker re-checks - // (`generatedAt`) before it spends anything. - [REACTION_LINE_QUEUE]: { - policy: "exclusive", - reason: - "Day-scoped per-kind keys carry the user's local date; under standard they coalesce nothing and a retried spine job could stack a second provider call behind the first.", - }, - [DATA_ARRIVAL_QUEUE]: { - policy: "exclusive", - reason: - "Day-scoped arrival keys carry the user's local date; under standard they coalesce nothing and a chatty ingest queues a job per batch. The worker claims a durable unique row, so a concurrent second run is pure duplicated work.", - }, - // The sleep-arrival trigger has seven write seams that can each enqueue for - // the same user and the same local date. The key is - // `morning-refresh::`, and `exclusive` turns that into a - // real at-most-once-per-user-per-local-morning contract across queued, - // active, and retry-backoff states. - // - // `singletonSeconds` is NOT the tool here and the existing comment at the - // enqueue site says so: its window is a floor() over wall-clock, which does - // not line up with a user's local date, so it would both split one morning - // across two buckets and merge two mornings into one. The local date already - // lives in the key; the policy just has to honour it. - [MORNING_DIGEST_REFRESH_QUEUE]: { - policy: "exclusive", - reason: - "Seven sleep-write seams enqueue the same user+localDate key; exclusive makes the at-most-once-per-morning contract real. A wall-clock singletonSeconds window cannot express a local date.", - }, -}; - -/** - * Register every status / insight / computed-score queue. Returns the queue - * names created (for the boot-level aggregate assertion). - */ -export async function registerStatusQueues( - boss: PgBoss, -): Promise { - await createAndSchedule(boss, allQueues, schedules, queuePolicies); - await retireQueues(boss, RETIRED_STATUS_QUEUES); - - await createAndWork( - boss, - GENERAL_STATUS_QUEUE, - { localConcurrency: 1 }, - lockedPass( - GENERAL_STATUS_QUEUE, - () => WHOLE_PASS, - heavyLane(GENERAL_STATUS_QUEUE, handleGeneralStatusGenerate), - ), - ); - // v1.7.0 — nightly comprehensive-insight pre-generation. - // - // v1.16.8 — localConcurrency 2 (was 1). The queue carries BOTH the - // scheduled 04:30 cohort walk AND the visit-triggered per-user force - // warms; with a single slot a force warm enqueued during the nightly - // batch sat behind the entire cohort and the visiting user stared at - // a cold dashboard for the duration. Two slots let one force warm run - // alongside the batch while still bounding provider-level concurrency - // (the cohort walk is itself sequential per user, and the content-hash - // gate + per-user budget gate inside the handler cover the rare - // double-tick overlap the old single slot serialised away). - await createAndWork( - boss, - INSIGHT_PREGENERATE_QUEUE, - { localConcurrency: 2 }, - // The cohort walk is one pass at a time; a forced warm is one per - // account, so it never waits behind the walk. - lockedPass( - INSIGHT_PREGENERATE_QUEUE, - (job) => - job.data?.force && job.data.userId - ? `user:${job.data.userId}` - : WHOLE_PASS, - heavyLane( - INSIGHT_PREGENERATE_QUEUE, - handleInsightPregenerateJob, - (job) => !(job.data?.force && job.data.userId), - ), - ), - ); - // Event-driven morning digest refresh. One forced comprehensive - // regeneration per user per local morning, enqueued by the sleep-arrival - // trigger. Single-flight: the enqueue-side singletonKey already collapses a - // night's samples into one job, and the handler's marker re-check makes a - // rare double-fire a no-op, so one slot is enough and bounds the provider - // concurrency the same way the nightly path does. - await createAndWork( - boss, - MORNING_DIGEST_REFRESH_QUEUE, - { localConcurrency: 1 }, - async (jobs): Promise => { - return withBackgroundEvent("job.morning_digest_refresh", async (evt) => { - // The counts the batch already decides on: one refresh per job, and - // the finalised subset that reaches the morning push seam. - let refreshed = 0; - let finalised = 0; - for (const job of jobs) { - try { - const result = await runMorningDigestRefresh( - getWorkerPrisma(), - job.data, - ); - refreshed++; - evt.addMeta( - "morning_refresh", - `${result.status}:${result.comprehensive ?? "none"}`, - ); - // The day just finalised (last night's sleep folded in), so - // this is the natural, event-driven moment to fire the calm morning - // push carrying the FINAL digest. Fully fault-isolated inside the - // seam; the opt-in / morning-window / once-per-day gates all live - // there, so a user who never opted in or is outside the window is a - // clean no-op. The fallback cron covers the no-sleep-arrived case. - if (result.status === "finalised") { - finalised++; - const pushResult = await maybeDispatchDailyBriefing( - getWorkerPrisma(), - job.data.userId, - new Date(), - ); - evt.addMeta("daily_briefing_push", pushResult); - } - } catch (err) { - recordError(); - workerLog("error", "[morning-digest-refresh] pass failed", err); - // Rethrow so the queue's retry policy re-runs the refresh; the - // marker re-check keeps a retry idempotent. - throw err; - } - } - return jobDone({ refreshed, finalised }); - }); - }, - ); - // v1.8.3 — on-demand per-metric status generation enqueued by the - // read-only status route on a cold card. Low concurrency so a first - // visit that cold-misses several cards can't saturate the Prisma pool - // or fan out an unbounded number of concurrent provider calls. - await createAndWork( - boss, - INSIGHT_STATUS_GENERATE_QUEUE, - { localConcurrency: INSIGHT_STATUS_GENERATE_CONCURRENCY }, - handleInsightStatusGenerate, - ); - // v1.10.0 — computed scores (WX-C). Nightly Recovery-score compute + - // store. The cron tick carries an empty payload; the runner iterates every - // eligible user and upserts one `COMPUTED RECOVERY_SCORE` row per scored - // day (idempotent — a re-fire overwrites in place). Single-flight so two - // ticks never double-walk the cohort. - await createAndWork( - boss, - RECOVERY_SCORE_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - try { - const summary = await runRecoveryScore(getWorkerPrisma()); - workerLog( - "info", - `[recovery-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - return jobDone({ - considered: summary.considered, - stored: summary.stored, - insufficient: summary.insufficient, - errored: summary.errored, - }); - } catch (err) { - recordError(); - workerLog("error", "[recovery-score] pass failed", err); - throw err; - } - }, - ); - // v1.15.20 — proactive Coach nudge. Single-flight; the push-attempts - // ledger caps a user at one nudge per rolling week, so an overlapping - // tick would only waste reads. Deterministic triggers, no AI call. - await createAndWork( - boss, - COACH_NUDGE_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - return withBackgroundEvent("job.coach_nudge", async (evt) => { - try { - const summary = await runCoachNudgeTick( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.coach_nudge", - result: { - candidates_scanned: summary.candidatesScanned, - dispatched: summary.dispatched, - persisted: summary.persisted, - skipped_opted_out: summary.skippedOptedOut, - skipped_no_provider: summary.skippedNoProvider, - skipped_unavailable: summary.skippedUnavailable, - skipped_recent_nudge: summary.skippedRecentNudge, - skipped_recent_engagement: summary.skippedRecentEngagement, - skipped_no_trigger: summary.skippedNoTrigger, - skipped_no_channel: summary.skippedNoChannel, - failed: summary.failed, - }, - }); - // A per-user failure never fails this pass — it rides out as the - // `failed` count, exactly as it does in the wide event. - return jobDone({ - candidates_scanned: summary.candidatesScanned, - dispatched: summary.dispatched, - persisted: summary.persisted, - failed: summary.failed, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }, - ); - // v1.16.11 — medication low-stock pass. Single-flight; the - // per-medication stamp makes a re-fire idempotent (already-notified - // crossings skip), so an overlapping tick would only waste reads. - await createAndWork( - boss, - MEDICATION_LOW_STOCK_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - return withBackgroundEvent("job.medication_low_stock", async (evt) => { - try { - const summary = await runMedicationLowStockTick( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.medication_low_stock", - result: { - users_scanned: summary.usersScanned, - skipped_threshold_off: summary.skippedThresholdOff, - medications_evaluated: summary.medicationsEvaluated, - notified: summary.notified, - rearmed: summary.rearmed, - skipped_already_notified: summary.skippedAlreadyNotified, - skipped_above_threshold: summary.skippedAboveThreshold, - skipped_no_runway: summary.skippedNoRunway, - skipped_no_channel: summary.skippedNoChannel, - failed: summary.failed, - }, - }); - // A per-medication failure never fails this pass — it rides out as - // the `failed` count. - return jobDone({ - users_scanned: summary.usersScanned, - medications_evaluated: summary.medicationsEvaluated, - notified: summary.notified, - rearmed: summary.rearmed, - failed: summary.failed, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }, - ); - // v1.31.0 — the data-arrival spine. Single-flight: the worker's whole job is - // to claim a durable unique row, so two slots would only race each other for - // a claim exactly one of them can win. It makes no provider call — a - // module-graph test pins that — so one slot costs nothing in throughput. - await createAndWork( - boss, - DATA_ARRIVAL_QUEUE, - { localConcurrency: 1 }, - handleDataArrival, - ); - // v1.31.0 — the arrival reaction line. Single-flight: this is the spine's - // ONE provider call, and a second concurrent slot would only race the same - // durable `generatedAt` claim exactly one of them can win. - await createAndWork( - boss, - REACTION_LINE_QUEUE, - { localConcurrency: 1 }, - handleReactionLine, - ); - // v1.31.0 — the per-workout Activity Insight. Serial: this IS the provider - // path the spine deliberately does not walk, so it stays off the request pool - // and cannot fan out concurrent completions when several workouts land in one - // sync. - await createAndWork( - boss, - WORKOUT_INSIGHT_GENERATE_QUEUE, - { localConcurrency: WORKOUT_INSIGHT_GENERATE_CONCURRENCY }, - handleWorkoutInsightGenerate, - ); - // Daily-briefing fallback slot. Single-flight; the `push_attempts` - // frequency cap makes an overlapping tick a no-op (a second same-day attempt - // is suppressed), and the tick only loads a digest for opted-in users at - // their exact local fallback hour. No provider call — it reads the cached - // digest and dispatches the deterministic line + cached lead. - await createAndWork( - boss, - DAILY_BRIEFING_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - return withBackgroundEvent("job.daily_briefing", async (evt) => { - try { - const summary = await runDailyBriefingTick( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.daily_briefing", - result: { - candidates_scanned: summary.candidatesScanned, - in_slot: summary.inSlot, - sent: summary.sent, - suppressed_frequency: summary.suppressedFrequency, - no_digest: summary.noDigest, - opted_out: summary.optedOut, - no_channel: summary.noChannel, - outside_window: summary.outsideWindow, - failed: summary.failed, - }, - }); - // Most ticks land outside every user's slot; that is the tick doing - // its job, so a zero `sent` is a success with zero counts. - return jobDone({ - candidates_scanned: summary.candidatesScanned, - in_slot: summary.inSlot, - sent: summary.sent, - failed: summary.failed, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }, - ); - // v1.10.0 — computed scores (WX-E). Nightly Stress-score (HRV-derived - // proxy) compute + store. Single-flight so two ticks never double-walk - // the cohort. The runner iterates every eligible user and upserts one - // `COMPUTED STRESS_SCORE` row per scored day (idempotent — a re-fire - // overwrites in place). - await createAndWork( - boss, - STRESS_SCORE_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - try { - const summary = await runStressScore(getWorkerPrisma()); - workerLog( - "info", - `[stress-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - return jobDone({ - considered: summary.considered, - stored: summary.stored, - insufficient: summary.insufficient, - errored: summary.errored, - }); - } catch (err) { - recordError(); - workerLog("error", "[stress-score] pass failed", err); - throw err; - } - }, - ); - // v1.10.0 — computed scores (WX-E). Nightly Strain-score (Banister TRIMP - // cardio-load) compute + store. Single-flight; upserts one `COMPUTED - // STRAIN_SCORE` row per scored day (idempotent). - await createAndWork( - boss, - STRAIN_SCORE_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - try { - const summary = await runStrainScore(getWorkerPrisma()); - workerLog( - "info", - `[strain-score] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, - ); - return jobDone({ - considered: summary.considered, - stored: summary.stored, - insufficient: summary.insufficient, - errored: summary.errored, - }); - } catch (err) { - recordError(); - workerLog("error", "[strain-score] pass failed", err); - throw err; - } - }, - ); - // v1.11.0 — period-narrative warm. A scheduled tick (no `userId`) runs the - // boundary-gated nightly fan-out; a `userId` payload runs a single-user warm - // enqueued by the read-only GET on a cold/stale read. Single-flight so two - // ticks never double-walk the cohort; the per-user budget gate covers the - // fan-out and the enqueue `singletonKey` covers the single-user path. - await createAndWork( - boss, - PERIOD_NARRATIVE_QUEUE, - { localConcurrency: 1 }, - lockedPass( - PERIOD_NARRATIVE_QUEUE, - (job) => (job.data?.userId ? `user:${job.data.userId}` : WHOLE_PASS), - heavyLane( - PERIOD_NARRATIVE_QUEUE, - async (jobs): Promise => { - // Two modes on one queue, so the outcome counts them separately: a - // non-boundary night is a scheduled pass that generated nothing, which - // is a success with zero counts, not an idle queue. - let singleUser = 0; - let scheduled = 0; - let generated = 0; - let failed = 0; - let stoppedEarly = false; - for (const job of jobs) { - try { - if (job.data?.userId) { - await warmOneNarrative(job.data); - singleUser++; - } else { - const summary = await runPeriodNarrativeWarm( - getWorkerPrisma(), - { - shouldStop: jobBudget(jobs), - }, - ); - scheduled++; - generated += summary.generated; - failed += summary.failed; - stoppedEarly ||= summary.stoppedEarly; - workerLog( - "info", - `[period-narrative] periods=${summary.periods.join(",") || "none"} total=${summary.total} generated=${summary.generated} cached=${summary.cached} skipped=${summary.skipped} insufficient=${summary.insufficient} failed=${summary.failed} budget=${summary.budgetBlocked}`, - ); - } - } catch (err) { - recordError(); - await reportWorkerError(PERIOD_NARRATIVE_QUEUE, err, { - mode: job.data?.userId ? "single-user" : "scheduled", - }); - throw err; - } - } - return jobDone({ - single_user: singleUser, - scheduled, - generated, - failed, - stopped_early: stoppedEarly, - }); - }, - (job) => !job.data?.userId, - ), - ), - ); - // v1.11.1 — combined Coach memory refresh: rolling conversation summary + - // durable fact extraction for one long conversation. localConcurrency 1 so a - // burst of long-conversation turns can't fan out concurrent provider calls; - // each step is budget-gated inside runStatusCompletion and fault-isolated. - await createAndWork( - boss, - COACH_MEMORY_REFRESH_QUEUE, - { localConcurrency: 1 }, - async (jobs): Promise => { - // An incomplete payload is skipped rather than failed, so it has to be - // countable — otherwise a queue full of them reads as a clean run. - let refreshed = 0; - let skippedIncomplete = 0; - for (const job of jobs) { - if (!job.data?.conversationId || !job.data?.userId) { - skippedIncomplete++; - continue; - } - try { - await runCoachMemoryRefresh(job.data); - refreshed++; - } catch (err) { - recordError(); - workerLog("error", "[coach-memory-refresh] failed", err); - throw err; - } - } - return jobDone({ - refreshed, - skipped_incomplete: skippedIncomplete, - }); - }, - ); - - // v1.22 — daily Coach-reminder sweep. Single-flight; surfacing an - // overdue reminder into the user's Coach thread and minting plan-review - // reminders are both idempotent across re-fires (an already-surfaced - // reminder is not re-written; a plan whose reviewDate was cleared is not - // re-minted). No provider call, no push — the message body is a - // deterministic localized template around the user's own note. - await createAndWork( - boss, - COACH_REMINDER_SWEEP_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - return withBackgroundEvent("job.coach_reminder_sweep", async (evt) => { - try { - const summary = await runCoachReminderSweep( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.coach_reminder_sweep", - result: { - reminders_due: summary.remindersDue, - plan_reviews_minted: summary.planReviewsMinted, - context_surfaced: summary.contextSurfaced, - nag_dismissed: summary.nagDismissed, - errored: summary.errored, - }, - }); - return jobDone({ - reminders_due: summary.remindersDue, - plan_reviews_minted: summary.planReviewsMinted, - context_surfaced: summary.contextSurfaced, - nag_dismissed: summary.nagDismissed, - errored: summary.errored, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }, - ); - - // v1.22 — n-of-1 experiment review worker: reads the plan-review - // reminders the sweep minted and writes the grounded before/after read-back - // into the plan's encrypted outcome column, flipping it to `reviewed`. - // Idempotent (a reviewed plan is not re-processed). No provider call, no push. - await createAndWork( - boss, - COACH_PLAN_REVIEW_QUEUE, - { localConcurrency: 1 }, - async (): Promise => { - return withBackgroundEvent("job.coach_plan_review", async (evt) => { - try { - const summary = await runCoachPlanReviewTick( - getWorkerPrisma(), - new Date(), - ); - evt.setBackground({ - task_name: "job.coach_plan_review", - result: { - reviewed: summary.reviewed, - insufficient: summary.insufficient, - errored: summary.errored, - }, - }); - return jobDone({ - reviewed: summary.reviewed, - insufficient: summary.insufficient, - errored: summary.errored, - }); - } catch (err) { - evt.setError(err); - recordError(); - throw err; - } - }); - }, - ); - - return allQueues; -} diff --git a/src/lib/jobs/reminder/rollup-handlers.ts b/src/lib/jobs/reminder/rollup-handlers.ts new file mode 100644 index 000000000..74c765d02 --- /dev/null +++ b/src/lib/jobs/reminder/rollup-handlers.ts @@ -0,0 +1,582 @@ +/** + * Handlers of the rollup and consolidation queues: the measurement and mood + * rollup folds, the consolidation and repair passes, the supplement nutrient + * recompute, and the nightly drain that runs the cumulative, daily-mean and + * dense-retention passes on one tick. The registry binds them + * (`registry/rollup.ts`) and loads this module on the first job. + */ +import type { Job, PgBoss } from "pg-boss"; + +import { + DRAIN_CONTINUATION_DELAY_SECONDS, + nextDrainContinuation, +} from "@/lib/jobs/drain-continuation"; +import { + DRAIN_CUMULATIVE_QUEUE, + type DrainCumulativePayload, +} from "@/lib/jobs/drain-cumulative-queue"; +import { runDenseIntradayHourlyRebuildForUser } from "@/lib/jobs/dense-intraday-hourly-rebuild"; +import type { DenseIntradayHourlyRebuildPayload } from "@/lib/jobs/dense-intraday-hourly-rebuild"; +import { + DENSE_INTRADAY_RETENTION_ENABLED, + runDenseIntradayRetentionForUser, + type DenseIntradayRetentionPayload, +} from "@/lib/jobs/dense-intraday-retention"; +import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { + runMeanConsolidationForUser, + type MeanConsolidationPayload, +} from "@/lib/jobs/mean-consolidation"; +import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { + runStepConsolidationForUser, + type StepConsolidationPayload, +} from "@/lib/jobs/step-consolidation"; +import { + runStepConsolidationRepairForUser, + type StepConsolidationRepairPayload, +} from "@/lib/jobs/step-consolidation-repair"; +import { recordError } from "@/lib/jobs/worker-status"; +import { + consolidateDailyMean, + MEAN_CONSOLIDATION_CUTOFF_HOURS, +} from "@/lib/measurements/consolidate-daily-mean"; +import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention"; +import { + drainPerSampleCumulative, + DRAIN_CUMULATIVE_CUTOFF_HOURS, +} from "@/lib/measurements/drain-per-sample-cumulative"; +import { + runSupplementNutrientRecompute, + type SupplementNutrientRecomputePayload, +} from "@/lib/medications/ingredients/recompute-job"; +import { + runCumulativePrRederivationForUser, + type CumulativePrRederivePayload, +} from "@/lib/personal-records/cumulative-pr-rederivation"; +import { + recomputeUserRollups, + type RollupFullBackfillPayload, + type RollupRecomputePayload, +} from "@/lib/rollups/measurement-rollups"; +import { + recomputeUserMoodRollups, + type MoodRollupFullBackfillPayload, + type MoodRollupRecomputePayload, +} from "@/lib/rollups/mood-rollups"; + +import { cronIsTheRetry } from "./registrar-shared"; +import { getWorkerPrisma, workerLog } from "./shared"; + +/** + * Folds the WEEK / MONTH / YEAR buckets the write-path hooks enqueue; the DAY + * bucket is recomputed by the hook itself. No per-bucket isolation on + * purpose: a recompute that throws fails the job, so reaching the end means + * every bucket in the batch was folded. + */ +export async function handleRollupRecompute( + jobs: Job[], +): Promise { + for (const job of jobs) { + const payload = job.data; + await recomputeUserRollups(payload.userId, { + types: [payload.type], + granularities: [payload.granularity], + from: new Date(payload.from), + to: new Date(payload.to), + }); + } + return jobDone({ buckets_recomputed: jobs.length }); +} + +/** + * The full fold of one uncovered account across the default window and every + * granularity. Each granularity commits on its own, so a run stopped on its + * budget throws and the retry folds the rest. + */ +export async function handleRollupFullBackfill( + jobs: Job[], +): Promise { + const shouldStop = jobBudget(jobs); + let rowsUpsertedTotal = 0; + for (const job of jobs) { + const { userId } = job.data; + const { rowsUpserted, durationMs, stopped } = await recomputeUserRollups( + userId, + { shouldStop }, + ); + if (stopped) { + throw new Error( + `rollup-full-backfill for user ${userId} stopped on its job budget; the retry folds the rest`, + ); + } + rowsUpsertedTotal += rowsUpserted; + workerLog( + "info", + `[rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, + ); + } + return jobDone({ + users: jobs.length, + rows_upserted: rowsUpsertedTotal, + }); +} + +/** Collapses legacy granular step rows into one daily total per day. */ +export async function handleStepConsolidation( + jobs: Job[], +): Promise { + let daysTotal = 0; + let legacyRowsTotal = 0; + for (const job of jobs) { + const { userId } = job.data; + const { daysConsolidated, legacyRowsSoftDeleted } = + await runStepConsolidationForUser(userId); + daysTotal += daysConsolidated; + legacyRowsTotal += legacyRowsSoftDeleted; + workerLog( + "info", + `[step-consolidation] user=${userId} days=${daysConsolidated} legacyRowsSoftDeleted=${legacyRowsSoftDeleted}`, + ); + } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + legacy_rows_soft_deleted: legacyRowsTotal, + }); +} + +/** + * Resurrects the provider daily-total step rows the earlier consolidation + * swept and removes the shadow manual totals it minted. Per-day failures are + * the repair's own and ride out as a count. + */ +export async function handleStepConsolidationRepair( + jobs: Job[], +): Promise { + let rowsResurrected = 0; + let wedgeSkipped = 0; + let manualMintsRemoved = 0; + let daysRecomputed = 0; + let failures = 0; + for (const job of jobs) { + const { userId } = job.data; + const summary = await runStepConsolidationRepairForUser(userId); + rowsResurrected += summary.rowsResurrected; + wedgeSkipped += summary.wedgeSkipped; + manualMintsRemoved += summary.manualMintsRemoved; + daysRecomputed += summary.daysRecomputed; + failures += summary.failures; + workerLog( + "info", + `[step-consolidation-repair] user=${userId} resurrected=${summary.rowsResurrected} wedgeSkipped=${summary.wedgeSkipped} manualMintsRemoved=${summary.manualMintsRemoved} daysRecomputed=${summary.daysRecomputed} failures=${summary.failures}`, + ); + } + return jobDone({ + users: jobs.length, + rows_resurrected: rowsResurrected, + wedge_skipped: wedgeSkipped, + manual_mints_removed: manualMintsRemoved, + days_recomputed: daysRecomputed, + failures, + }); +} + +/** One account's supplement doses recomputed into the day's micronutrients. */ +export async function handleSupplementNutrientRecompute( + jobs: Job[], +): Promise { + let rowsWritten = 0; + for (const job of jobs) { + const result = await runSupplementNutrientRecompute(job.data); + rowsWritten += result.rowsWritten; + } + return jobDone({ users: jobs.length, rows_written: rowsWritten }); +} + +/** + * Deletes an inflated cumulative-type personal record and re-runs detection + * silently, so the honestly derived best takes its place. + */ +export async function handleCumulativePrRederive( + jobs: Job[], +): Promise { + let rowsDeleted = 0; + let rowsReinserted = 0; + for (const job of jobs) { + const { userId } = job.data; + const summary = await runCumulativePrRederivationForUser(userId); + rowsDeleted += summary.rowsDeleted; + rowsReinserted += summary.rowsReinserted; + workerLog( + "info", + `[cumulative-pr-rederive] user=${userId} rowsDeleted=${summary.rowsDeleted} rowsReinserted=${summary.rowsReinserted}`, + ); + } + return jobDone({ + users: jobs.length, + rows_deleted: rowsDeleted, + rows_reinserted: rowsReinserted, + }); +} + +/** + * Collapses each completed day of per-sample high-frequency mean rows to its + * mean. Out of budget is a completed run: the folded days are committed and + * the nightly tick carries on from the first day this run did not reach. + */ +export async function handleMeanConsolidation( + jobs: Job[], +): Promise { + const shouldStop = jobBudget(jobs); + let daysTotal = 0; + let perSampleRowsTotal = 0; + let stoppedEarly = false; + for (const job of jobs) { + const { userId } = job.data; + const summary = await runMeanConsolidationForUser(userId, shouldStop); + const { daysConsolidated, perSampleRowsSoftDeleted } = summary; + stoppedEarly ||= summary.stoppedEarly; + daysTotal += daysConsolidated; + perSampleRowsTotal += perSampleRowsSoftDeleted; + workerLog( + "info", + `[mean-consolidation] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted}`, + ); + } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + per_sample_rows_soft_deleted: perSampleRowsTotal, + stopped_early: stoppedEarly, + }); +} + +/** + * Folds one account's dense-tier (HRV / HR / SpO2) samples older than the + * retention window to hourly means, keeping the in-window intra-day shape the + * Stress engine reads. Out of budget completes the job; the nightly tick + * continues from the first day this run did not reach. + */ +export async function handleDenseIntradayRetention( + jobs: Job[], +): Promise { + const shouldStop = jobBudget(jobs); + let daysTotal = 0; + let perSampleRowsTotal = 0; + let derivedRestingRowsTotal = 0; + let stoppedEarly = false; + for (const job of jobs) { + const { userId } = job.data; + try { + const summary = await runDenseIntradayRetentionForUser( + userId, + shouldStop, + ); + const { + daysConsolidated, + perSampleRowsSoftDeleted, + derivedRestingRowsUpserted, + } = summary; + stoppedEarly ||= summary.stoppedEarly; + daysTotal += daysConsolidated; + perSampleRowsTotal += perSampleRowsSoftDeleted; + derivedRestingRowsTotal += derivedRestingRowsUpserted; + workerLog( + "info", + `[dense-intraday-retention] user=${userId} days=${daysConsolidated} perSampleRowsSoftDeleted=${perSampleRowsSoftDeleted} derivedRestingRowsUpserted=${derivedRestingRowsUpserted}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[dense-intraday-retention] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + users: jobs.length, + days_consolidated: daysTotal, + per_sample_rows_soft_deleted: perSampleRowsTotal, + derived_resting_rows_upserted: derivedRestingRowsTotal, + stopped_early: stoppedEarly, + }); +} + +/** + * Reconstructs hourly means from the tombstones behind a pre-hourly daily row + * and retires the daily row. Out of budget fails the job so pg-boss retries + * (the boot send allows three), and the retry starts at the first day not yet + * rebuilt; the queue is `exclusive`, so it cannot hand itself on while active. + */ +export async function handleDenseIntradayHourlyRebuild( + jobs: Job[], +): Promise { + const shouldStop = jobBudget(jobs); + let stoppedEarly = false; + let daysRebuiltTotal = 0; + let hourlyRowsTotal = 0; + let dailyRowsRetiredTotal = 0; + let daysSkippedTotal = 0; + for (const job of jobs) { + const { userId } = job.data; + try { + const summary = await runDenseIntradayHourlyRebuildForUser( + userId, + shouldStop, + ); + const { + daysRebuilt, + hourlyRowsUpserted, + dailyRowsRetired, + daysSkippedNoTombstones, + } = summary; + stoppedEarly ||= summary.stoppedEarly; + daysRebuiltTotal += daysRebuilt; + hourlyRowsTotal += hourlyRowsUpserted; + dailyRowsRetiredTotal += dailyRowsRetired; + daysSkippedTotal += daysSkippedNoTombstones; + workerLog( + "info", + `[dense-intraday-hourly-rebuild] user=${userId} daysRebuilt=${daysRebuilt} hourlyRowsUpserted=${hourlyRowsUpserted} dailyRowsRetired=${dailyRowsRetired} daysSkippedNoTombstones=${daysSkippedNoTombstones}`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[dense-intraday-hourly-rebuild] user=${userId} failed`, + err, + ); + throw err; + } + } + const did = { + users: jobs.length, + stopped_early: stoppedEarly, + days_rebuilt: daysRebuiltTotal, + hourly_rows_upserted: hourlyRowsTotal, + daily_rows_retired: dailyRowsRetiredTotal, + days_skipped_no_tombstones: daysSkippedTotal, + }; + if (stoppedEarly) { + return jobFailed( + "stopped before every day was rebuilt; the retry resumes", + undefined, + did, + ); + } + return jobDone(did); +} + +/** Folds the WEEK / MONTH / YEAR mood buckets the mood-entry hooks enqueue. */ +export async function handleMoodRollupRecompute( + jobs: Job[], +): Promise { + for (const job of jobs) { + const payload = job.data; + await recomputeUserMoodRollups(payload.userId, { + granularities: [payload.granularity], + from: new Date(payload.from), + to: new Date(payload.to), + }); + } + return jobDone({ buckets_recomputed: jobs.length }); +} + +/** + * The full mood fold of one account. Delete-then-refold, so the backfill + * cleans up any row minted under an older keying; the brief empty window + * reads correctly through the live fallback. + */ +export async function handleMoodRollupFullBackfill( + jobs: Job[], +): Promise { + let rowsUpsertedTotal = 0; + for (const job of jobs) { + const { userId } = job.data; + try { + const { rowsUpserted, durationMs } = await recomputeUserMoodRollups( + userId, + { replace: true }, + ); + rowsUpsertedTotal += rowsUpserted; + workerLog( + "info", + `[mood-rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, + ); + } catch (err) { + recordError(); + workerLog( + "error", + `[mood-rollup-full-backfill] user=${userId} failed`, + err, + ); + throw err; + } + } + return jobDone({ + users: jobs.length, + rows_upserted: rowsUpsertedTotal, + }); +} + +/** + * The nightly drain: the cumulative (SUM) drain, the daily-mean drain and the + * dense intra-day retention drain, in turn, on one budget. They are + * independent passes, so one failing does not stop the others; the first + * failure is reported once the tick is over and fails the job. A tick that ran + * out of budget hands the rest to a bounded follow-up. + */ +export function drainCumulativeHandler( + boss: PgBoss, +): (jobs: Job[]) => Promise { + return async (jobs) => { + const shouldStop = jobBudget(jobs); + let stoppedEarly = false; + let firstFailure: { reason: string; cause: unknown } | undefined; + let passesFailed = 0; + let bucketsCollapsed = 0; + let perSampleRowsDeleted = 0; + let meanDaysConsolidated = 0; + let meanRowsSoftDeleted = 0; + let denseDaysConsolidated = 0; + let denseRowsSoftDeleted = 0; + let denseSkipped = false; + + for (const job of jobs) { + try { + const summary = await drainPerSampleCumulative(getWorkerPrisma(), { + dryRun: false, + cutoffHours: DRAIN_CUMULATIVE_CUTOFF_HOURS, + shouldStop, + log: (line) => workerLog("info", line), + }); + stoppedEarly ||= summary.stoppedEarly; + bucketsCollapsed += summary.totals.bucketsCollapsed; + perSampleRowsDeleted += summary.totals.perSampleRowsDeleted; + workerLog( + "info", + `[drain-cumulative] triggeredAt=${job.data.triggeredAt} usersScanned=${summary.totals.usersScanned} bucketsCollapsed=${summary.totals.bucketsCollapsed} perSampleRowsDeleted=${summary.totals.perSampleRowsDeleted} dailyRowsUpserted=${summary.totals.dailyRowsUpserted}`, + ); + } catch (err) { + recordError(); + await reportWorkerError("drain-cumulative", err); + passesFailed++; + firstFailure ??= { reason: "cumulative drain failed", cause: err }; + } + + // The daily-mean drain rides the same tick, so spot samples between + // reboots do not accumulate raw; the 36-hour grace keeps today's + // in-flight watch syncs raw. + try { + const meanSummary = await consolidateDailyMean(getWorkerPrisma(), { + dryRun: false, + cutoffHours: MEAN_CONSOLIDATION_CUTOFF_HOURS, + shouldStop, + log: (line) => workerLog("info", line), + }); + stoppedEarly ||= meanSummary.stoppedEarly; + meanDaysConsolidated += meanSummary.totals.daysConsolidated; + meanRowsSoftDeleted += meanSummary.totals.perSampleRowsSoftDeleted; + workerLog( + "info", + `[mean-consolidation] triggeredAt=${job.data.triggeredAt} usersScanned=${meanSummary.totals.usersScanned} daysConsolidated=${meanSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${meanSummary.totals.perSampleRowsSoftDeleted} dailyRowsUpserted=${meanSummary.totals.dailyRowsUpserted} daysFailed=${meanSummary.totals.daysFailed}`, + ); + } catch (err) { + recordError(); + await reportWorkerError("mean-consolidation", err, { + tick: "nightly", + }); + passesFailed++; + firstFailure ??= { reason: "mean consolidation failed", cause: err }; + } + + // The dense intra-day retention drain: HRV and pulse only, keeping the + // last DENSE_INTRADAY_RETENTION_DAYS raw for the Stress engine. These + // types are never in the destructive daily-mean allowlist. + try { + if (!DENSE_INTRADAY_RETENTION_ENABLED) { + denseSkipped = true; + workerLog( + "info", + "[dense-intraday-retention] disabled via DENSE_INTRADAY_RETENTION_ENABLED — skipping the nightly walk (operator kill-switch)", + ); + } else { + const denseSummary = await runDenseIntradayRetention( + getWorkerPrisma(), + { + dryRun: false, + shouldStop, + log: (line) => workerLog("info", line), + }, + ); + stoppedEarly ||= denseSummary.stoppedEarly; + denseDaysConsolidated += denseSummary.totals.daysConsolidated; + denseRowsSoftDeleted += denseSummary.totals.perSampleRowsSoftDeleted; + workerLog( + "info", + `[dense-intraday-retention] triggeredAt=${job.data.triggeredAt} usersScanned=${denseSummary.totals.usersScanned} daysConsolidated=${denseSummary.totals.daysConsolidated} perSampleRowsSoftDeleted=${denseSummary.totals.perSampleRowsSoftDeleted} hourlyRowsUpserted=${denseSummary.totals.hourlyRowsUpserted} dailyRowsRetired=${denseSummary.totals.dailyRowsRetired}`, + ); + } + } catch (err) { + recordError(); + await reportWorkerError("dense-intraday-retention", err, { + tick: "nightly", + }); + passesFailed++; + firstFailure ??= { reason: "dense retention failed", cause: err }; + } + } + + const continuation = Math.max( + 0, + ...jobs.map((job) => job.data.continuation ?? 0), + ); + const next = nextDrainContinuation({ + stoppedEarly, + daysFolded: + bucketsCollapsed + meanDaysConsolidated + denseDaysConsolidated, + continuation, + }); + let continued = false; + if (next !== null) { + try { + const followUp: DrainCumulativePayload = { + triggeredAt: new Date().toISOString(), + continuation: next, + }; + continued = + (await boss.send(DRAIN_CUMULATIVE_QUEUE, followUp, { + ...cronIsTheRetry, + startAfter: DRAIN_CONTINUATION_DELAY_SECONDS, + })) !== null; + } catch (err) { + // The next nightly tick picks the work up regardless; a lost + // follow-up only delays it. + workerLog("error", "[drain-cumulative] follow-up send failed", err); + } + } + + const did = { + stopped_early: stoppedEarly, + continuation, + continued, + passes_failed: passesFailed, + buckets_collapsed: bucketsCollapsed, + per_sample_rows_deleted: perSampleRowsDeleted, + mean_days_consolidated: meanDaysConsolidated, + mean_rows_soft_deleted: meanRowsSoftDeleted, + dense_days_consolidated: denseDaysConsolidated, + dense_rows_soft_deleted: denseRowsSoftDeleted, + dense_skipped: denseSkipped, + }; + + if (firstFailure) { + return jobFailed(firstFailure.reason, firstFailure.cause, did); + } + return jobDone(did); + }; +} diff --git a/src/lib/jobs/reminder/status-handlers.ts b/src/lib/jobs/reminder/status-handlers.ts new file mode 100644 index 000000000..501bf2601 --- /dev/null +++ b/src/lib/jobs/reminder/status-handlers.ts @@ -0,0 +1,394 @@ +/** + * Handlers of the status, score and Coach queues that are not a module's own + * export: the morning digest refresh, the nightly computed scores, the Coach + * ticks, the low-stock pass, the daily-briefing fallback, the period-narrative + * warm and the Coach memory refresh. The registry binds them + * (`registry/status.ts`) and loads this module on the first job. + */ +import type { Job } from "pg-boss"; + +import type { CoachMemoryRefreshPayload } from "@/lib/ai/coach/coach-memory-shared"; +import { runCoachMemoryRefresh } from "@/lib/ai/coach/coach-memory-refresh-worker"; +import { maybeDispatchDailyBriefing } from "@/lib/daily/daily-briefing-push"; +import { runCoachNudgeTick } from "@/lib/jobs/coach-nudge"; +import { runCoachPlanReviewTick } from "@/lib/jobs/coach-plan-review"; +import { runCoachReminderSweep } from "@/lib/jobs/coach-reminder-sweep"; +import { runDailyBriefingTick } from "@/lib/jobs/daily-briefing"; +import { jobBudget } from "@/lib/jobs/job-budget"; +import { jobDone, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { runMedicationLowStockTick } from "@/lib/jobs/medication-low-stock"; +import { + runMorningDigestRefresh, + type MorningDigestRefreshPayload, +} from "@/lib/jobs/morning-digest-refresh"; +import { + runPeriodNarrativeWarm, + warmOneNarrative, + type PeriodNarrativePayload, +} from "@/lib/jobs/period-narrative-warm"; +import { PERIOD_NARRATIVE_QUEUE } from "@/lib/jobs/period-narrative-queue"; +import { runRecoveryScore } from "@/lib/jobs/recovery-score"; +import { reportWorkerError } from "@/lib/jobs/report-worker-error"; +import { runStrainScore } from "@/lib/jobs/strain-score"; +import { runStressScore } from "@/lib/jobs/stress-score"; +import { recordError } from "@/lib/jobs/worker-status"; +import { withBackgroundEvent } from "@/lib/logging/background"; + +import { getWorkerPrisma, workerLog } from "./shared"; + +/** + * One forced comprehensive regeneration per user per local morning, enqueued + * by the sleep-arrival trigger. A finalised day is the event-driven moment for + * the morning push carrying the final digest; the push seam owns its own + * opt-in, window and once-per-day gates, and the fallback cron covers the + * mornings no sleep arrives. + */ +export async function handleMorningDigestRefresh( + jobs: Job[], +): Promise { + return withBackgroundEvent("job.morning_digest_refresh", async (evt) => { + // The counts the batch already decides on: one refresh per job, and + // the finalised subset that reaches the morning push seam. + let refreshed = 0; + let finalised = 0; + for (const job of jobs) { + try { + const result = await runMorningDigestRefresh( + getWorkerPrisma(), + job.data, + ); + refreshed++; + evt.addMeta( + "morning_refresh", + `${result.status}:${result.comprehensive ?? "none"}`, + ); + if (result.status === "finalised") { + finalised++; + const pushResult = await maybeDispatchDailyBriefing( + getWorkerPrisma(), + job.data.userId, + new Date(), + ); + evt.addMeta("daily_briefing_push", pushResult); + } + } catch (err) { + recordError(); + workerLog("error", "[morning-digest-refresh] pass failed", err); + // Rethrow so the queue's retry policy re-runs the refresh; the + // marker re-check keeps a retry idempotent. + throw err; + } + } + return jobDone({ refreshed, finalised }); + }); +} + +interface ScoreSummary { + considered: number; + stored: number; + insufficient: number; + errored: number; +} + +/** + * A nightly computed-score pass: the cron tick carries an empty payload and + * the runner walks every eligible user, upserting one `COMPUTED` row per + * scored day (a re-fire overwrites in place). + */ +function scorePass( + name: string, + run: () => Promise, +): () => Promise { + return async () => { + try { + const summary = await run(); + workerLog( + "info", + `[${name}] considered=${summary.considered} stored=${summary.stored} insufficient=${summary.insufficient} errored=${summary.errored}`, + ); + return jobDone({ + considered: summary.considered, + stored: summary.stored, + insufficient: summary.insufficient, + errored: summary.errored, + }); + } catch (err) { + recordError(); + workerLog("error", `[${name}] pass failed`, err); + throw err; + } + }; +} + +export const handleRecoveryScore = scorePass("recovery-score", () => + runRecoveryScore(getWorkerPrisma()), +); +export const handleStressScore = scorePass("stress-score", () => + runStressScore(getWorkerPrisma()), +); +export const handleStrainScore = scorePass("strain-score", () => + runStrainScore(getWorkerPrisma()), +); + +/** + * The proactive Coach nudge. Deterministic triggers, no AI call; a per-user + * failure never fails the pass, it rides out as the `failed` count. + */ +export async function handleCoachNudge(): Promise { + return withBackgroundEvent("job.coach_nudge", async (evt) => { + try { + const summary = await runCoachNudgeTick(getWorkerPrisma(), new Date()); + evt.setBackground({ + task_name: "job.coach_nudge", + result: { + candidates_scanned: summary.candidatesScanned, + dispatched: summary.dispatched, + persisted: summary.persisted, + skipped_opted_out: summary.skippedOptedOut, + skipped_no_provider: summary.skippedNoProvider, + skipped_unavailable: summary.skippedUnavailable, + skipped_recent_nudge: summary.skippedRecentNudge, + skipped_recent_engagement: summary.skippedRecentEngagement, + skipped_no_trigger: summary.skippedNoTrigger, + skipped_no_channel: summary.skippedNoChannel, + failed: summary.failed, + }, + }); + return jobDone({ + candidates_scanned: summary.candidatesScanned, + dispatched: summary.dispatched, + persisted: summary.persisted, + failed: summary.failed, + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); +} + +/** + * The medication low-stock pass. The per-medication stamp makes a re-fire + * idempotent; a per-medication failure rides out as the `failed` count. + */ +export async function handleMedicationLowStock(): Promise { + return withBackgroundEvent("job.medication_low_stock", async (evt) => { + try { + const summary = await runMedicationLowStockTick( + getWorkerPrisma(), + new Date(), + ); + evt.setBackground({ + task_name: "job.medication_low_stock", + result: { + users_scanned: summary.usersScanned, + skipped_threshold_off: summary.skippedThresholdOff, + medications_evaluated: summary.medicationsEvaluated, + notified: summary.notified, + rearmed: summary.rearmed, + skipped_already_notified: summary.skippedAlreadyNotified, + skipped_above_threshold: summary.skippedAboveThreshold, + skipped_no_runway: summary.skippedNoRunway, + skipped_no_channel: summary.skippedNoChannel, + failed: summary.failed, + }, + }); + return jobDone({ + users_scanned: summary.usersScanned, + medications_evaluated: summary.medicationsEvaluated, + notified: summary.notified, + rearmed: summary.rearmed, + failed: summary.failed, + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); +} + +/** + * The daily-briefing fallback slot. Most ticks land outside every user's + * slot; that is the tick doing its job, so a zero `sent` is a success. + */ +export async function handleDailyBriefing(): Promise { + return withBackgroundEvent("job.daily_briefing", async (evt) => { + try { + const summary = await runDailyBriefingTick(getWorkerPrisma(), new Date()); + evt.setBackground({ + task_name: "job.daily_briefing", + result: { + candidates_scanned: summary.candidatesScanned, + in_slot: summary.inSlot, + sent: summary.sent, + suppressed_frequency: summary.suppressedFrequency, + no_digest: summary.noDigest, + opted_out: summary.optedOut, + no_channel: summary.noChannel, + outside_window: summary.outsideWindow, + failed: summary.failed, + }, + }); + return jobDone({ + candidates_scanned: summary.candidatesScanned, + in_slot: summary.inSlot, + sent: summary.sent, + failed: summary.failed, + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); +} + +/** + * The period-narrative warm: a scheduled tick (no `userId`) runs the + * boundary-gated nightly fan-out, a `userId` payload warms one account for + * the read-only GET. Two modes on one queue, so the outcome counts them + * apart: a non-boundary night generates nothing and is still a success. + */ +export async function handlePeriodNarrative( + jobs: Job[], +): Promise { + let singleUser = 0; + let scheduled = 0; + let generated = 0; + let failed = 0; + let stoppedEarly = false; + for (const job of jobs) { + try { + if (job.data?.userId) { + await warmOneNarrative(job.data); + singleUser++; + } else { + const summary = await runPeriodNarrativeWarm(getWorkerPrisma(), { + shouldStop: jobBudget(jobs), + }); + scheduled++; + generated += summary.generated; + failed += summary.failed; + stoppedEarly ||= summary.stoppedEarly; + workerLog( + "info", + `[period-narrative] periods=${summary.periods.join(",") || "none"} total=${summary.total} generated=${summary.generated} cached=${summary.cached} skipped=${summary.skipped} insufficient=${summary.insufficient} failed=${summary.failed} budget=${summary.budgetBlocked}`, + ); + } + } catch (err) { + recordError(); + await reportWorkerError(PERIOD_NARRATIVE_QUEUE, err, { + mode: job.data?.userId ? "single-user" : "scheduled", + }); + throw err; + } + } + return jobDone({ + single_user: singleUser, + scheduled, + generated, + failed, + stopped_early: stoppedEarly, + }); +} + +/** + * The Coach memory refresh for one long conversation. An incomplete payload + * is skipped rather than failed, and counted, so a queue full of them does + * not read as a clean run. + */ +export async function handleCoachMemoryRefresh( + jobs: Job[], +): Promise { + let refreshed = 0; + let skippedIncomplete = 0; + for (const job of jobs) { + if (!job.data?.conversationId || !job.data?.userId) { + skippedIncomplete++; + continue; + } + try { + await runCoachMemoryRefresh(job.data); + refreshed++; + } catch (err) { + recordError(); + workerLog("error", "[coach-memory-refresh] failed", err); + throw err; + } + } + return jobDone({ + refreshed, + skipped_incomplete: skippedIncomplete, + }); +} + +/** + * The daily Coach-reminder sweep. Surfacing an overdue reminder and minting + * plan-review reminders are both idempotent across re-fires; no provider call, + * no push. + */ +export async function handleCoachReminderSweep(): Promise { + return withBackgroundEvent("job.coach_reminder_sweep", async (evt) => { + try { + const summary = await runCoachReminderSweep( + getWorkerPrisma(), + new Date(), + ); + evt.setBackground({ + task_name: "job.coach_reminder_sweep", + result: { + reminders_due: summary.remindersDue, + plan_reviews_minted: summary.planReviewsMinted, + context_surfaced: summary.contextSurfaced, + nag_dismissed: summary.nagDismissed, + errored: summary.errored, + }, + }); + return jobDone({ + reminders_due: summary.remindersDue, + plan_reviews_minted: summary.planReviewsMinted, + context_surfaced: summary.contextSurfaced, + nag_dismissed: summary.nagDismissed, + errored: summary.errored, + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); +} + +/** + * The n-of-1 experiment review: writes the grounded before/after read-back + * into the plan's encrypted outcome. A reviewed plan is not re-processed. + */ +export async function handleCoachPlanReview(): Promise { + return withBackgroundEvent("job.coach_plan_review", async (evt) => { + try { + const summary = await runCoachPlanReviewTick( + getWorkerPrisma(), + new Date(), + ); + evt.setBackground({ + task_name: "job.coach_plan_review", + result: { + reviewed: summary.reviewed, + insufficient: summary.insufficient, + errored: summary.errored, + }, + }); + return jobDone({ + reviewed: summary.reviewed, + insufficient: summary.insufficient, + errored: summary.errored, + }); + } catch (err) { + evt.setError(err); + recordError(); + throw err; + } + }); +} diff --git a/src/lib/jobs/restore-drill-queue.ts b/src/lib/jobs/restore-drill-queue.ts new file mode 100644 index 000000000..397def8b5 --- /dev/null +++ b/src/lib/jobs/restore-drill-queue.ts @@ -0,0 +1,23 @@ +/** + * The queue facts of `restore-drill.ts`: its name and, where it has them, its + * schedule, concurrency and send options. They live apart from the job so the + * worker registry and every sender can read them without loading it. + */ +export const RESTORE_DRILL_QUEUE = "data-restore-drill"; + +export const RESTORE_DRILL_CRON = "11 4 1 * *"; + +/** + * Four hours, the off-host upload's own window: the drill downloads and + * stream-parses every object that upload wrote in one night. + */ +export const RESTORE_DRILL_EXPIRE_SECONDS = 4 * 60 * 60; + +/** + * What the monthly schedule sends with: the expiry above and no retry + * (`retryLimit: 0`, the maintenance schedules' `cronIsTheRetry`). + */ +export const RESTORE_DRILL_SEND_OPTIONS = { + expireInSeconds: RESTORE_DRILL_EXPIRE_SECONDS, + retryLimit: 0, +} as const; diff --git a/src/lib/jobs/restore-drill.ts b/src/lib/jobs/restore-drill.ts index 1a6854bc1..a2d38c476 100644 --- a/src/lib/jobs/restore-drill.ts +++ b/src/lib/jobs/restore-drill.ts @@ -48,24 +48,7 @@ import { reportWorkerError } from "@/lib/jobs/report-worker-error"; import { jobBudget } from "@/lib/jobs/job-budget"; import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; import { withBackgroundEvent } from "@/lib/logging/background"; - -export const RESTORE_DRILL_QUEUE = "data-restore-drill"; -export const RESTORE_DRILL_CRON = "11 4 1 * *"; - -/** - * Four hours, the off-host upload's own window: the drill downloads and - * stream-parses every object that upload wrote in one night. - */ -export const RESTORE_DRILL_EXPIRE_SECONDS = 4 * 60 * 60; - -/** - * What the monthly schedule sends with: the expiry above and no retry - * (`retryLimit: 0`, the maintenance schedules' `cronIsTheRetry`). - */ -export const RESTORE_DRILL_SEND_OPTIONS = { - expireInSeconds: RESTORE_DRILL_EXPIRE_SECONDS, - retryLimit: 0, -} as const; +import { RESTORE_DRILL_QUEUE } from "./restore-drill-queue"; /** * A drill run that finds the newest backup older than this is reported diff --git a/src/lib/jobs/sleep-timeline-backfill-queue.ts b/src/lib/jobs/sleep-timeline-backfill-queue.ts new file mode 100644 index 000000000..c120b9920 --- /dev/null +++ b/src/lib/jobs/sleep-timeline-backfill-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `sleep-timeline-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const SLEEP_TIMELINE_BACKFILL_QUEUE = "sleep-timeline-backfill"; + +/** + * Serial concurrency — a pass deletes a source's sleep rows then walks a + * re-sync that is rate-bounded by the upstream API cap. Concurrency-1 keeps it + * from crowding the request pool, matching the other backfill queues. + */ +export const SLEEP_TIMELINE_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/sleep-timeline-backfill.ts b/src/lib/jobs/sleep-timeline-backfill.ts index 619a632da..2adbfa5d5 100644 --- a/src/lib/jobs/sleep-timeline-backfill.ts +++ b/src/lib/jobs/sleep-timeline-backfill.ts @@ -40,15 +40,7 @@ import { syncUserSleep as syncWithingsSleep, } from "@/lib/withings/sync-sleep"; import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; - -export const SLEEP_TIMELINE_BACKFILL_QUEUE = "sleep-timeline-backfill"; - -/** - * Serial concurrency — a pass deletes a source's sleep rows then walks a - * re-sync that is rate-bounded by the upstream API cap. Concurrency-1 keeps it - * from crowding the request pool, matching the other backfill queues. - */ -export const SLEEP_TIMELINE_BACKFILL_CONCURRENCY = 1; +import { SLEEP_TIMELINE_BACKFILL_QUEUE } from "./sleep-timeline-backfill-queue"; export type SleepTimelineProvider = "WHOOP" | "WITHINGS"; diff --git a/src/lib/jobs/strain-score-queue.ts b/src/lib/jobs/strain-score-queue.ts new file mode 100644 index 000000000..0417a725e --- /dev/null +++ b/src/lib/jobs/strain-score-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `strain-score.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const STRAIN_SCORE_QUEUE = "strain-score-compute"; + +/** + * Daily at 04:55 Europe/Berlin — inside the existing maintenance window, + * after the recovery-score (04:45) + stress-score (04:50) passes so the + * nightly score writes stay ordered and don't pile on one boss poll. + */ +export const STRAIN_SCORE_CRON = "55 4 * * *"; diff --git a/src/lib/jobs/strain-score.ts b/src/lib/jobs/strain-score.ts index 2d019fa05..afe09ddab 100644 --- a/src/lib/jobs/strain-score.ts +++ b/src/lib/jobs/strain-score.ts @@ -21,15 +21,6 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { persistStrainScore } from "@/lib/insights/strain-score"; import { runScoreBatch } from "@/lib/insights/score-row"; -export const STRAIN_SCORE_QUEUE = "strain-score-compute"; - -/** - * Daily at 04:55 Europe/Berlin — inside the existing maintenance window, - * after the recovery-score (04:45) + stress-score (04:50) passes so the - * nightly score writes stay ordered and don't pile on one boss poll. - */ -export const STRAIN_SCORE_CRON = "55 4 * * *"; - /** Per-run user cap — same starvation guard the other score passes use. */ export const STRAIN_SCORE_BATCH_CAP = 500; diff --git a/src/lib/jobs/strava-backfill-queue.ts b/src/lib/jobs/strava-backfill-queue.ts new file mode 100644 index 000000000..db638639f --- /dev/null +++ b/src/lib/jobs/strava-backfill-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `strava-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const STRAVA_BACKFILL_QUEUE = "strava-backfill"; + +/** + * Serial concurrency — a backfill walks the account's activity history and is + * rate-bounded by Strava's 200-req / 15-min cap; concurrency-1 keeps it from + * crowding the request pool, matching `WHOOP_BACKFILL_CONCURRENCY`. + */ +export const STRAVA_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/strava-backfill.ts b/src/lib/jobs/strava-backfill.ts index 10bb6dd21..a18a85d73 100644 --- a/src/lib/jobs/strava-backfill.ts +++ b/src/lib/jobs/strava-backfill.ts @@ -27,15 +27,7 @@ import { withPageWalkLedger, } from "@/lib/integrations/page-walk"; import { syncStravaHistory } from "@/lib/strava/sync"; - -export const STRAVA_BACKFILL_QUEUE = "strava-backfill"; - -/** - * Serial concurrency — a backfill walks the account's activity history and is - * rate-bounded by Strava's 200-req / 15-min cap; concurrency-1 keeps it from - * crowding the request pool, matching `WHOOP_BACKFILL_CONCURRENCY`. - */ -export const STRAVA_BACKFILL_CONCURRENCY = 1; +import { STRAVA_BACKFILL_QUEUE } from "./strava-backfill-queue"; export interface StravaBackfillPayload { userId: string; diff --git a/src/lib/jobs/stress-score-queue.ts b/src/lib/jobs/stress-score-queue.ts new file mode 100644 index 000000000..70261c31a --- /dev/null +++ b/src/lib/jobs/stress-score-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `stress-score.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const STRESS_SCORE_QUEUE = "stress-score-compute"; + +/** + * Daily at 04:50 Europe/Berlin — inside the existing maintenance window, + * after the dense intra-day retention drain (03:50) and the recovery-score + * pass (04:45) so the HRV inputs it reads are settled. + */ +export const STRESS_SCORE_CRON = "50 4 * * *"; diff --git a/src/lib/jobs/stress-score.ts b/src/lib/jobs/stress-score.ts index 04b25cb1d..8a73866bf 100644 --- a/src/lib/jobs/stress-score.ts +++ b/src/lib/jobs/stress-score.ts @@ -22,15 +22,6 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { persistStressScore } from "@/lib/insights/stress-score"; import { runScoreBatch } from "@/lib/insights/score-row"; -export const STRESS_SCORE_QUEUE = "stress-score-compute"; - -/** - * Daily at 04:50 Europe/Berlin — inside the existing maintenance window, - * after the dense intra-day retention drain (03:50) and the recovery-score - * pass (04:45) so the HRV inputs it reads are settled. - */ -export const STRESS_SCORE_CRON = "50 4 * * *"; - /** Per-run user cap — same starvation guard the recovery-score pass uses. */ export const STRESS_SCORE_BATCH_CAP = 500; diff --git a/src/lib/jobs/tls-pin-monitor-queue.ts b/src/lib/jobs/tls-pin-monitor-queue.ts new file mode 100644 index 000000000..f08599e58 --- /dev/null +++ b/src/lib/jobs/tls-pin-monitor-queue.ts @@ -0,0 +1,15 @@ +/** + * The queue facts of `tls-pin-monitor.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +/** + * pg-boss queue name + cron. Every 6 hours at :07 (offset off the :00 / :05 + * / :08 hourly sync crons so the probe doesn't pile onto a busy boss poll). + * GTS leaves renew roughly every ~90 days; a 6-hour cadence surfaces a + * change well inside the ≥11-day re-pin window the runbook targets while + * costing one short outbound TLS handshake per tick. + */ +export const TLS_PIN_MONITOR_QUEUE = "tls-pin-monitor"; + +export const TLS_PIN_MONITOR_CRON = "7 */6 * * *"; diff --git a/src/lib/jobs/tls-pin-monitor.ts b/src/lib/jobs/tls-pin-monitor.ts index 3c1c5e394..16cc93c10 100644 --- a/src/lib/jobs/tls-pin-monitor.ts +++ b/src/lib/jobs/tls-pin-monitor.ts @@ -40,16 +40,6 @@ import { type LeafProbeResult, } from "@/lib/tls/leaf-spki"; -/** - * pg-boss queue name + cron. Every 6 hours at :07 (offset off the :00 / :05 - * / :08 hourly sync crons so the probe doesn't pile onto a busy boss poll). - * GTS leaves renew roughly every ~90 days; a 6-hour cadence surfaces a - * change well inside the ≥11-day re-pin window the runbook targets while - * costing one short outbound TLS handshake per tick. - */ -export const TLS_PIN_MONITOR_QUEUE = "tls-pin-monitor"; -export const TLS_PIN_MONITOR_CRON = "7 */6 * * *"; - export interface TlsPinMonitorSummary { /** "ok" served pin is in the known set; "changed" it is not; "skipped" no target / no baseline; "probe_failed" the TLS probe threw. */ outcome: "ok" | "changed" | "skipped" | "probe_failed"; diff --git a/src/lib/jobs/whoop-backfill-queue.ts b/src/lib/jobs/whoop-backfill-queue.ts new file mode 100644 index 000000000..21ea1793e --- /dev/null +++ b/src/lib/jobs/whoop-backfill-queue.ts @@ -0,0 +1,13 @@ +/** + * The queue facts of `whoop-backfill.ts`: its name and, where it has them, its + * schedule and concurrency. They live apart from the handler so the worker + * registry and every sender can read them without loading the job. + */ +export const WHOOP_BACKFILL_QUEUE = "whoop-backfill"; + +/** + * Serial concurrency — a backfill walks years of history for one account and + * is rate-bounded by WHOOP's 100 req/min cap; concurrency-1 keeps it from + * crowding the request pool, matching `ROLLUP_FULL_BACKFILL_CONCURRENCY`. + */ +export const WHOOP_BACKFILL_CONCURRENCY = 1; diff --git a/src/lib/jobs/whoop-backfill.ts b/src/lib/jobs/whoop-backfill.ts index 83d87ad82..d61822d8a 100644 --- a/src/lib/jobs/whoop-backfill.ts +++ b/src/lib/jobs/whoop-backfill.ts @@ -26,15 +26,7 @@ import { withPageWalkLedger, } from "@/lib/integrations/page-walk"; import { syncUserWhoop } from "@/lib/whoop/sync"; - -export const WHOOP_BACKFILL_QUEUE = "whoop-backfill"; - -/** - * Serial concurrency — a backfill walks years of history for one account and - * is rate-bounded by WHOOP's 100 req/min cap; concurrency-1 keeps it from - * crowding the request pool, matching `ROLLUP_FULL_BACKFILL_CONCURRENCY`. - */ -export const WHOOP_BACKFILL_CONCURRENCY = 1; +import { WHOOP_BACKFILL_QUEUE } from "./whoop-backfill-queue"; export interface WhoopBackfillPayload { userId: string; diff --git a/src/lib/medications/intake-import-admission.ts b/src/lib/medications/intake-import-admission.ts index 591140da9..21a487b49 100644 --- a/src/lib/medications/intake-import-admission.ts +++ b/src/lib/medications/intake-import-admission.ts @@ -23,12 +23,12 @@ */ import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { - MEDICATION_INTAKE_IMPORT_QUEUE, MEDICATION_INTAKE_IMPORT_SEND_OPTIONS, MEDICATION_INTAKE_IMPORT_STALE_AFTER_MS, type MedicationImportPayload, type MedicationImportProgress, } from "@/lib/jobs/medication-intake-import"; +import { MEDICATION_INTAKE_IMPORT_QUEUE } from "@/lib/jobs/medication-intake-import-queue"; import { auditLog } from "@/lib/auth/audit"; import { prisma, toJson } from "@/lib/db"; diff --git a/src/lib/support/report.ts b/src/lib/support/report.ts index d186ee20b..fbbfa8fa3 100644 --- a/src/lib/support/report.ts +++ b/src/lib/support/report.ts @@ -38,12 +38,10 @@ import { PROBLEMS_BUFFER_MAX, } from "@/lib/logging/in-memory-buffer"; import type { WideEvent } from "@/lib/logging/types"; -import { - offhostBackupConfigured, - OFFHOST_BACKUP_QUEUE, -} from "@/lib/jobs/offhost-backup"; +import { offhostBackupConfigured } from "@/lib/jobs/offhost-backup"; +import { OFFHOST_BACKUP_QUEUE } from "@/lib/jobs/offhost-backup-queue"; import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy"; -import { RESTORE_DRILL_QUEUE } from "@/lib/jobs/restore-drill"; +import { RESTORE_DRILL_QUEUE } from "@/lib/jobs/restore-drill-queue"; import { JOB_FAILURE_WINDOW_HOURS, readFailingQueues, diff --git a/src/lib/whoop/webhook-handler.ts b/src/lib/whoop/webhook-handler.ts index a815cabba..a3c05e829 100644 --- a/src/lib/whoop/webhook-handler.ts +++ b/src/lib/whoop/webhook-handler.ts @@ -34,6 +34,11 @@ import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; import type { MeasurementType, Prisma } from "@/generated/prisma/client"; import { WHOOP_RECOVERY_FIELD_TAGS } from "./client"; +import { + WHOOP_RECOVERY_SYNC_QUEUE, + WHOOP_SLEEP_SYNC_QUEUE, + WHOOP_WORKOUT_SYNC_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; /** * Maximum age of a webhook timestamp before it is treated as a replay. Five @@ -48,9 +53,9 @@ const WHOOP_SIGNATURE_MAX_AGE_MS = 5 * 60 * 1000; * webhook (poll-only) so it is intentionally absent. */ const WHOOP_RESOURCE_QUEUE: Record = { - recovery: "whoop-recovery-sync", - sleep: "whoop-sleep-sync", - workout: "whoop-workout-sync", + recovery: WHOOP_RECOVERY_SYNC_QUEUE, + sleep: WHOOP_SLEEP_SYNC_QUEUE, + workout: WHOOP_WORKOUT_SYNC_QUEUE, }; /** diff --git a/src/lib/withings/webhook-handler.ts b/src/lib/withings/webhook-handler.ts index 0405653bf..f831c98ca 100644 --- a/src/lib/withings/webhook-handler.ts +++ b/src/lib/withings/webhook-handler.ts @@ -31,6 +31,10 @@ import { getClientIp, safeJson } from "@/lib/api-response"; import { annotate, getEvent } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { enqueueWithingsEcgSync } from "@/lib/jobs/withings-ecg-queue"; +import { + WITHINGS_ACTIVITY_QUEUE, + WITHINGS_SLEEP_QUEUE, +} from "@/lib/jobs/integration-poll-queues"; /** * v1.4.25 — appli categories that have their own sync routine @@ -41,8 +45,6 @@ import { enqueueWithingsEcgSync } from "@/lib/jobs/withings-ecg-queue"; */ const WITHINGS_ACTIVITY_APPLI = 16; const WITHINGS_SLEEP_APPLI = 44; -const WITHINGS_ACTIVITY_QUEUE = "withings-activity-sync"; -const WITHINGS_SLEEP_QUEUE = "withings-sleep-sync"; /** * Constant-time comparison helper. Returns false unless both inputs have diff --git a/tests/integration/backup-restore-job.test.ts b/tests/integration/backup-restore-job.test.ts index 670143467..1902b918a 100644 --- a/tests/integration/backup-restore-job.test.ts +++ b/tests/integration/backup-restore-job.test.ts @@ -44,12 +44,12 @@ import { setGlobalBoss } from "@/lib/jobs/boss-instance"; import { runJob } from "@/lib/jobs/run-job"; import { BACKUP_RESTORE_MAX_ATTEMPTS, - BACKUP_RESTORE_QUEUE, handleBackupRestore, readBackupRestoreJob, runBackupRestoreJob, sweepInterruptedRestores, } from "@/lib/jobs/backup-restore"; +import { BACKUP_RESTORE_QUEUE } from "@/lib/jobs/backup-restore-queue"; import { backupPayloadSchema } from "@/lib/validations/backup"; import { cookieJar, headerJar } from "./mock-next-headers"; diff --git a/tests/integration/health-connect-import.test.ts b/tests/integration/health-connect-import.test.ts index e7dceb5bd..3c965acac 100644 --- a/tests/integration/health-connect-import.test.ts +++ b/tests/integration/health-connect-import.test.ts @@ -34,9 +34,9 @@ import { importHealthConnectExport } from "@/lib/import/health-connect/import"; import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention"; import { HEALTH_CONNECT_IMPORT_KIND, - HEALTH_CONNECT_IMPORT_QUEUE, runHealthConnectImport, } from "@/lib/jobs/health-connect-import-worker"; +import { HEALTH_CONNECT_IMPORT_QUEUE } from "@/lib/jobs/health-connect-import-queue"; import { _setWorkerPrismaForTests, reconcileOrphanImportJobs, diff --git a/tests/integration/queue-registration-parity.test.ts b/tests/integration/queue-registration-parity.test.ts index 40ff5d72b..544432dc0 100644 --- a/tests/integration/queue-registration-parity.test.ts +++ b/tests/integration/queue-registration-parity.test.ts @@ -71,21 +71,8 @@ function recordingBoss(real: PgBoss): PgBoss { } async function registerEveryQueue(target: PgBoss): Promise { - const [integration, status, rollup, reminders, maintenance] = - await Promise.all([ - import("@/lib/jobs/reminder/register-integration-sync"), - import("@/lib/jobs/reminder/register-status"), - import("@/lib/jobs/reminder/register-rollup"), - import("@/lib/jobs/reminder/register-reminders"), - import("@/lib/jobs/reminder/register-maintenance"), - ]); - returned.push( - ...(await integration.registerIntegrationSyncQueues(target)), - ...(await status.registerStatusQueues(target)), - ...(await rollup.registerRollupQueues(target)), - ...(await reminders.registerReminderQueues(target)), - ...(await maintenance.registerMaintenanceQueues(target)), - ); + const { registerAllJobGroups } = await import("@/lib/jobs/registry/index"); + returned.push(...(await registerAllJobGroups(target))); } beforeAll(async () => { @@ -142,4 +129,18 @@ describe("queue registration", () => { // golden. expect(returned.length).toBeGreaterThan(100); }); + + it("loads a function for every handler it declares", async () => { + // A handler module loads on its queue's first job; a broken import would + // otherwise surface only when that job arrives in production. + const { allJobDefinitions } = await import("@/lib/jobs/registry/index"); + const definitions = allJobDefinitions(); + expect(definitions.length).toBe(returned.length); + const unloaded: string[] = []; + for (const definition of definitions) { + const handler = await definition.load({ boss }); + if (typeof handler !== "function") unloaded.push(definition.queue); + } + expect(unloaded).toEqual([]); + }); }); diff --git a/tests/integration/routes/auth-me-documents-auto-ai-read.test.ts b/tests/integration/routes/auth-me-documents-auto-ai-read.test.ts index 6071eb8e2..fec91f067 100644 --- a/tests/integration/routes/auth-me-documents-auto-ai-read.test.ts +++ b/tests/integration/routes/auth-me-documents-auto-ai-read.test.ts @@ -27,10 +27,8 @@ import type { Prisma, User } from "@/generated/prisma/client"; import { GET, PATCH } from "@/app/api/auth/me/documents-auto-ai-read/route"; import { aiCapabilityForJob } from "@/lib/ai/capabilities/gate"; import { setGlobalBoss } from "@/lib/jobs/boss-instance"; -import { - DOCUMENT_SUMMARY_CATCHUP_QUEUE, - enqueueSummaryCatchUp, -} from "@/lib/jobs/document-summary-catchup"; +import { enqueueSummaryCatchUp } from "@/lib/jobs/document-summary-catchup"; +import { DOCUMENT_SUMMARY_CATCHUP_QUEUE } from "@/lib/jobs/document-summary-catchup-queue"; import { anonymous, callRoute } from "../../kit/call-route"; import { makeSession, makeUser, uniqueSuffix } from "../../kit/factories";