diff --git a/docs/self-hosting/scaling.md b/docs/self-hosting/scaling.md index c5f0b0071..6d703d9ab 100644 --- a/docs/self-hosting/scaling.md +++ b/docs/self-hosting/scaling.md @@ -250,3 +250,66 @@ The implementation lives in `src/lib/db.ts → getConnectionBudget()` (split into `getPrismaPoolMax()` and `getPgBossPoolMax()`); the 20-slot default was chosen after measuring on a 4-CPU production container — see the inline rationale in `src/lib/db.ts`. + +## Large accounts + +An account that holds tens of millions of readings, most often a +full-history Google Health backfill of minute-by-minute heart rate, runs +into the per-statement limit the app sets on every database connection: +60 seconds, from `DATABASE_STATEMENT_TIMEOUT_MS` (unset means 60000, `0` +turns the limit off). + +### What it looks like + +- Dashboard cards stay empty, or `/api/daily/digest` and other reads + answer 500. +- The log shows `job.run.failed` for `rollup-full-backfill` with + `consecutiveFailures` counting up, and Postgres error `57014`: + `canceling statement due to statement timeout`. +- The admin status page lists `rollup-full-backfill` under failing + background jobs, and every admin gets a push once a system job has + failed three times in a row, a reminder at most once a day while it keeps + failing, and a notice when it runs again. + +### What the app does about it + +- The full rollup backfill, which runs at boot for any account whose + summary tier is incomplete (and after a release that adds a column to it), + folds the five-year window one calendar year at a time. Every statement of + it runs under its own limit of five minutes, whatever + `DATABASE_STATEMENT_TIMEOUT_MS` says, and a run that reaches its time budget + hands the remaining years to a follow-up job. The readers that fall back to + the raw readings while that tier is incomplete are fast again once it has + finished. +- Heart rate older than 90 days is folded into one mean per hour, for Apple + Health, Health Connect, Google Health and Oura alike. The DAY summaries keep + the figures taken from the raw samples, and a later backfill or full sync + does not write the folded samples back. On an account with a dense history + the first fold works through the backlog over a few nights, one day per + transaction, and leaves an account alone while its summary tier is still + being rebuilt. Postgres reuses the freed table pages for new rows; to give + the index space back to the disk, run the index rebuild of the measurement + maintenance job from Admin, System status, once the backlog is through + (`docs/ops/measurement-maintenance.md`). + +### When to raise the limit + +Normally never: the long background passes bring their own limits. Raise +`DATABASE_STATEMENT_TIMEOUT_MS` only when request reads still time out on a +large account after the backfill has finished (the failing-jobs card is +empty), or when something between the app and Postgres sets a lower limit. +In that case also check `SHOW statement_timeout;` as the database role +HealthLog connects with: a limit set on the role or the database, or by a +pooler, applies too. Recreate the container after changing the variable so +every connection picks it up: + +```yaml +# docker-compose.yml +services: + app: + environment: + DATABASE_STATEMENT_TIMEOUT_MS: "180000" # three minutes +``` + +A higher limit lets one slow statement hold a pool connection longer, so +raise it in steps rather than turning it off. diff --git a/messages/de.json b/messages/de.json index 3fbe7be8f..d71e38a98 100644 --- a/messages/de.json +++ b/messages/de.json @@ -9260,6 +9260,13 @@ "titleFailing": "Sicherungen sind gefährdet", "titleReminder": "Sicherungen sind weiterhin gefährdet", "titleRecovered": "Sicherungen laufen wieder normal" + }, + "systemJobAlert": { + "titleFailing": "Hintergrundjob schlägt fehl: {queue}", + "titleReminder": "Hintergrundjob schlägt weiter fehl: {queue}", + "titleRecovered": "Hintergrundjob läuft wieder: {queue}", + "bodyFailing": "Der Hintergrundjob {queue} ist {failures}-mal in Folge fehlgeschlagen und steht auf der Admin-Statusseite. Letzter Fehler: {error}", + "bodyRecovered": "Der Hintergrundjob {queue} ist wieder erfolgreich gelaufen." } }, "i18n": { diff --git a/messages/en.json b/messages/en.json index a9d17bec0..49c2d6fcd 100644 --- a/messages/en.json +++ b/messages/en.json @@ -9260,6 +9260,13 @@ "titleFailing": "Backups are at risk", "titleReminder": "Backups are still at risk", "titleRecovered": "Backups are back to normal" + }, + "systemJobAlert": { + "titleFailing": "Background job failing: {queue}", + "titleReminder": "Background job still failing: {queue}", + "titleRecovered": "Background job working again: {queue}", + "bodyFailing": "The background job {queue} has failed {failures} times in a row and is listed on the admin status page. Last error: {error}", + "bodyRecovered": "The background job {queue} has run successfully again." } }, "i18n": { diff --git a/messages/es.json b/messages/es.json index 88fa5e906..664fbb6b2 100644 --- a/messages/es.json +++ b/messages/es.json @@ -9260,6 +9260,13 @@ "titleFailing": "Las copias de seguridad están en riesgo", "titleReminder": "Las copias de seguridad siguen en riesgo", "titleRecovered": "Las copias de seguridad vuelven a la normalidad" + }, + "systemJobAlert": { + "titleFailing": "Tarea en segundo plano con fallos: {queue}", + "titleReminder": "La tarea en segundo plano sigue fallando: {queue}", + "titleRecovered": "La tarea en segundo plano vuelve a funcionar: {queue}", + "bodyFailing": "La tarea en segundo plano {queue} ha fallado {failures} veces seguidas y aparece en la página de estado de administración. Último error: {error}", + "bodyRecovered": "La tarea en segundo plano {queue} se ha vuelto a ejecutar correctamente." } }, "i18n": { diff --git a/messages/fr.json b/messages/fr.json index 318a161f7..c48261e70 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -9260,6 +9260,13 @@ "titleFailing": "Les sauvegardes sont menacées", "titleReminder": "Les sauvegardes sont toujours menacées", "titleRecovered": "Les sauvegardes sont revenues à la normale" + }, + "systemJobAlert": { + "titleFailing": "Tâche d'arrière-plan en échec : {queue}", + "titleReminder": "Tâche d'arrière-plan toujours en échec : {queue}", + "titleRecovered": "Tâche d'arrière-plan rétablie : {queue}", + "bodyFailing": "La tâche d'arrière-plan {queue} a échoué {failures} fois de suite ; vous la trouverez sur la page d'état de l'administration. Dernière erreur : {error}", + "bodyRecovered": "La tâche d'arrière-plan {queue} s'est de nouveau exécutée avec succès." } }, "i18n": { diff --git a/messages/it.json b/messages/it.json index f0b82d04e..f44262fe3 100644 --- a/messages/it.json +++ b/messages/it.json @@ -9260,6 +9260,13 @@ "titleFailing": "I backup sono a rischio", "titleReminder": "I backup sono ancora a rischio", "titleRecovered": "I backup sono tornati alla normalità" + }, + "systemJobAlert": { + "titleFailing": "Attività in background non riuscita: {queue}", + "titleReminder": "Attività in background ancora non riuscita: {queue}", + "titleRecovered": "Attività in background di nuovo funzionante: {queue}", + "bodyFailing": "L'attività in background {queue} non è riuscita {failures} volte di seguito ed è elencata nella pagina di stato dell'amministrazione. Ultimo errore: {error}", + "bodyRecovered": "L'attività in background {queue} è stata di nuovo eseguita correttamente." } }, "i18n": { diff --git a/messages/ko.json b/messages/ko.json index 190f1e83d..01cea67ec 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -9260,6 +9260,13 @@ "titleFailing": "백업이 위험해요", "titleReminder": "백업이 여전히 위험해요", "titleRecovered": "백업이 다시 정상이에요" + }, + "systemJobAlert": { + "titleFailing": "백그라운드 작업이 실패하고 있어요: {queue}", + "titleReminder": "백그라운드 작업이 계속 실패하고 있어요: {queue}", + "titleRecovered": "백그라운드 작업이 다시 작동해요: {queue}", + "bodyFailing": "백그라운드 작업 {queue}이(가) {failures}번 연속 실패했어요. 관리자 상태 페이지에서 확인할 수 있어요. 마지막 오류: {error}", + "bodyRecovered": "백그라운드 작업 {queue}이(가) 다시 정상적으로 실행됐어요." } }, "i18n": { diff --git a/messages/nl.json b/messages/nl.json index 2fa0ed1b9..9fe4b8f08 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -9260,6 +9260,13 @@ "titleFailing": "Back-ups lopen gevaar", "titleReminder": "Back-ups lopen nog steeds gevaar", "titleRecovered": "Back-ups zijn weer in orde" + }, + "systemJobAlert": { + "titleFailing": "Achtergrondtaak mislukt: {queue}", + "titleReminder": "Achtergrondtaak mislukt nog steeds: {queue}", + "titleRecovered": "Achtergrondtaak werkt weer: {queue}", + "bodyFailing": "De achtergrondtaak {queue} is {failures} keer achter elkaar mislukt en staat op de beheerstatuspagina. Laatste fout: {error}", + "bodyRecovered": "De achtergrondtaak {queue} is weer met succes uitgevoerd." } }, "i18n": { diff --git a/messages/pl.json b/messages/pl.json index cb9b94acc..94fc0bcaf 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -9260,6 +9260,13 @@ "titleFailing": "Kopie zapasowe są zagrożone", "titleReminder": "Kopie zapasowe są nadal zagrożone", "titleRecovered": "Kopie zapasowe znów działają normalnie" + }, + "systemJobAlert": { + "titleFailing": "Zadanie w tle kończy się błędem: {queue}", + "titleReminder": "Zadanie w tle nadal kończy się błędem: {queue}", + "titleRecovered": "Zadanie w tle znów działa: {queue}", + "bodyFailing": "Zadanie w tle {queue} zakończyło się błędem {failures} razy z rzędu i jest widoczne na stronie stanu w panelu administratora. Ostatni błąd: {error}", + "bodyRecovered": "Zadanie w tle {queue} znów wykonało się poprawnie." } }, "i18n": { diff --git a/messages/pt-BR.json b/messages/pt-BR.json index dfa846444..0e3fd7d56 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -9260,6 +9260,13 @@ "titleFailing": "Os backups estão em risco", "titleReminder": "Os backups continuam em risco", "titleRecovered": "Os backups voltaram ao normal" + }, + "systemJobAlert": { + "titleFailing": "Tarefa em segundo plano falhando: {queue}", + "titleReminder": "Tarefa em segundo plano ainda falhando: {queue}", + "titleRecovered": "Tarefa em segundo plano funcionando de novo: {queue}", + "bodyFailing": "A tarefa em segundo plano {queue} falhou {failures} vezes seguidas e aparece na página de status da administração. Último erro: {error}", + "bodyRecovered": "A tarefa em segundo plano {queue} voltou a ser executada com sucesso." } }, "i18n": { diff --git a/prisma/migrations/0490_system_job_alert_notices/migration.sql b/prisma/migrations/0490_system_job_alert_notices/migration.sql new file mode 100644 index 000000000..fa78e55e2 --- /dev/null +++ b/prisma/migrations/0490_system_job_alert_notices/migration.sql @@ -0,0 +1,24 @@ +-- System job alert: what the admins were last told about a background job of +-- the instance that keeps failing. +-- +-- `system_job_alert_notices` holds, per pg-boss queue, the state the admins +-- were last notified about and when. It is the whole memory of the hourly +-- evaluation (`src/lib/jobs/system-job-alert.ts`): a queue that has failed +-- three runs in a row notifies once, is reminded at most once a day while it +-- keeps failing, and its first successful run after that is announced once. +-- A row exists only for a queue something was announced about. +-- +-- One new table, nothing else touched, so the migration takes no lock on an +-- existing table and running it again changes nothing. +-- +-- Reversibility: dropping the table loses only the memory of what was +-- announced; the next evaluation announces a still-failing queue once more. + +CREATE TABLE IF NOT EXISTS "system_job_alert_notices" ( + "queue" TEXT NOT NULL, + "notified_state" TEXT, + "notified_at" TIMESTAMP(3), + CONSTRAINT "system_job_alert_notices_pkey" PRIMARY KEY ("queue"), + CONSTRAINT "system_job_alert_notices_state_check" + CHECK ("notified_state" IS NULL OR "notified_state" IN ('ok', 'failing')) +); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 648b62a75..0e566b0a6 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -5958,6 +5958,23 @@ model RecoveryReadinessNotice { @@map("recovery_readiness_notices") } +/// Per background queue of the instance, the state the admins were last +/// notified about. The hourly system job evaluation compares a queue's run +/// history against it: three failed runs in a row notify once, a queue still +/// failing is reminded at most once a day, and its first successful run after +/// that is announced once. The update is a compare-and-set on both columns, so +/// two workers never send the same notice. Written and read by +/// `notifySystemJobFailures`. +model SystemJobAlertNotice { + /// The pg-boss queue name (`rollup-full-backfill`, …). + queue String @id + /// `ok` or `failing`; null once a queue's history has aged out. + notifiedState String? @map("notified_state") + notifiedAt DateTime? @map("notified_at") + + @@map("system_job_alert_notices") +} + /// The recovery kit an admin confirmed: every key a restore needs, by kind, /// id and fingerprint, never the key itself. The kit is due again as soon as /// a key it needs is not in this set (a rotation, a new backup key). Written diff --git a/src/__tests__/baselines/source-file-lines.json b/src/__tests__/baselines/source-file-lines.json index 66e675fd4..31b681f6f 100644 --- a/src/__tests__/baselines/source-file-lines.json +++ b/src/__tests__/baselines/source-file-lines.json @@ -1,7 +1,7 @@ { "$comment": "Production files over 800 lines and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.", "count": 84, - "total": 94928, + "total": 94686, "entries": { "app/medications/page-client.tsx": 839, "components/charts/mood-chart.tsx": 1166, @@ -71,14 +71,14 @@ "lib/managed-profiles/handover.ts": 986, "lib/mcp/record-search.ts": 1467, "lib/mcp/rich-reads.ts": 1467, - "lib/measurements/dense-intraday-retention.ts": 1057, + "lib/measurements/dense-intraday-retention.ts": 1010, "lib/measurements/import-apple-health-export.ts": 1722, "lib/medications/scheduling/cadence.ts": 976, "lib/medications/scheduling/recurrence.ts": 1100, "lib/medications/scheduling/slot-upsert.ts": 912, "lib/notifications/senders/apns.ts": 1100, "lib/oura/sync.ts": 900, - "lib/rollups/measurement-rollups.ts": 1496, + "lib/rollups/measurement-rollups.ts": 1302, "lib/search/search-records.ts": 996, "lib/sharing/grants.ts": 806, "lib/support/report.ts": 903, diff --git a/src/__tests__/baselines/version-stamp-comments.json b/src/__tests__/baselines/version-stamp-comments.json index c44b1da77..c1d8207df 100644 --- a/src/__tests__/baselines/version-stamp-comments.json +++ b/src/__tests__/baselines/version-stamp-comments.json @@ -1,7 +1,7 @@ { "$comment": "Comment lines that open with a release version, per file under src/. Maintained by src/__tests__/version-stamp-ratchet.test.ts.", "count": 2896, - "total": 9235, + "total": 9229, "entries": { "__tests__/account-payload-consumer-guard.test.ts": 2, "__tests__/acting-account-boundary-guard.test.ts": 7, @@ -175,9 +175,9 @@ "app/api/auth/refresh/route.ts": 2, "app/api/auth/register/__tests__/route.test.ts": 5, "app/api/auth/register/route.ts": 10, + "app/api/biomarkers/__tests__/error-contract.test.ts": 1, "app/api/biomarkers/[id]/__tests__/route.test.ts": 2, "app/api/biomarkers/[id]/route.ts": 6, - "app/api/biomarkers/__tests__/error-contract.test.ts": 1, "app/api/biomarkers/route.ts": 4, "app/api/c/[token]/unlock/__tests__/route.test.ts": 1, "app/api/c/[token]/unlock/route.ts": 1, @@ -187,12 +187,12 @@ "app/api/coach/about-me/questions/__tests__/route-module-gate.test.ts": 1, "app/api/coach/about-me/questions/route.ts": 3, "app/api/coach/about-me/route.ts": 4, - "app/api/coach/plans/[id]/route.ts": 2, "app/api/coach/plans/__tests__/route.test.ts": 1, + "app/api/coach/plans/[id]/route.ts": 2, "app/api/coach/plans/route.ts": 1, "app/api/coach/reminder-suggestions/route.ts": 1, - "app/api/coach/reminders/[id]/route.ts": 1, "app/api/coach/reminders/__tests__/route.test.ts": 1, + "app/api/coach/reminders/[id]/route.ts": 1, "app/api/coach/reminders/route.ts": 1, "app/api/coach/suggested-actions/__tests__/route.test.ts": 1, "app/api/coach/suggested-actions/route.ts": 1, @@ -200,9 +200,9 @@ "app/api/consent/ai/latest/__tests__/route.test.ts": 3, "app/api/consent/ai/latest/route.ts": 3, "app/api/consent/ai/route.ts": 3, - "app/api/custom-metrics/[id]/entries/[entryId]/route.ts": 2, "app/api/custom-metrics/[id]/entries/__tests__/note-encryption.test.ts": 1, "app/api/custom-metrics/[id]/entries/__tests__/soft-delete-restore.test.ts": 1, + "app/api/custom-metrics/[id]/entries/[entryId]/route.ts": 2, "app/api/custom-metrics/[id]/entries/route.ts": 4, "app/api/custom-metrics/[id]/route.ts": 1, "app/api/custom-metrics/route.ts": 1, @@ -212,8 +212,8 @@ "app/api/cycle/day-logs/[id]/route.ts": 4, "app/api/cycle/day-logs/route.ts": 1, "app/api/cycle/period/route.ts": 4, - "app/api/cycle/symptoms/custom/[key]/route.ts": 1, "app/api/cycle/symptoms/custom/__tests__/custom-crud.test.ts": 1, + "app/api/cycle/symptoms/custom/[key]/route.ts": 1, "app/api/cycle/symptoms/custom/route.ts": 3, "app/api/dashboard/chart-overlay-prefs/__tests__/route.test.ts": 1, "app/api/dashboard/chart-overlay-prefs/route.ts": 2, @@ -223,10 +223,11 @@ "app/api/dashboard/summary/route.ts": 5, "app/api/dashboard/widgets/__tests__/route.test.ts": 4, "app/api/dashboard/widgets/route.ts": 14, + "app/api/devices/__tests__/route.test.ts": 1, "app/api/devices/[id]/__tests__/route.test.ts": 1, "app/api/devices/[id]/route.ts": 1, - "app/api/devices/__tests__/route.test.ts": 1, "app/api/devices/route.ts": 4, + "app/api/documents/inbound/__tests__/route.test.ts": 1, "app/api/documents/inbound/[id]/__tests__/route.test.ts": 1, "app/api/documents/inbound/[id]/chat/__tests__/route.test.ts": 2, "app/api/documents/inbound/[id]/chat/route.ts": 1, @@ -241,7 +242,6 @@ "app/api/documents/inbound/[id]/suggest/route.ts": 2, "app/api/documents/inbound/[id]/summary/route.ts": 2, "app/api/documents/inbound/[id]/thumbnail/route.ts": 1, - "app/api/documents/inbound/__tests__/route.test.ts": 1, "app/api/documents/inbound/reindex/route.ts": 1, "app/api/documents/inbound/route.ts": 2, "app/api/documents/inbound/usage/route.ts": 1, @@ -260,10 +260,10 @@ "app/api/export/mood/route.ts": 2, "app/api/family-history/[id]/route.ts": 2, "app/api/family-history/route.ts": 1, - "app/api/fhir/Patient/$everything/route.ts": 1, "app/api/fhir/__tests__/observation-series.test.ts": 1, "app/api/fhir/__tests__/route.test.ts": 1, "app/api/fhir/metadata/route.ts": 1, + "app/api/fhir/Patient/$everything/route.ts": 1, "app/api/fitbit/callback/route.ts": 1, "app/api/fitbit/connect/route.ts": 1, "app/api/fitbit/credentials/route.ts": 1, @@ -292,11 +292,6 @@ "app/api/insights/breathing-screening/route.ts": 2, "app/api/insights/cards/__tests__/route.test.ts": 2, "app/api/insights/cards/route.ts": 9, - "app/api/insights/chat/[id]/attachments/[documentId]/route.ts": 1, - "app/api/insights/chat/[id]/attachments/__tests__/route.test.ts": 1, - "app/api/insights/chat/[id]/attachments/route.ts": 1, - "app/api/insights/chat/[id]/messages/[messageId]/trail/__tests__/route.test.ts": 1, - "app/api/insights/chat/[id]/route.ts": 1, "app/api/insights/chat/__tests__/dialog-harness.ts": 1, "app/api/insights/chat/__tests__/golden-transcripts.test.ts": 1, "app/api/insights/chat/__tests__/route-activity.test.ts": 1, @@ -308,14 +303,19 @@ "app/api/insights/chat/__tests__/route-snapshot-once.test.ts": 2, "app/api/insights/chat/__tests__/route-tool-mode.test.ts": 4, "app/api/insights/chat/__tests__/route-workout-scope.test.ts": 1, + "app/api/insights/chat/[id]/attachments/__tests__/route.test.ts": 1, + "app/api/insights/chat/[id]/attachments/[documentId]/route.ts": 1, + "app/api/insights/chat/[id]/attachments/route.ts": 1, + "app/api/insights/chat/[id]/messages/[messageId]/trail/__tests__/route.test.ts": 1, + "app/api/insights/chat/[id]/route.ts": 1, "app/api/insights/chat/fenced/__tests__/route.test.ts": 1, "app/api/insights/chat/fenced/route.ts": 1, "app/api/insights/chat/messages/[id]/feedback/route.ts": 3, "app/api/insights/chat/route.ts": 9, "app/api/insights/coach-read/route.ts": 2, - "app/api/insights/coach/facts/[id]/route.ts": 2, "app/api/insights/coach/facts/__tests__/route-memory.test.ts": 1, "app/api/insights/coach/facts/__tests__/route.test.ts": 1, + "app/api/insights/coach/facts/[id]/route.ts": 2, "app/api/insights/coach/facts/route.ts": 1, "app/api/insights/coach/nudge-status/route.ts": 1, "app/api/insights/coach/seen/route.ts": 1, @@ -326,9 +326,9 @@ "app/api/insights/derived/batch/__tests__/route.test.ts": 3, "app/api/insights/derived/batch/route.ts": 6, "app/api/insights/derived/route.ts": 5, + "app/api/insights/ecg/__tests__/route.test.ts": 1, "app/api/insights/ecg/[id]/__tests__/route.test.ts": 1, "app/api/insights/ecg/[id]/route.ts": 2, - "app/api/insights/ecg/__tests__/route.test.ts": 1, "app/api/insights/ecg/route.ts": 1, "app/api/insights/feedback/route.ts": 2, "app/api/insights/generate/__tests__/route.test.ts": 12, @@ -367,8 +367,8 @@ "app/api/internal/deploy-webhook/route.ts": 1, "app/api/internal/web-vitals/__tests__/route.test.ts": 1, "app/api/internal/web-vitals/route.ts": 1, - "app/api/labs/[id]/route.ts": 7, "app/api/labs/__tests__/error-contract.test.ts": 1, + "app/api/labs/[id]/route.ts": 7, "app/api/labs/ocr/capability/route.ts": 1, "app/api/labs/ocr/commit/__tests__/route.test.ts": 2, "app/api/labs/ocr/commit/route.ts": 4, @@ -388,13 +388,13 @@ "app/api/measurement-reminders/[id]/skip/route.ts": 1, "app/api/measurement-reminders/[id]/snooze/route.ts": 1, "app/api/measurement-reminders/route.ts": 3, - "app/api/measurements/[id]/__tests__/route.test.ts": 2, - "app/api/measurements/[id]/route.ts": 16, "app/api/measurements/__tests__/arrival-wiring.test.ts": 1, "app/api/measurements/__tests__/group-by-day.test.ts": 8, "app/api/measurements/__tests__/put-duplicate-timestamp.test.ts": 1, "app/api/measurements/__tests__/range-aggregation-route.test.ts": 3, "app/api/measurements/__tests__/zod-multi-issue.test.ts": 4, + "app/api/measurements/[id]/__tests__/route.test.ts": 2, + "app/api/measurements/[id]/route.ts": 16, "app/api/measurements/batch/__tests__/aggregated-hr-wire.test.ts": 1, "app/api/measurements/batch/__tests__/duplicate-prefilter.test.ts": 2, "app/api/measurements/batch/__tests__/hr-bucket-observability.test.ts": 1, @@ -411,6 +411,7 @@ "app/api/measurements/series-batch/route.ts": 1, "app/api/measurements/series/__tests__/route.test.ts": 5, "app/api/measurements/series/route.ts": 6, + "app/api/medications/__tests__/route.test.ts": 10, "app/api/medications/[id]/__tests__/route.test.ts": 8, "app/api/medications/[id]/api-endpoint/route.ts": 2, "app/api/medications/[id]/cadence/route.ts": 10, @@ -422,10 +423,10 @@ "app/api/medications/[id]/dose-history/route.ts": 14, "app/api/medications/[id]/glp1/__tests__/route.test.ts": 3, "app/api/medications/[id]/glp1/route.ts": 8, - "app/api/medications/[id]/intake/[eventId]/__tests__/route.test.ts": 9, - "app/api/medications/[id]/intake/[eventId]/route.ts": 4, "app/api/medications/[id]/intake/__tests__/route.test.ts": 11, "app/api/medications/[id]/intake/__tests__/source-provenance.test.ts": 1, + "app/api/medications/[id]/intake/[eventId]/__tests__/route.test.ts": 9, + "app/api/medications/[id]/intake/[eventId]/route.ts": 4, "app/api/medications/[id]/intake/bulk-delete/route.ts": 7, "app/api/medications/[id]/intake/import/__tests__/route.test.ts": 2, "app/api/medications/[id]/intake/import/route.ts": 6, @@ -433,12 +434,11 @@ "app/api/medications/[id]/inventory/[itemId]/route.ts": 8, "app/api/medications/[id]/inventory/route.ts": 4, "app/api/medications/[id]/phase-config/route.ts": 4, - "app/api/medications/[id]/schedule-revisions/[revisionId]/route.ts": 9, "app/api/medications/[id]/schedule-revisions/__tests__/route.test.ts": 1, + "app/api/medications/[id]/schedule-revisions/[revisionId]/route.ts": 9, "app/api/medications/[id]/schedule-revisions/route.ts": 3, "app/api/medications/[id]/side-effects/[logId]/route.ts": 2, "app/api/medications/[id]/side-effects/route.ts": 5, - "app/api/medications/__tests__/route.test.ts": 10, "app/api/medications/categories/[key]/route.ts": 1, "app/api/medications/categories/route.ts": 1, "app/api/medications/compliance/__tests__/route.test.ts": 2, @@ -455,9 +455,9 @@ "app/api/medications/route.ts": 2, "app/api/mental-health/assessments/__tests__/route.test.ts": 2, "app/api/mental-health/assessments/route.ts": 6, + "app/api/mood-entries/__tests__/route.test.ts": 7, "app/api/mood-entries/[id]/__tests__/route.test.ts": 4, "app/api/mood-entries/[id]/route.ts": 22, - "app/api/mood-entries/__tests__/route.test.ts": 7, "app/api/mood-entries/bulk-delete/__tests__/route.test.ts": 1, "app/api/mood-entries/bulk-delete/route.ts": 4, "app/api/mood-entries/bulk/__tests__/route.test.ts": 5, @@ -547,12 +547,12 @@ "app/api/withings/connect/__tests__/route.test.ts": 1, "app/api/withings/connect/route.ts": 3, "app/api/withings/status/route.ts": 1, - "app/api/withings/webhook/[token]/__tests__/route.test.ts": 1, "app/api/withings/webhook/__tests__/route.test.ts": 1, + "app/api/withings/webhook/[token]/__tests__/route.test.ts": 1, "app/api/withings/webhook/route.ts": 2, - "app/api/workouts/[id]/route.ts": 2, "app/api/workouts/__tests__/batch-create.test.ts": 3, "app/api/workouts/__tests__/canonical-dedup.test.ts": 3, + "app/api/workouts/[id]/route.ts": 2, "app/api/workouts/batch/route.ts": 3, "app/api/workouts/route.ts": 2, "app/auth/claim/page.tsx": 1, @@ -654,20 +654,20 @@ "app/loading.tsx": 2, "app/mcp/__tests__/route.test.ts": 5, "app/measurements/page.tsx": 5, - "app/medications/[id]/history/page.tsx": 1, - "app/medications/[id]/page.tsx": 2, "app/medications/__tests__/category-filter.test.tsx": 2, "app/medications/__tests__/detail-page-history-surface.test.ts": 1, "app/medications/__tests__/list-view-toggle.test.tsx": 5, "app/medications/__tests__/page-prefetch.test.tsx": 1, "app/medications/__tests__/take-all-due-affordance.test.tsx": 1, + "app/medications/[id]/history/page.tsx": 1, + "app/medications/[id]/page.tsx": 2, "app/medications/new/page.tsx": 1, "app/medications/page-client.tsx": 50, "app/mental-wellbeing/page.tsx": 1, "app/mood/page-client.tsx": 2, "app/not-found.tsx": 2, - "app/onboarding/[step]/page.tsx": 1, "app/onboarding/__tests__/page.test.tsx": 1, + "app/onboarding/[step]/page.tsx": 1, "app/onboarding/handover/page.tsx": 1, "app/onboarding/page.tsx": 2, "app/privacy/__tests__/page.test.tsx": 1, @@ -2747,7 +2747,7 @@ "lib/rollups/measurement-coverage.ts": 1, "lib/rollups/measurement-read-wmy.ts": 4, "lib/rollups/measurement-read.ts": 10, - "lib/rollups/measurement-rollups.ts": 18, + "lib/rollups/measurement-rollups.ts": 12, "lib/rollups/mood-rollups.ts": 4, "lib/rollups/tiered-context.ts": 6, "lib/score-config/rows.ts": 1, diff --git a/src/__tests__/error-level-discipline-guard.test.ts b/src/__tests__/error-level-discipline-guard.test.ts index edbc87410..c3432c8be 100644 --- a/src/__tests__/error-level-discipline-guard.test.ts +++ b/src/__tests__/error-level-discipline-guard.test.ts @@ -163,6 +163,11 @@ const ALLOWED: Readonly> = { count: 1, reason: "No /proc/diskstats outside Linux; see captureHostMetric.", }, + "lib/jobs/system-job-alert.ts::readTerminalRuns": { + count: 1, + reason: + "No pgboss schema on a web-only deployment; null means no queue to ask, and the evaluation then judges nothing.", + }, "lib/jobs/job-failures.ts::readActiveJobsStartedBefore": { count: 1, reason: diff --git a/src/app/api/admin/rollups/recompute/route.ts b/src/app/api/admin/rollups/recompute/route.ts index 480b36a97..63f612c49 100644 --- a/src/app/api/admin/rollups/recompute/route.ts +++ b/src/app/api/admin/rollups/recompute/route.ts @@ -15,8 +15,8 @@ * { userId?: string } * * - `userId` — single user mode. Synchronously calls - * `recomputeUserRollups(userId)` and returns the row count when - * the fold completes. The call is awaited inside the request, so + * `foldUserRollupHistory(userId)` (the full fold, a calendar year per + * statement) and returns the row count when the fold completes. The call is awaited inside the request, so * the operator sees the duration in the response. * - `userId` omitted — calls `enqueueBootTimeRollupBackfill()` to * re-run the worker-boot discovery (`{ enqueued, skipped }` shape). @@ -37,8 +37,8 @@ import { import { annotate } from "@/lib/logging/context"; import { enqueueBootTimeRollupBackfill, - recomputeUserRollups, -} from "@/lib/rollups/measurement-rollups"; + foldUserRollupHistory, +} from "@/lib/rollups/full-backfill"; interface RecomputeBody { userId?: string; @@ -68,9 +68,9 @@ export const POST = apiHandler(async (request: NextRequest) => { details: { targetUserId: userId, mode: "single" }, }); - let result: Awaited>; + let result: Awaited>; try { - result = await recomputeUserRollups(userId); + result = await foldUserRollupHistory(userId); } catch (err) { const message = err instanceof Error ? err.message : String(err); return apiError(message, 500); diff --git a/src/lib/analytics/range-delta.ts b/src/lib/analytics/range-delta.ts index 4bee215a5..64c056aec 100644 --- a/src/lib/analytics/range-delta.ts +++ b/src/lib/analytics/range-delta.ts @@ -316,6 +316,8 @@ export async function computeRangeDelta( userId, type, windowDays * 2, + undefined, + new Date(now), ); if (!resolved) { const empty: WindowAggregate = { diff --git a/src/lib/data-wipe/wipe-plan.ts b/src/lib/data-wipe/wipe-plan.ts index 9edc727b8..6900918ed 100644 --- a/src/lib/data-wipe/wipe-plan.ts +++ b/src/lib/data-wipe/wipe-plan.ts @@ -358,6 +358,8 @@ export const INSTANCE_SCOPED: Readonly> = { "the monthly restore drill's last verdict for the instance's off-host bucket; says nothing about any account", RecoveryReadinessNotice: "per backup signal, what the admins were last told; a fact about the instance's backups, not about any account", + SystemJobAlertNotice: + "per background queue, what the admins were last told about its failures; a fact about the instance's jobs, not about any account", RecoveryKitConfirmation: "the instance's recovery kit confirmation (key ids and fingerprints); the confirming admin is a plain id, not a relation", DatabaseBackupRun: diff --git a/src/lib/google-health/mapping.md b/src/lib/google-health/mapping.md index eb958bf6b..0d92bef67 100644 --- a/src/lib/google-health/mapping.md +++ b/src/lib/google-health/mapping.md @@ -89,7 +89,7 @@ Scope: `googlehealth.health_metrics_and_measurements.readonly`. | `daily-heart-rate-variability` | `dailyHeartRateVariability.averageHeartRateVariabilityMilliseconds` | `HRV_RMSSD` | ms | `hrv` | daily | Google documents the daily average as RMSSD, so it lands in `HRV_RMSSD`, apart from the SDNN `HEART_RATE_VARIABILITY`. | | `daily-resting-heart-rate` | `dailyRestingHeartRate.beatsPerMinute` (int64 string) | `RESTING_HEART_RATE` | bpm | `rhr` | daily | | | `daily-respiratory-rate` | `dailyRespiratoryRate.breathsPerMinute` (number) | `RESPIRATORY_RATE` | breaths/min | `resp_rate` | daily | `respiratory-rate` does NOT exist in the catalogue. Schema is `{date, breathsPerMinute}` — the earlier `dailyRespiratoryRateBpm` leaf never existed. | -| `heart-rate` | `heartRate.beatsPerMinute` (int64 string) | `PULSE` | bpm | `hr` | sample | Intraday spot HR; per-minute volume — consider the 14-day-max rollup if PULSE should stay coarse. | +| `heart-rate` | `heartRate.beatsPerMinute` (int64 string) | `PULSE` | bpm | `hr` | sample | Intraday spot HR, one row per sample; older than 90 days it folds to hourly means (`dense-intraday-sources.ts`). | | `blood-glucose` | `bloodGlucose.bloodGlucoseMilligramsPerDeciliter` (number) | `BLOOD_GLUCOSE` | mg/dL | `glucose` | sample | Already in the canonical mg/dL storage unit. Meal/timing/specimen enums not stored. | | `core-body-temperature` | `coreBodyTemperature.temperatureCelsius` (number) | `BODY_TEMPERATURE` | celsius | `core_temp` | sample | Core slot — distinct from SKIN_TEMPERATURE / WRIST_TEMPERATURE. | | `daily-sleep-temperature-derivations` | `dailySleepTemperatureDerivations.nightlyTemperatureCelsius` (number) | `WRIST_TEMPERATURE` | celsius | `wrist_temp` | daily | ABSOLUTE nightly skin temperature (mean over the night's samples) — the schema refutes the earlier "deviation-only" premise. Baseline/stddev derivations not stored. | diff --git a/src/lib/google-health/sync-core.ts b/src/lib/google-health/sync-core.ts index 44f200e28..7553f1268 100644 --- a/src/lib/google-health/sync-core.ts +++ b/src/lib/google-health/sync-core.ts @@ -36,6 +36,7 @@ import { type MeasurementReconciliationVerdict, } from "@/lib/measurements/reconcile-external-measurement"; import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; +import { withoutFoldedSamples } from "@/lib/measurements/folded-window"; import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; import { recordSyncFailure, type FailureKind } from "@/lib/integrations/status"; import { collapseToTypeDayKeys } from "@/lib/rollups/measurement-rollups"; @@ -1128,7 +1129,7 @@ export async function upsertGoogleHealthMeasurements( // before they can become rows. A refused reading is dropped and tallied on // the ambient event, never clamped and never written for a later reader to // discover. - const { kept: readings, dropped: naturalKeyTwins } = collapseNaturalKeyTwins( + const { kept: collapsed, dropped: naturalKeyTwins } = collapseNaturalKeyTwins( dropImplausibleMeasurements("googleHealth", incoming, (r) => ({ type: r.type, value: r.value, @@ -1137,14 +1138,18 @@ export async function upsertGoogleHealthMeasurements( if (naturalKeyTwins > 0) { annotate({ meta: { "googleHealth.naturalKeyTwins": naturalKeyTwins } }); } + // A sample of an hour the dense fold already took is not written again. + const { kept: readings } = await withoutFoldedSamples( + prisma, + userId, + "GOOGLE_HEALTH", + collapsed, + ); if (readings.length === 0) return { imported: 0, touched: [], inserted: [] }; // Probe EVERY existing row (live AND tombstoned) for the batch's externalIds - // in a single query. The full unique index guarantees at most one row per - // `(type, externalId)` key, and a tombstoned match must take the UPDATE - // branch (which clears `deletedAt`) — treating it as absent would plan an - // insert that `skipDuplicates` silently drops against the tombstone's key, - // permanently wedging the key (see TOMBSTONES RESURRECT above). + // in one query: a tombstoned match must take the UPDATE branch, or the + // insert is silently dropped against its key (see TOMBSTONES RESURRECT). const externalIds = readings.map((r) => r.externalId); interface ProbedRow { id: string; @@ -1162,10 +1167,8 @@ export async function upsertGoogleHealthMeasurements( source: "GOOGLE_HEALTH", externalId: { in: externalIds }, }, - // The payload fields ride along so the update branch can skip a no-op - // overwrite: the 24 h overlap re-fetches every recent row hourly, and an - // unconditional write would bump `syncVersion` (iOS delta churn) on rows - // whose values did not change. + // The payload rides along so the update branch can skip a no-op + // overwrite (the hourly 24 h overlap would bump `syncVersion` for nothing). select: { id: true, type: true, @@ -1193,9 +1196,8 @@ export async function upsertGoogleHealthMeasurements( ]), ); } catch (err) { - // A probe failure must not strand the whole batch; fall back to treating - // every row as fresh (`skipDuplicates` absorbs the collision on the full - // unique index — those rows simply retry on the next sync tick). + // A probe failure must not strand the batch: every row is treated as + // fresh, and `skipDuplicates` absorbs the collisions until the next tick. getEvent()?.addWarning(`google-health: row probe failed: ${err}`); } @@ -1225,11 +1227,9 @@ export async function upsertGoogleHealthMeasurements( const key = `${type} ${r.externalId}`; const existingRow = rowByKey.get(key); if (existingRow) { - // Skip the no-op overwrite: a LIVE row whose payload matches the - // re-fetched reading exactly gains nothing from a write, and the - // unconditional `syncVersion` bump churned the DB + every iOS delta pull - // hourly for the whole 24 h overlap window. A TOMBSTONED row always - // updates — the write is what resurrects it (`deletedAt: null`). + // Skip the no-op overwrite of a LIVE row whose payload matches exactly + // (it would churn `syncVersion` and every iOS delta pull). A TOMBSTONED + // row always updates: the write is what resurrects it. const unchanged = existingRow.deletedAt === null && existingRow.value === r.value && diff --git a/src/lib/illness/correlation-read.ts b/src/lib/illness/correlation-read.ts index aa50005d1..2b6f983bd 100644 --- a/src/lib/illness/correlation-read.ts +++ b/src/lib/illness/correlation-read.ts @@ -217,7 +217,13 @@ async function readWindowMeans( 1, Math.ceil((to.getTime() - from.getTime()) / MS_PER_DAY), ); - const resolved = await readBestGranularityRollups(userId, type, windowDays); + const resolved = await readBestGranularityRollups( + userId, + type, + windowDays, + undefined, + now, + ); if ( resolved && resolved.granularity === "DAY" && diff --git a/src/lib/insights/derived/baseline.ts b/src/lib/insights/derived/baseline.ts index f17913712..7b0f2a6bd 100644 --- a/src/lib/insights/derived/baseline.ts +++ b/src/lib/insights/derived/baseline.ts @@ -274,7 +274,13 @@ export async function readDayMeanSeries( // band from WEEK/MONTH `sd`, and the center reads the DAY `mean` // which composes exactly. `readBestGranularityRollups` with a // window < 91 days resolves to DAY by construction. - const resolved = await readBestGranularityRollups(userId, type, windowDays); + const resolved = await readBestGranularityRollups( + userId, + type, + windowDays, + undefined, + now, + ); if ( resolved && resolved.granularity === "DAY" && diff --git a/src/lib/insights/graded-series.ts b/src/lib/insights/graded-series.ts index 4f82da485..616745700 100644 --- a/src/lib/insights/graded-series.ts +++ b/src/lib/insights/graded-series.ts @@ -537,6 +537,7 @@ export async function buildGradedSeriesWithRollups( type, 365, priority, + now, ); // Yearly slice: route a multi-year window through the tier. const yearlyRouted = await readBestGranularityRollups( @@ -544,6 +545,7 @@ export async function buildGradedSeriesWithRollups( type, 1095, priority, + now, ); const monthlyCovered = diff --git a/src/lib/jobs/__tests__/system-job-alert.test.ts b/src/lib/jobs/__tests__/system-job-alert.test.ts new file mode 100644 index 000000000..9719308d4 --- /dev/null +++ b/src/lib/jobs/__tests__/system-job-alert.test.ts @@ -0,0 +1,87 @@ +/** + * The system job alert's verdict on one queue's ledger (#1298). The notice + * flow itself runs against a real pg-boss in + * `tests/integration/system-job-alert.test.ts`. + */ +import { describe, expect, it } from "vitest"; + +import { + judgeQueueRuns, + SYSTEM_JOB_FAILURE_STREAK, + watchedQueues, + type TerminalRun, +} from "@/lib/jobs/system-job-alert"; + +const at = new Date("2026-10-29T10:00:00.000Z"); + +function run(overrides: Partial = {}): TerminalRun { + return { + state: "failed", + retryCount: 0, + allFailed: false, + completedOn: at, + error: "boom", + expireSeconds: null, + ...overrides, + }; +} + +describe("judgeQueueRuns", () => { + it("is inactive with no run left in the ledger", () => { + expect(judgeQueueRuns([]).state).toBe("inactive"); + }); + + it("counts every attempt of a run, so one job that used up its retries is a streak", () => { + const verdict = judgeQueueRuns([run({ retryCount: 3 })]); + expect(verdict).toMatchObject({ state: "failing", failures: 4 }); + expect(verdict.lastError).toBe("boom"); + }); + + it("calls the streak at the threshold, not before", () => { + const below = Array.from({ length: SYSTEM_JOB_FAILURE_STREAK - 1 }, () => + run(), + ); + expect(judgeQueueRuns(below).state).toBe("pending"); + expect(judgeQueueRuns([...below, run()]).state).toBe("failing"); + }); + + it("ends the streak at the newest success", () => { + const verdict = judgeQueueRuns([ + run({ error: "newest" }), + run({ state: "completed", error: null }), + run(), + run(), + run(), + ]); + expect(verdict).toMatchObject({ + state: "ok", + failures: 1, + lastError: "newest", + }); + }); + + it("counts a completed run that says every candidate failed", () => { + const allFailed = run({ state: "completed", allFailed: true }); + expect(judgeQueueRuns([allFailed, allFailed, allFailed]).state).toBe( + "failing", + ); + }); + + it("explains a bare timeout the way the admin card does", () => { + const verdict = judgeQueueRuns([ + run({ error: "job timed out", expireSeconds: 3600, retryCount: 2 }), + ]); + expect(verdict.lastError).toMatch( + /^job timed out: did not finish within its 1 h limit/, + ); + }); +}); + +describe("watchedQueues", () => { + it("watches the system queues and none with an account-facing surface", () => { + const queues = watchedQueues(); + expect(queues).toContain("rollup-full-backfill"); + expect(queues).toContain("system-job-alert"); + expect(queues).not.toContain("google-health-sync"); + }); +}); diff --git a/src/lib/jobs/dense-intraday-hourly-rebuild.ts b/src/lib/jobs/dense-intraday-hourly-rebuild.ts index e5a8c0f8a..b58219802 100644 --- a/src/lib/jobs/dense-intraday-hourly-rebuild.ts +++ b/src/lib/jobs/dense-intraday-hourly-rebuild.ts @@ -30,10 +30,8 @@ import { prisma } from "@/lib/db"; import { annotate } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { DENSE_INTRADAY_RETENTION_ENABLED } from "@/lib/jobs/dense-intraday-retention"; -import { - DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_RETENTION_TYPES, -} from "@/lib/measurements/dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_DAYS } from "@/lib/measurements/dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_TYPES } from "@/lib/measurements/dense-intraday-sources"; import { runDenseIntradayHourlyRebuild } from "@/lib/measurements/dense-intraday-hourly-rebuild"; import { DENSE_INTRADAY_HOURLY_REBUILD_QUEUE } from "./dense-intraday-hourly-rebuild-queue"; diff --git a/src/lib/jobs/dense-intraday-retention.ts b/src/lib/jobs/dense-intraday-retention.ts index 27b7322ae..6b679fbfe 100644 --- a/src/lib/jobs/dense-intraday-retention.ts +++ b/src/lib/jobs/dense-intraday-retention.ts @@ -18,16 +18,19 @@ * `src/lib/jobs/reminder-worker.ts` so pg-boss provisions it at boot; an * unregistered queue silently never drains. */ -import { dropPendingDeletion } from "@/lib/account-deletion/active-account"; +import { ACTIVE_ACCOUNT } from "@/lib/account-deletion/active-account"; import { prisma } from "@/lib/db"; import { annotate } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { runDenseIntradayRetention, - DENSE_INTRADAY_RETENTION_TYPES, DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_FOLD_SOURCES, } from "@/lib/measurements/dense-intraday-retention"; +import { + DENSE_INTRADAY_FOLD_SOURCES, + DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE, +} from "@/lib/measurements/dense-intraday-sources"; +import { isRollupFullBackfillPending } from "@/lib/rollups/full-backfill"; import { DENSE_INTRADAY_RETENTION_QUEUE } from "./dense-intraday-retention-queue"; /** @@ -103,6 +106,7 @@ export async function runDenseIntradayRetentionForUser( const summary = await runDenseIntradayRetention(prisma, { userId, shouldStop, + deferUser: isRollupFullBackfillPending, log: () => { // Silent inside the queue handler — the worker logs the totals. }, @@ -133,8 +137,8 @@ export async function runDenseIntradayRetentionForUser( /** * Boot-time discovery. Finds every user holding at least one LIVE - * per-sample dense-tier row (a row of a `DENSE_INTRADAY_FOLD_SOURCES` source - * and a `DENSE_INTRADAY_RETENTION_TYPES` type whose externalId does NOT start + * per-sample dense-tier row (a row of a source and type the fold takes, + * `DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE`, whose externalId does NOT start * with the daily-stats prefix, that is not tombstoned, AND whose * `measuredAt` is older than the retention window) and enqueues one * retention job per account. @@ -161,36 +165,40 @@ export async function enqueueBootTimeDenseIntradayRetention(): Promise<{ } try { - // The dense-tier type set is a closed compile-time list of enum - // members; splice-free — Prisma binds the `type IN (...)` array as - // parameters. - const types = Array.from(DENSE_INTRADAY_RETENTION_TYPES); - if (types.length === 0) { - return { enqueued: 0, skipped: 0, error: null }; - } - const windowStart = new Date( Date.now() - DENSE_INTRADAY_RETENTION_DAYS * 24 * 60 * 60 * 1000, ); - // `groupBy` compiles to a server-side `GROUP BY` — Prisma's - // `distinct` dedups in the client AFTER pulling every matching row, - // an instance-wide scan of every out-of-window dense-tier row just - // to list user ids. - const users = await dropPendingDeletion( - prisma, - await prisma.measurement.groupBy({ - by: ["userId"], - where: { - source: { in: [...DENSE_INTRADAY_FOLD_SOURCES] }, - type: { in: types }, - deletedAt: null, - NOT: { externalId: { startsWith: "stats:" } }, - measuredAt: { lt: windowStart }, - }, - }), - (u) => u.userId, - ); + // One probe per account and source for the first raw sample the fold + // would take, which stops at that row. A grouped count over every such + // row read the whole out-of-window history of every account to list + // their ids, and on an account with tens of millions of heart-rate rows + // that is most of the table (#1297). + const accounts = await prisma.user.findMany({ + where: ACTIVE_ACCOUNT, + select: { id: true }, + orderBy: { createdAt: "asc" }, + }); + const users: Array<{ userId: string }> = []; + for (const { id } of accounts) { + for (const source of DENSE_INTRADAY_FOLD_SOURCES) { + const raw = await prisma.measurement.findFirst({ + where: { + userId: id, + source, + type: { in: [...DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE[source]] }, + deletedAt: null, + NOT: { externalId: { startsWith: "stats:" } }, + measuredAt: { lt: windowStart }, + }, + select: { id: true }, + }); + if (raw) { + users.push({ userId: id }); + break; + } + } + } if (users.length === 0) { return { enqueued: 0, skipped: 0, error: null }; diff --git a/src/lib/jobs/job-failures.ts b/src/lib/jobs/job-failures.ts index 3b4b8f004..c9bf7a82b 100644 --- a/src/lib/jobs/job-failures.ts +++ b/src/lib/jobs/job-failures.ts @@ -145,7 +145,7 @@ function explainTimeout(message: string, expireSeconds: number | null): string { * serialises to `{ message, stack, … }`; a thrown non-Error can be any JSON at * all, so the raw text is the fallback rather than an invented placeholder. */ -function presentError( +export function presentError( raw: string | null, expireSeconds: number | null = null, ): string { diff --git a/src/lib/jobs/job-outcome.ts b/src/lib/jobs/job-outcome.ts index 3ede654d1..40bc1d872 100644 --- a/src/lib/jobs/job-outcome.ts +++ b/src/lib/jobs/job-outcome.ts @@ -307,6 +307,10 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "summary_pending_healed", "suppressed_client_managed", "suppressed_discreet", + // The hourly system job alert: queues failing now, notices sent, admins. + "system_job_admins", + "system_job_notices", + "system_jobs_failing", "thumbnails_enqueued", "tls_pin_monitor_known_count", "tls_pin_monitor_outcome", @@ -330,6 +334,8 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet = new Set([ "users_synced", "users_useful", "wedge_skipped", + // The full rollup backfill: calendar-year windows one run folded. + "windows_folded", ]); export const MAX_JOB_FACTS = 32; diff --git a/src/lib/jobs/measurement-fold-repair.ts b/src/lib/jobs/measurement-fold-repair.ts index c9c42e3b0..8a8738e41 100644 --- a/src/lib/jobs/measurement-fold-repair.ts +++ b/src/lib/jobs/measurement-fold-repair.ts @@ -116,12 +116,11 @@ import { hourOfDayForUserTz, } from "@/lib/measurements/consolidation-tz"; import { - DENSE_INTRADAY_FOLD_SOURCES, DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_RETENTION_TYPES, deriveDailyRestingFromPulse, hourlyStatsExternalId, } from "@/lib/measurements/dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_TYPES } from "@/lib/measurements/dense-intraday-sources"; import { absorbIntoFold, foldConstituentHorizon, @@ -160,6 +159,9 @@ interface RepairStep { foldedAfterMs: number; } +/** Sources the dense fold took when this repair was written; later ones never split a window. */ +const REPAIRED_DENSE_SOURCES = ["APPLE_HEALTH", "HEALTH_CONNECT"] as const; + /** Fixed order, so a resume position (`step`) means the same on every run. */ export const FOLD_REPAIR_STEPS: readonly RepairStep[] = [ ...[...HIGH_FREQUENCY_MEAN_TYPES].map((type): RepairStep => ({ @@ -168,7 +170,7 @@ export const FOLD_REPAIR_STEPS: readonly RepairStep[] = [ grain: "day", foldedAfterMs: CONSOLIDATION_GRACE_CUTOFF_HOURS * HOUR_MS, })), - ...DENSE_INTRADAY_FOLD_SOURCES.flatMap((source) => + ...REPAIRED_DENSE_SOURCES.flatMap((source) => [...DENSE_INTRADAY_RETENTION_TYPES].map((type): RepairStep => ({ type, source, diff --git a/src/lib/jobs/queue-runtime.ts b/src/lib/jobs/queue-runtime.ts index 7d6a3ae9d..297405af8 100644 --- a/src/lib/jobs/queue-runtime.ts +++ b/src/lib/jobs/queue-runtime.ts @@ -51,7 +51,7 @@ import { MEDICATION_INTAKE_IMPORT_SEND_OPTIONS } from "@/lib/jobs/medication-int import { HEALTH_CONNECT_IMPORT_SEND_OPTIONS } from "@/lib/jobs/health-connect-import-worker"; import { MEASUREMENT_MAINTENANCE_SEND_OPTIONS } from "@/lib/jobs/measurement-maintenance"; import { PULSE_DAY_MEAN_SEND_OPTIONS } from "@/lib/jobs/pulse-day-mean"; -import { ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS } from "@/lib/rollups/measurement-rollups"; +import { ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS } from "@/lib/rollups/full-backfill"; import { MORNING_DIGEST_REFRESH_EXPIRE_SECONDS } from "@/lib/jobs/morning-digest-refresh-shared"; import { INSIGHT_STATUS_GENERATE_EXPIRE_SECONDS } from "@/lib/jobs/insight-status-generate-queue"; import { WORKOUT_INSIGHT_GENERATE_EXPIRE_SECONDS } from "@/lib/jobs/workout-insight-generate-shared"; @@ -261,7 +261,7 @@ export const QUEUE_RUNTIME: Readonly> = { fn: "handleRollupFullBackfill", }, exclusive: "lockedPass", - why: "One account's five-year fold: four aggregates under the statement timeout and four persist transactions of up to 120 s each, about thirteen minutes at worst under the defaults and unbounded when the operator lifts the statement timeout.", + why: "One account's five-year fold, a calendar year at a time with every statement under its own five-minute limit; the run stops at its budget between windows and hands the remaining years to a follow-up.", }, "mood-rollup-recompute": short("One bucket."), "mood-rollup-full-backfill": short( @@ -386,6 +386,9 @@ export const QUEUE_RUNTIME: Readonly> = { "recovery-readiness": short( "A few indexed reads, one row per account from two backup ledgers, and at most one notice per signal.", ), + "system-job-alert": short( + "The newest fifty terminal runs of each system queue over pg-boss's completion index, and at most one notice per queue.", + ), "integration-watchdog-hourly": short( "A few indexed reads per account with a connection and no provider call; a 14-day history count only for a silent source.", ), diff --git a/src/lib/jobs/registry/rollup.ts b/src/lib/jobs/registry/rollup.ts index cb885c0dc..b4e68e834 100644 --- a/src/lib/jobs/registry/rollup.ts +++ b/src/lib/jobs/registry/rollup.ts @@ -56,9 +56,11 @@ import { import { ROLLUP_FULL_BACKFILL_CONCURRENCY, ROLLUP_FULL_BACKFILL_QUEUE, + type RollupFullBackfillPayload, +} from "@/lib/rollups/full-backfill"; +import { ROLLUP_RECOMPUTE_CONCURRENCY, ROLLUP_RECOMPUTE_QUEUE, - type RollupFullBackfillPayload, } from "@/lib/rollups/measurement-rollups"; import { MOOD_ROLLUP_FULL_BACKFILL_CONCURRENCY, @@ -213,7 +215,7 @@ const discovery: readonly BootDiscovery[] = [ name: "rollup-full-backfill", run: async () => ( - await import("@/lib/rollups/measurement-rollups") + await import("@/lib/rollups/full-backfill") ).enqueueBootTimeRollupBackfill(BOOT_BACKFILL_STAGGER_SECONDS * 0), }, { diff --git a/src/lib/jobs/registry/status.ts b/src/lib/jobs/registry/status.ts index e9e651947..53505e081 100644 --- a/src/lib/jobs/registry/status.ts +++ b/src/lib/jobs/registry/status.ts @@ -4,7 +4,8 @@ * the morning digest refresh, the data-arrival spine with its reaction line * and workout insight, the nightly scores, the period-narrative warm, the * Coach memory refresh, nudge, reminder sweep and plan review, the low-stock - * pass and the daily-briefing fallback. + * pass, the daily-briefing fallback, and the hourly alert to the admins + * about a system queue that keeps failing. * * Most queues here are keyless cron ticks, so they carry no policy. The * LLM-bound warm queues (pre-generate, period narrative, Coach memory) pass @@ -77,6 +78,12 @@ import { WORKOUT_INSIGHT_GENERATE_QUEUE, } from "@/lib/jobs/workout-insight-generate-shared"; +import { cronIsTheRetry } from "@/lib/jobs/reminder/registrar-shared"; +import { + SYSTEM_JOB_ALERT_CRON, + SYSTEM_JOB_ALERT_QUEUE, +} from "@/lib/jobs/system-job-alert-queue"; + import { defineJob, type JobDefinition, type JobGroup } from "./define-job"; const GENERAL_STATUS_QUEUE = "insights-general-status"; @@ -261,6 +268,18 @@ const jobs: readonly JobDefinition[] = [ work: { localConcurrency: 1 }, load: async () => (await handlers()).handleDailyBriefing, }), + // Hourly: tell the admins about a system queue that keeps failing, the way + // the recovery readiness evaluation tells them about a backup. Single-flight + // and no retries; the next tick is the retry, and the notice claim keeps + // two evaluations from sending the same notice. + defineJob({ + queue: SYSTEM_JOB_ALERT_QUEUE, + cron: SYSTEM_JOB_ALERT_CRON, + scheduleOptions: cronIsTheRetry, + work: { localConcurrency: 1 }, + load: async () => + (await import("@/lib/jobs/system-job-alert")).handleSystemJobAlert, + }), ]; export const STATUS_GROUP: JobGroup = { diff --git a/src/lib/jobs/reminder/rollup-handlers.ts b/src/lib/jobs/reminder/rollup-handlers.ts index 74c765d02..d9bde2063 100644 --- a/src/lib/jobs/reminder/rollup-handlers.ts +++ b/src/lib/jobs/reminder/rollup-handlers.ts @@ -56,8 +56,14 @@ import { type CumulativePrRederivePayload, } from "@/lib/personal-records/cumulative-pr-rederivation"; import { - recomputeUserRollups, + foldUserRollupHistory, + isRollupFullBackfillPending, + ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + sendRollupFullBackfillContinuation, type RollupFullBackfillPayload, +} from "@/lib/rollups/full-backfill"; +import { + recomputeUserRollups, type RollupRecomputePayload, } from "@/lib/rollups/measurement-rollups"; import { @@ -91,35 +97,47 @@ export async function handleRollupRecompute( } /** - * The full fold of one uncovered account across the default window and every - * granularity. Each granularity commits on its own, so a run stopped on its - * budget throws and the retry folds the rest. + * The full fold of one account across the five-year window, a calendar year + * at a time and every statement under the full backfill's own limit (#1298). + * A run that reaches its job budget hands the years it did not reach to a + * follow-up and completes; only a run that could not finish even one window + * fails, and its retry starts that window again. */ export async function handleRollupFullBackfill( jobs: Job[], ): Promise { const shouldStop = jobBudget(jobs); let rowsUpsertedTotal = 0; + let windowsTotal = 0; + let continued = 0; for (const job of jobs) { - const { userId } = job.data; - const { rowsUpserted, durationMs, stopped } = await recomputeUserRollups( - userId, - { shouldStop }, - ); - if (stopped) { - throw new Error( - `rollup-full-backfill for user ${userId} stopped on its job budget; the retry folds the rest`, - ); + const { userId, resumeBefore } = job.data; + const done = await foldUserRollupHistory(userId, { + before: resumeBefore ? new Date(resumeBefore) : undefined, + shouldStop, + statementTimeoutMs: ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + }); + rowsUpsertedTotal += done.rowsUpserted; + windowsTotal += done.windowsFolded; + if (done.resumeBefore) { + if (done.windowsFolded === 0) { + throw new Error( + `rollup-full-backfill for user ${userId} stopped on its job budget before it folded a window; the retry starts that window again`, + ); + } + await sendRollupFullBackfillContinuation(userId, done.resumeBefore); + continued += 1; } - rowsUpsertedTotal += rowsUpserted; workerLog( "info", - `[rollup-full-backfill] user=${userId} rows=${rowsUpserted} duration=${durationMs}ms`, + `[rollup-full-backfill] user=${userId} rows=${done.rowsUpserted} windows=${done.windowsFolded} duration=${done.durationMs}ms${done.resumeBefore ? ` continuesBefore=${done.resumeBefore.toISOString()}` : ""}`, ); } return jobDone({ users: jobs.length, rows_upserted: rowsUpsertedTotal, + windows_folded: windowsTotal, + continued, }); } @@ -510,6 +528,7 @@ export function drainCumulativeHandler( { dryRun: false, shouldStop, + deferUser: isRollupFullBackfillPending, log: (line) => workerLog("info", line), }, ); diff --git a/src/lib/jobs/subsystem-surface.ts b/src/lib/jobs/subsystem-surface.ts index 419541cca..681cc2250 100644 --- a/src/lib/jobs/subsystem-surface.ts +++ b/src/lib/jobs/subsystem-surface.ts @@ -67,6 +67,7 @@ const subsystemSurface = { "coach-plan-review": { audience: "system" }, "medication-low-stock": { audience: "system" }, "daily-briefing": { audience: "system" }, + "system-job-alert": { audience: "system" }, // register-rollup.ts "rollup-recompute": { audience: "system" }, diff --git a/src/lib/jobs/system-job-alert-queue.ts b/src/lib/jobs/system-job-alert-queue.ts new file mode 100644 index 000000000..2671edd0e --- /dev/null +++ b/src/lib/jobs/system-job-alert-queue.ts @@ -0,0 +1,12 @@ +/** + * The queue facts of `system-job-alert.ts`: its name and schedule. They live + * apart from the job so the worker registry can read them without loading it. + */ +export const SYSTEM_JOB_ALERT_QUEUE = "system-job-alert"; + +/** + * Hourly at :43, a minute no other schedule uses. A notice goes out once per + * change whatever the cadence, and an hour is soon enough to hear that a + * background job keeps failing. + */ +export const SYSTEM_JOB_ALERT_CRON = "43 * * * *"; diff --git a/src/lib/jobs/system-job-alert.ts b/src/lib/jobs/system-job-alert.ts new file mode 100644 index 000000000..d6174eec7 --- /dev/null +++ b/src/lib/jobs/system-job-alert.ts @@ -0,0 +1,303 @@ +/** + * Push to every admin when a background job of the instance keeps failing: + * once when a queue has failed `SYSTEM_JOB_FAILURE_STREAK` runs in a row, a + * reminder at most once a day while it keeps failing, and once when it runs + * successfully again. Nothing else. + * + * Why (#1298). A full rollup backfill that failed on every boot showed only + * on the admin page's card of failing background jobs. From the account's + * side the only symptom was empty dashboard cards, and nothing pointed an + * operator at the card. The recovery readiness evaluation already pushes a + * stopped backup; this does the same for the queues whose failure has no + * account-facing status of its own (the `system` audience in + * `subsystem-surface.ts`). + * + * The streak is read from pg-boss's own ledger, newest terminal run first: a + * run that failed (or completed and said every candidate failed, + * `did.all_failed`) adds its attempts, retries included, and the first run + * that succeeded ends the streak. A queue with no terminal run left in the + * ledger is not judged at all. + * + * The memory is one row per announced queue (`SystemJobAlertNotice`), claimed + * with a compare-and-set before the notice is sent, exactly as the readiness + * notices are (`readiness-notify.ts`, whose decision this reuses), so two + * workers send a notice once and a crash between the claim and the send loses + * a notice rather than repeating it. Delivery is the ordinary localised path, + * as the other operator alerts use: a `SYSTEM_ALERT` to every admin, in their + * locale, on the channels they enabled. + */ +import type { Job } from "pg-boss"; + +import { ACTIVE_ACCOUNT } from "@/lib/account-deletion/active-account"; +import { prisma } from "@/lib/db"; +import { presentError } from "@/lib/jobs/job-failures"; +import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome"; +import { SUBSYSTEM_SURFACE } from "@/lib/jobs/subsystem-surface"; +import { withBackgroundEvent } from "@/lib/logging/background"; +import { dispatchLocalisedNotification } from "@/lib/notifications/dispatch-localised"; +import { + decideNotice, + type NoticeKind, + type StoredNotice, +} from "@/lib/recovery/readiness-notify"; + +/** Failed runs in a row, retries included, before the admins are told. */ +export const SYSTEM_JOB_FAILURE_STREAK = 3; + +/** Newest terminal runs read per queue. */ +const LOOKBACK_ROWS = 50; + +/** Where the notice takes the admin: the page with the failing-jobs card. */ +const ADMIN_STATUS_PATH = "/admin/system-status"; + +/** One terminal run as the ledger holds it, newest first. */ +export interface TerminalRun { + state: "completed" | "failed"; + /** Retries pg-boss ran before this run settled. */ + retryCount: number; + /** The run completed but reported that every candidate failed. */ + allFailed: boolean; + completedOn: Date; + error: string | null; + expireSeconds: number | null; +} + +/** What the ledger says about one queue. */ +export interface QueueVerdict { + /** + * `failing`: the streak reached the threshold. `ok`: the newest run, or a + * run after the last failures, succeeded. `pending`: fewer failures than + * the threshold and no success to judge by. `inactive`: no run left. + */ + state: "failing" | "ok" | "pending" | "inactive"; + /** Failed attempts in a row, newest first. */ + failures: number; + /** The newest failure's message, redacted and truncated. */ + lastError: string; +} + +/** Pure: judge one queue's newest terminal runs. */ +export function judgeQueueRuns(runs: readonly TerminalRun[]): QueueVerdict { + if (runs.length === 0) + return { state: "inactive", failures: 0, lastError: "" }; + let failures = 0; + let lastError: string | null = null; + let succeeded = false; + for (const run of runs) { + if (run.state === "completed" && !run.allFailed) { + succeeded = true; + break; + } + failures += run.retryCount + 1; + lastError ??= presentError(run.error, run.expireSeconds); + } + const state = + failures >= SYSTEM_JOB_FAILURE_STREAK + ? "failing" + : succeeded + ? "ok" + : "pending"; + return { state, failures, lastError: lastError ?? "" }; +} + +/** The queues this alert watches: every one whose failure has no other surface. */ +export function watchedQueues(): string[] { + return Object.entries(SUBSYSTEM_SURFACE) + .filter(([, surface]) => surface.audience === "system") + .map(([queue]) => queue); +} + +/** + * The newest terminal runs of each queue, newest first, one statement over + * pg-boss's `(name, completed_on)` index. `null` when the queue schema is not + * there (a web-only deployment): no queue to ask is not a healthy queue. + */ +export async function readTerminalRuns( + queues: readonly string[], +): Promise | null> { + try { + const rows = await prisma.$queryRaw< + Array<{ + name: string; + state: string; + retry_count: number; + all_failed: string | null; + completed_on: Date; + error: string | null; + expire_seconds: number | null; + }> + >` + SELECT q.name, j.state, j.retry_count, j.all_failed, j.completed_on, + j.error, j.expire_seconds + FROM unnest(${queues as string[]}::text[]) AS q(name) + CROSS JOIN LATERAL ( + SELECT state, retry_count, completed_on, expire_seconds, + output->'did'->>'all_failed' AS all_failed, + COALESCE( + output->>'message', + output->'value'->>'message', + output::text + ) AS error + FROM pgboss.job + WHERE name = q.name + AND completed_on IS NOT NULL + AND state IN ('completed', 'failed') + ORDER BY completed_on DESC + LIMIT ${LOOKBACK_ROWS} + ) j + ORDER BY q.name, j.completed_on DESC + `; + const byQueue = new Map(); + for (const row of rows) { + const runs = byQueue.get(row.name) ?? []; + runs.push({ + state: row.state === "failed" ? "failed" : "completed", + retryCount: Number(row.retry_count ?? 0), + allFailed: row.all_failed === "true", + completedOn: row.completed_on, + error: row.error, + expireSeconds: row.expire_seconds, + }); + byQueue.set(row.name, runs); + } + return byQueue; + } catch { + return null; + } +} + +function titleKey(kind: NoticeKind): string { + if (kind === "recovered") + return "notifications.systemJobAlert.titleRecovered"; + if (kind === "reminder") return "notifications.systemJobAlert.titleReminder"; + return "notifications.systemJobAlert.titleFailing"; +} + +export interface SystemJobAlertOutcome { + /** Queues at or past the threshold now. */ + failing: number; + /** Notices claimed and sent, by queue. */ + sent: Array<{ queue: string; kind: NoticeKind }>; + /** Admins each notice went to. */ + admins: number; +} + +/** + * Judge every watched queue against what the admins were last told, claim + * every notice that is due, and send it. + */ +export async function notifySystemJobFailures( + now: Date = new Date(), +): Promise { + const outcome: SystemJobAlertOutcome = { failing: 0, sent: [], admins: 0 }; + const runs = await readTerminalRuns(watchedQueues()); + if (runs === null) return outcome; + + // account-deletion: one row per queue of the instance; no account in it. + const rows = await prisma.systemJobAlertNotice.findMany(); + const byQueue = new Map(rows.map((row) => [row.queue, row])); + const admins = await prisma.user.findMany({ + where: { role: "ADMIN", ...ACTIVE_ACCOUNT }, + select: { id: true }, + }); + outcome.admins = admins.length; + + for (const queue of watchedQueues()) { + const verdict = judgeQueueRuns(runs.get(queue) ?? []); + if (verdict.state === "failing") outcome.failing += 1; + // Too few failures to call it a streak and no success to call it over. + if (verdict.state === "pending") continue; + + const row = byQueue.get(queue); + const stored: StoredNotice | null = row + ? { + notifiedState: + row.notifiedState === "failing" || row.notifiedState === "ok" + ? row.notifiedState + : null, + notifiedAt: row.notifiedAt, + } + : null; + const decision = decideNotice(stored, verdict.state, now); + // With no admin yet there is nobody to tell, so nothing is claimed: the + // first admin hears about a queue that is still failing. + if (!decision.next || admins.length === 0) continue; + + // Claim: only the evaluation that moves the row from what it read sends. + const claimed = row + ? // account-deletion: the one queue row read above; no account in it. + ( + await prisma.systemJobAlertNotice.updateMany({ + where: { + queue, + notifiedState: row.notifiedState, + notifiedAt: row.notifiedAt, + }, + data: { + notifiedState: decision.next.notifiedState, + notifiedAt: decision.next.notifiedAt, + }, + }) + ).count + : ( + await prisma.systemJobAlertNotice.createMany({ + data: [ + { + queue, + notifiedState: decision.next.notifiedState, + notifiedAt: decision.next.notifiedAt, + }, + ], + skipDuplicates: true, + }) + ).count; + if (claimed !== 1 || decision.send === null) continue; + + for (const admin of admins) { + await dispatchLocalisedNotification({ + userId: admin.id, + eventType: "SYSTEM_ALERT", + titleKey: titleKey(decision.send), + messageKey: + decision.send === "recovered" + ? "notifications.systemJobAlert.bodyRecovered" + : "notifications.systemJobAlert.bodyFailing", + params: { + queue, + failures: verdict.failures, + error: verdict.lastError, + }, + metadata: { + source: "system-job-alert", + url: ADMIN_STATUS_PATH, + queue, + kind: decision.send, + }, + }); + } + outcome.sent.push({ queue, kind: decision.send }); + } + return outcome; +} + +/** The hourly evaluation. Single-flight; the next tick is the retry. */ +export async function handleSystemJobAlert( + jobs: Job[], +): Promise { + void jobs; + return withBackgroundEvent("job.system_job_alert", async (evt) => { + try { + const outcome = await notifySystemJobFailures(); + const did = { + system_jobs_failing: outcome.failing, + system_job_notices: outcome.sent.length, + system_job_admins: outcome.admins, + }; + for (const [key, value] of Object.entries(did)) evt.addMeta(key, value); + return jobDone(did); + } catch (err) { + evt.addWarning(`system-job-alert failed: ${err}`); + return jobFailed("system job alert evaluation failed", err); + } + }); +} diff --git a/src/lib/measurements/__tests__/dense-intraday-drain-exemption.test.ts b/src/lib/measurements/__tests__/dense-intraday-drain-exemption.test.ts index fb0311aa7..5878092f0 100644 --- a/src/lib/measurements/__tests__/dense-intraday-drain-exemption.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-drain-exemption.test.ts @@ -26,7 +26,7 @@ import { CUMULATIVE_HK_TYPES, HIGH_FREQUENCY_MEAN_TYPES, } from "../apple-health-mapping"; -import { DENSE_INTRADAY_RETENTION_TYPES } from "../dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_TYPES } from "../dense-intraday-sources"; describe("dense intra-day retention — drain exemption (WX-E)", () => { it("EXEMPTS HEART_RATE_VARIABILITY from the destructive daily-mean drain", () => { diff --git a/src/lib/measurements/__tests__/dense-intraday-retention.test.ts b/src/lib/measurements/__tests__/dense-intraday-retention.test.ts index 2d76f13ae..8d44ef417 100644 --- a/src/lib/measurements/__tests__/dense-intraday-retention.test.ts +++ b/src/lib/measurements/__tests__/dense-intraday-retention.test.ts @@ -21,8 +21,8 @@ vi.mock("@/lib/rollups/measurement-rollups", () => ({ import { runDenseIntradayRetention, DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_RETENTION_TYPES, } from "../dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_TYPES } from "../dense-intraday-sources"; import type { PerSampleRow } from "../drain-per-sample-cumulative"; import { foldBoundary } from "../consolidation-tz"; import type { MeasurementType, PrismaClient } from "@/generated/prisma/client"; @@ -161,7 +161,17 @@ describe("runDenseIntradayRetention — retention bound", () => { } }); - it("scans exactly the dense-tier types, once per fold source, Apple Health first", async () => { + it("leaves an account alone while its rollup tier is being rebuilt", async () => { + // A full rollup backfill of the account still has to read the raw + // samples; folding them first would leave it only hourly means (#1298). + const { mock, findManyMeasurement } = buildPrismaMock({}); + const deferUser = vi.fn(async () => true); + await runDenseIntradayRetention(mock, { deferUser, log: () => {} }); + expect(deferUser).toHaveBeenCalled(); + expect(findManyMeasurement).not.toHaveBeenCalled(); + }); + + it("scans each fold source's dense types, once per source, Apple Health first", async () => { const { mock, findManyMeasurement } = buildPrismaMock({}); await runDenseIntradayRetention(mock, { log: () => {} }); @@ -172,9 +182,13 @@ describe("runDenseIntradayRetention — retention bound", () => { const types = Array.from(DENSE_INTRADAY_RETENTION_TYPES); // One walk per source, each scoped to its one source: two sources // never share a mean, and manual or Withings rows are never scanned. + // The provider syncs fold only what they store sample by sample: heart + // rate, never their daily SpO2 (#1297). expect(scans).toEqual([ ...types.map((type) => `APPLE_HEALTH:${type}`), ...types.map((type) => `HEALTH_CONNECT:${type}`), + "GOOGLE_HEALTH:PULSE", + "OURA:PULSE", ]); }); }); diff --git a/src/lib/measurements/__tests__/folded-window.test.ts b/src/lib/measurements/__tests__/folded-window.test.ts index d83a3d464..4b36a5b98 100644 --- a/src/lib/measurements/__tests__/folded-window.test.ts +++ b/src/lib/measurements/__tests__/folded-window.test.ts @@ -14,6 +14,7 @@ import { findCompactionTombstones, findFoldedWindowDuplicates, isFoldedWindowCandidate, + withoutFoldedSamples, } from "../folded-window"; const DAY = 86_400_000; @@ -332,3 +333,95 @@ describe("Health Connect", () => { ]); }); }); + +describe("provider syncs (#1297)", () => { + const old = new Date("2026-03-01T10:15:00.000Z"); + const hour = "stats:HKQuantityTypeIdentifierHeartRate:2026-03-01T10"; + + it("takes the heart rate Google Health and Oura store per sample, and nothing else of theirs", () => { + for (const source of ["GOOGLE_HEALTH", "OURA"]) { + expect(isFoldedWindowCandidate(raw("PULSE", old, { source }), NOW)).toBe( + true, + ); + // A nightly or daily SpO2 value is no intraday stream: never folded. + expect( + isFoldedWindowCandidate(raw("OXYGEN_SATURATION", old, { source }), NOW), + ).toBe(false); + } + // Neither stores intraday heart rate as samples. + for (const source of ["FITBIT", "WHOOP", "WITHINGS"]) { + expect(isFoldedWindowCandidate(raw("PULSE", old, { source }), NOW)).toBe( + false, + ); + } + }); + + it("drops the samples of an hour the fold took, under the write's own source", async () => { + // The externalId read finds the hour's row; the reach read finds nothing. + const findMany = vi.fn( + async (args: { + where: { source: string; externalId: { in?: string[] } }; + }) => + args.where.source === "GOOGLE_HEALTH" && + args.where.externalId.in?.includes(hour) + ? [{ type: "PULSE", externalId: hour }] + : [], + ); + const findUnique = vi.fn(async () => ({ timezone: "UTC" })); + const rows = [ + { type: "PULSE", externalId: "hr:a", measuredAt: old, value: 61 }, + { + type: "PULSE", + externalId: "hr:b", + measuredAt: new Date("2026-03-01T12:15:00.000Z"), + value: 62, + }, + { + type: "PULSE", + externalId: "hr:c", + measuredAt: new Date(NOW.getTime() - DAY), + value: 63, + }, + ]; + const { kept, dropped } = await withoutFoldedSamples( + { + measurement: { findMany }, + user: { findUnique }, + } as unknown as Parameters[0], + "u1", + "GOOGLE_HEALTH", + rows, + { now: NOW }, + ); + expect(dropped).toBe(1); + expect(kept.map((row) => row.externalId)).toEqual(["hr:b", "hr:c"]); + for (const call of findMany.mock.calls) { + expect(call[0].where.source).toBe("GOOGLE_HEALTH"); + } + }); + + it("asks nothing for a write of recent samples", async () => { + const findMany = vi.fn(); + const findUnique = vi.fn(); + const rows = [ + { + type: "PULSE", + externalId: "heartrate:x", + measuredAt: new Date(NOW.getTime() - 2 * 3_600_000), + }, + ]; + const { kept } = await withoutFoldedSamples( + { + measurement: { findMany }, + user: { findUnique }, + } as unknown as Parameters[0], + "u1", + "OURA", + rows, + { now: NOW }, + ); + expect(kept).toEqual(rows); + expect(findMany).not.toHaveBeenCalled(); + expect(findUnique).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/measurements/consolidation-base.ts b/src/lib/measurements/consolidation-base.ts index c74e79e52..5cab6ae06 100644 --- a/src/lib/measurements/consolidation-base.ts +++ b/src/lib/measurements/consolidation-base.ts @@ -76,6 +76,14 @@ export interface ConsolidationOptions { * day this one did not reach. */ shouldStop?: () => boolean; + /** + * Asked once per account before it is walked; `true` leaves the account + * for a later run, and nothing of it is read or written. The dense fold + * passes `isRollupFullBackfillPending`: while an account's rollup tier is + * rebuilt from its raw readings, folding those readings would leave the + * rebuild only hourly means to take the old days' figures from. + */ + deferUser?: (userId: string) => Promise; } /** @@ -458,6 +466,10 @@ export async function runConsolidation( const users = await loadConsolidationUsers(prismaClient, options.userId); walk: for (const user of users) { + if (await options.deferUser?.(user.id)) { + annotate({ meta: { consolidation_deferred_user: true } }); + continue; + } // The zone the server publishes for the account (`/api/auth/me`), which // is the zone the phone cuts its `stats:` days and hour buckets in and // the one the hour-bucket writer stores them under. A stored value the diff --git a/src/lib/measurements/dense-intraday-hourly-rebuild.ts b/src/lib/measurements/dense-intraday-hourly-rebuild.ts index 0c49f4c61..6f34f3d0a 100644 --- a/src/lib/measurements/dense-intraday-hourly-rebuild.ts +++ b/src/lib/measurements/dense-intraday-hourly-rebuild.ts @@ -72,11 +72,11 @@ import { localDayWindow } from "@/lib/tz/local-day"; import { meanBucketValue } from "./consolidate-daily-mean"; import { DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_RETENTION_TYPES, adoptOrMintHourlyRows, bucketRowsByLocalHour, hourlyStatsExternalId, } from "./dense-intraday-retention"; +import { DENSE_INTRADAY_RETENTION_TYPES } from "./dense-intraday-sources"; import { bucketSpan, enqueueRollupRecompute, diff --git a/src/lib/measurements/dense-intraday-retention.ts b/src/lib/measurements/dense-intraday-retention.ts index fd7a1babc..e75cd7546 100644 --- a/src/lib/measurements/dense-intraday-retention.ts +++ b/src/lib/measurements/dense-intraday-retention.ts @@ -56,7 +56,6 @@ import { FOLD_TRANSACTION_OPTIONS, } from "@/lib/export/restore-lock"; import type { - MeasurementSource, MeasurementType, Prisma, PrismaClient, @@ -67,7 +66,11 @@ import { dailyStatsExternalId, hkIdentifierForType, } from "./apple-health-mapping"; -import { runConsolidation, type DayWriteOutcome } from "./consolidation-base"; +import { + runConsolidation, + type ConsolidationOptions, + type DayWriteOutcome, +} from "./consolidation-base"; import { canonicalHourlyTimestamp, hourOfDayForUserTz, @@ -78,6 +81,11 @@ import { percentile } from "@/lib/insights/strain-score"; import type { PerSampleRow } from "./drain-per-sample-cumulative"; import { DEFAULT_TIMEZONE } from "@/lib/tz/format"; import { localDayWindow } from "@/lib/tz/local-day"; +import { + DENSE_INTRADAY_FOLD_SOURCES, + DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE, + type DenseIntradayFoldSource, +} from "./dense-intraday-sources"; import { absorbIntoFold, foldConstituentHorizon, @@ -121,40 +129,6 @@ import { * they did against the raw stream's successor rows before. */ -/** - * The dense-tier types. These are the high-frequency intra-day signals that - * accumulate raw per-sample rows the destructive daily-mean drain never - * touches: - * - * - `HEART_RATE_VARIABILITY` + `PULSE` — the daytime shape the Stress - * engine reads (v1.10.0 / v1.18.11). - * - `OXYGEN_SATURATION` — Apple Watch's Blood-Oxygen app samples SpO2 - * periodically through the day and overnight; the readings map to - * `aggregation: "latest"` and sit in NEITHER `CUMULATIVE_HK_TYPES` nor - * `HIGH_FREQUENCY_MEAN_TYPES`, so every sample piled up raw forever. - * Folded here to hourly-mean rows with the daily min/max preserved on - * the rollup tier (the lowest sample is the overnight-desaturation - * nadir — clinically the meaningful figure, not the mean). - * - * Every member is deliberately EXEMPT from the destructive daily-mean drain - * (`HIGH_FREQUENCY_MEAN_TYPES`); the disjointness from that set is the - * load-bearing invariant a regression test pins — a type in both would be - * folded by two drains at once and corrupt the value. - * - * `RESPIRATORY_RATE` is deliberately NOT here: it is already a member of - * `HIGH_FREQUENCY_MEAN_TYPES`, so the nightly mean-consolidation already - * collapses it to a daily-mean row. Adding it here would double-fold it. - * - * One fold grain per tier: every member folds HOURLY. A per-type grain - * split would fork the write path and the rebuild for no reader benefit. - */ -export const DENSE_INTRADAY_RETENTION_TYPES: ReadonlySet = - new Set([ - "HEART_RATE_VARIABILITY", - "PULSE", - "OXYGEN_SATURATION", - ]); - /** * Retention bound (days). Per-sample dense-tier rows older than this window * fold to hourly-mean rows and the raw rows are deleted; rows inside the @@ -173,23 +147,6 @@ export const DENSE_INTRADAY_RETENTION_TYPES: ReadonlySet = */ export const DENSE_INTRADAY_RETENTION_DAYS = 90; -/** - * The sources whose raw dense samples fold (v1.42). Apple Health from the - * start; Health Connect because its importer writes the samples of the last - * 90 days raw, exactly like an Apple Health sync, and without a fold every - * new export would add another quarter of minute-by-minute heart rate that - * never ages into hourly means. Each source folds into its OWN `stats:` rows, - * so an hour two sources both covered keeps one mean per source, the same - * way the raw samples kept their source. - */ -export const DENSE_INTRADAY_FOLD_SOURCES = [ - "APPLE_HEALTH", - "HEALTH_CONNECT", -] as const satisfies readonly MeasurementSource[]; - -export type DenseIntradayFoldSource = - (typeof DENSE_INTRADAY_FOLD_SOURCES)[number]; - /** The stats externalId prefix marks an already-collapsed row. */ const DAILY_STATS_PREFIX = "stats:"; @@ -485,13 +442,8 @@ export interface DenseIntradayRetentionSummary { }; } -export interface DenseIntradayRetentionOptions { - /** Limit the pass to a single user. Default = every user. */ - userId?: string; - /** Preview-only mode — no DB writes. */ - dryRun?: boolean; - /** Logger sink — defaults to `console.log`. */ - log?: (line: string) => void; +/** The shared drain options (`ConsolidationOptions`) and the retention window. */ +export interface DenseIntradayRetentionOptions extends ConsolidationOptions { /** * Retention window in days. Per-sample rows OLDER than `now() - * retentionDays` are folded; rows inside the window stay raw. Defaults to @@ -500,11 +452,6 @@ export interface DenseIntradayRetentionOptions { * full-collapse, never by the scheduled pass. */ retentionDays?: number; - /** - * Ends the pass cleanly before the next day when it returns `true`; the - * summary then reports `stoppedEarly`. See `ConsolidationOptions`. - */ - shouldStop?: () => boolean; } /** @@ -590,7 +537,7 @@ export async function runDenseIntradayRetention( await runConsolidation({ prismaClient, options: { ...options, cutoffHours }, - types: DENSE_INTRADAY_RETENTION_TYPES, + types: DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE[source], hkIdentifierForType, dailyStatsExternalId, statsPrefix: DAILY_STATS_PREFIX, @@ -1034,8 +981,8 @@ export async function runDenseIntradayRetention( }, }); - // One walk per source, Apple Health first so its behaviour and its order - // are exactly what they were before Health Connect folded too. A walk that + // One walk per source, in the map's order (Apple Health first, so its + // behaviour is what it was before other sources folded too). A walk that // stopped early stops the pass: the next run resumes where it left off. for (const source of DENSE_INTRADAY_FOLD_SOURCES) { const walk = await walkSource(source); diff --git a/src/lib/measurements/dense-intraday-sources.ts b/src/lib/measurements/dense-intraday-sources.ts new file mode 100644 index 000000000..3c1891e54 --- /dev/null +++ b/src/lib/measurements/dense-intraday-sources.ts @@ -0,0 +1,86 @@ +/** + * What the dense hourly fold (`dense-intraday-retention.ts`) takes: the + * dense-tier types, and for each source the types it stores one row per + * intraday sample of. The `folded_window` ingest guard (`folded-window.ts`) + * reads the same map, so a source's raw samples are refused at ingest for + * exactly the hours the fold has taken from it. + */ +import type { + MeasurementSource, + MeasurementType, +} from "@/generated/prisma/client"; + +/** + * The dense-tier types. These are the high-frequency intra-day signals that + * accumulate raw per-sample rows the destructive daily-mean drain never + * touches: + * + * - `HEART_RATE_VARIABILITY` + `PULSE` — the daytime shape the Stress + * engine reads. + * - `OXYGEN_SATURATION` — Apple Watch's Blood-Oxygen app samples SpO2 + * periodically through the day and overnight; the readings map to + * `aggregation: "latest"` and sit in NEITHER `CUMULATIVE_HK_TYPES` nor + * `HIGH_FREQUENCY_MEAN_TYPES`, so every sample piled up raw forever. + * Folded here to hourly-mean rows with the daily min/max preserved on + * the rollup tier (the lowest sample is the overnight-desaturation + * nadir — clinically the meaningful figure, not the mean). + * + * Every member is deliberately EXEMPT from the destructive daily-mean drain + * (`HIGH_FREQUENCY_MEAN_TYPES`); the disjointness from that set is the + * load-bearing invariant a regression test pins — a type in both would be + * folded by two drains at once and corrupt the value. + * + * `RESPIRATORY_RATE` is deliberately NOT here: it is already a member of + * `HIGH_FREQUENCY_MEAN_TYPES`, so the nightly mean-consolidation already + * collapses it to a daily-mean row. Adding it here would double-fold it. + * + * One fold grain per tier: every member folds HOURLY. A per-type grain + * split would fork the write path and the rebuild for no reader benefit. + */ +export const DENSE_INTRADAY_RETENTION_TYPES: ReadonlySet = + new Set([ + "HEART_RATE_VARIABILITY", + "PULSE", + "OXYGEN_SATURATION", + ]); + +/** + * The sources whose raw dense samples fold, and which dense types of each. + * A source belongs here only for a type it stores sample by sample through + * the day; a type it stores as one value a day (a nightly SpO2 average, a + * recovery's HRV) is no intraday stream, and folding that row would only + * move it to another hour. + * + * - Apple Health and Health Connect: every dense type. Both write the + * samples of an export raw, and without the fold every new export would + * add another quarter of minute-by-minute heart rate. + * - Google Health: heart rate, one row per sample from the `heart-rate` + * collection (SpO2 comes from the daily collection). A full-history + * backfill brought tens of millions of these onto one account (#1297). + * - Oura: heart rate, a reading every few minutes from the `heartrate` + * collection (SpO2 is the nightly average). + * + * Fitbit does not sync intraday heart rate, and WHOOP, Withings and Polar + * write dense types only as one value per recovery, night or spot check, so + * none of them is here. + * + * Each source folds into its OWN `stats:` rows, so an hour two sources both + * covered keeps one mean per source, the same way the raw samples kept their + * source. The walk runs in this order. + */ +export const DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE = { + APPLE_HEALTH: DENSE_INTRADAY_RETENTION_TYPES, + HEALTH_CONNECT: DENSE_INTRADAY_RETENTION_TYPES, + GOOGLE_HEALTH: new Set(["PULSE"]), + OURA: new Set(["PULSE"]), +} as const satisfies Partial< + Record> +>; + +export type DenseIntradayFoldSource = + keyof typeof DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE; + +/** Every source the fold walks, in walk order. */ +export const DENSE_INTRADAY_FOLD_SOURCES = Object.keys( + DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE, +) as readonly DenseIntradayFoldSource[]; diff --git a/src/lib/measurements/folded-window.ts b/src/lib/measurements/folded-window.ts index 9fa41a6e1..eb84dfb2c 100644 --- a/src/lib/measurements/folded-window.ts +++ b/src/lib/measurements/folded-window.ts @@ -47,12 +47,14 @@ * of tombstones does not use that reach; a tombstone it misses stays class B, * which keeps a row longer, never loses one. * - * Health Connect folds its raw heart-rate history the same way under its own - * source (the importer writes samples of the last 90 days raw, and the - * nightly pass folds them once they age out), so the hour rule covers both - * sources and a covering row is always looked up under the sample's own - * source. The day rule stays Apple Health only: the nightly mean - * consolidation reads no other source. + * Every source the dense fold walks folds under its own source, for the + * types it stores sample by sample (`DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE`): + * Health Connect from its importer, and Google Health and Oura from their + * syncs, whose backfills walk years of heart rate (#1297). So there is one + * hour rule per source, and a covering row is always looked up under the + * sample's own source. The provider syncs ask the same question before they + * write (`withoutFoldedSamples`). The day rule stays Apple Health only: the + * nightly mean consolidation reads no other source. */ import type { MeasurementSource, @@ -71,11 +73,14 @@ import { hourOfDayForUserTz, } from "@/lib/measurements/consolidation-tz"; import { - DENSE_INTRADAY_FOLD_SOURCES, DENSE_INTRADAY_RETENTION_DAYS, - DENSE_INTRADAY_RETENTION_TYPES, hourlyStatsExternalId, } from "@/lib/measurements/dense-intraday-retention"; +import { + DENSE_INTRADAY_FOLD_SOURCES, + DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE, + DENSE_INTRADAY_RETENTION_TYPES, +} from "@/lib/measurements/dense-intraday-sources"; /** The per-entry status reason the batch result carries for such a row. */ export type FoldedWindowReason = "folded_window"; @@ -88,33 +93,38 @@ const HOUR_MS = 3_600_000; * `reachMs` is how far from a sample the anchor of its window can sit in any * zone (see the module comment), for the timezone-change check. */ -export const FOLDED_WINDOW_RULES = [ - { +export const FOLDED_WINDOW_RULES: readonly FoldedWindowRule[] = [ + // One hour rule per source the dense fold walks, with that source's types. + ...DENSE_INTRADAY_FOLD_SOURCES.map((source): FoldedWindowRule => ({ window: "hour", - sources: DENSE_INTRADAY_FOLD_SOURCES, - types: DENSE_INTRADAY_RETENTION_TYPES, + sources: [source], + types: DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE[source], foldedAfterMs: DENSE_INTRADAY_RETENTION_DAYS * 24 * HOUR_MS, reachMs: HOUR_MS / 2, - }, + })), { window: "day", - sources: ["APPLE_HEALTH"] as const, + sources: ["APPLE_HEALTH"], types: HIGH_FREQUENCY_MEAN_TYPES, foldedAfterMs: CONSOLIDATION_GRACE_CUTOFF_HOURS * HOUR_MS, reachMs: 14 * HOUR_MS, }, -] as const; +]; -type FoldedWindowRule = (typeof FOLDED_WINDOW_RULES)[number]; +interface FoldedWindowRule { + window: "hour" | "day"; + sources: readonly MeasurementSource[]; + types: ReadonlySet; + foldedAfterMs: number; + reachMs: number; +} function ruleFor( row: Pick, ): FoldedWindowRule | null { return ( FOLDED_WINDOW_RULES.find( - (rule) => - (rule.sources as readonly MeasurementSource[]).includes(row.source) && - rule.types.has(row.type), + (rule) => rule.sources.includes(row.source) && rule.types.has(row.type), ) ?? null ); } @@ -409,6 +419,46 @@ export async function findFoldedWindowDuplicates( ); } +/** + * The rows of one provider write without the raw samples whose hour the + * dense fold has already taken from that source (#1297). + * + * The provider syncs write first-write-wins on `(user, type, source, + * externalId)`, and the fold deletes the samples it averages, so a backfill + * or a full sync that walks old history would find no row for a folded + * sample and insert it again beside the hourly mean that already accounts + * for it. Google Health would even re-key the hourly row: a sample that fell + * on the hour's anchor shares its instant, and the sync's natural-key rescue + * then overwrote the mean with the sample. Rows too young to have been folded + * cost no query, so the hourly syncs never ask. + */ +export async function withoutFoldedSamples< + T extends { type: string; externalId?: string | null; measuredAt: Date }, +>( + client: LookupClient, + userId: string, + source: MeasurementSource, + rows: readonly T[], + options: { now?: Date } = {}, +): Promise<{ kept: T[]; dropped: number }> { + const folded = await findFoldedWindowDuplicates( + client, + userId, + rows.map((row) => ({ + type: row.type as MeasurementType, + source, + externalId: row.externalId ?? null, + measuredAt: row.measuredAt, + })), + options, + ); + if (folded.size === 0) return { kept: [...rows], dropped: 0 }; + return { + kept: rows.filter((_, index) => !folded.has(index)), + dropped: folded.size, + }; +} + /** A tombstone as the classifier reads it. */ export interface TombstoneCandidateRow extends FoldedWindowCandidateRow { deletedAt: Date | null; diff --git a/src/lib/oura/sync-heart-rate.ts b/src/lib/oura/sync-heart-rate.ts index 9c0a9a1f4..f2b8ff4b0 100644 --- a/src/lib/oura/sync-heart-rate.ts +++ b/src/lib/oura/sync-heart-rate.ts @@ -11,14 +11,16 @@ * Writes are first-write-wins: a sample is final once it is in Oura's cloud, * and it is keyed on its own instant (`heartrate:`), so a re-read of the * same range is an insert that `skipDuplicates` drops. A row the person - * deleted keeps its tombstone; the re-read does not bring it back. Oura's - * samples go raw, like Google Health's and Fitbit's minute stream: the dense - * hourly fold covers only the export sources (`DENSE_INTRADAY_FOLD_SOURCES`). + * deleted keeps its tombstone; the re-read does not bring it back. Samples + * older than the retention window are folded into hourly means like Google + * Health's (`DENSE_INTRADAY_FOLD_TYPES_BY_SOURCE`), and a re-read of a folded + * hour writes nothing (`withoutFoldedSamples`). */ import { prisma } from "@/lib/db"; import type { MeasurementType } from "@/generated/prisma/client"; import { emitInsertedMeasurementArrivals } from "@/lib/arrivals/measurement-emit"; import { dropImplausibleMeasurements } from "@/lib/measurements/plausibility-gate"; +import { withoutFoldedSamples } from "@/lib/measurements/folded-window"; import { collapseToTypeDayKeys } from "@/lib/rollups/measurement-rollups"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; import { forEachHeartRatePage } from "./client"; @@ -76,10 +78,17 @@ export async function writeOuraHeartRatePage( const m = mapHeartRateSample(s); if (m) mapped.push(m); } - const kept = dropImplausibleMeasurements("oura", mapped, (m) => ({ - type: m.type, - value: m.value, - })); + // A sample of an hour the dense fold already took is not written again: it + // was deleted with the fold, so `skipDuplicates` would not stop it. + const { kept } = await withoutFoldedSamples( + prisma, + userId, + "OURA", + dropImplausibleMeasurements("oura", mapped, (m) => ({ + type: m.type, + value: m.value, + })), + ); let inserted = 0; for (let i = 0; i < kept.length; i += OURA_HEART_RATE_WRITE_CHUNK) { const chunk = kept.slice(i, i + OURA_HEART_RATE_WRITE_CHUNK); diff --git a/src/lib/rollups/__tests__/full-backfill.test.ts b/src/lib/rollups/__tests__/full-backfill.test.ts new file mode 100644 index 000000000..5b87f54c2 --- /dev/null +++ b/src/lib/rollups/__tests__/full-backfill.test.ts @@ -0,0 +1,231 @@ +/** + * The full backfill's windowed fold and its handler (#1298): the windows it + * cuts, the per-statement limit every window runs under, and how a run that + * reaches its job budget hands the remaining years on instead of failing. + * + * The fold itself is mocked here; that the windows write exactly the buckets + * one pass over the whole span writes is pinned against Postgres in + * `tests/integration/rollup-full-backfill.test.ts`. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { Job } from "pg-boss"; + +const mocks = vi.hoisted(() => ({ + recomputeUserRollups: vi.fn(), + bossSend: vi.fn(), + getGlobalBoss: vi.fn(), +})); + +vi.mock("@/lib/rollups/measurement-rollups", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@/lib/rollups/measurement-rollups") + >()), + recomputeUserRollups: mocks.recomputeUserRollups, +})); + +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: () => mocks.getGlobalBoss(), +})); + +import { handleRollupFullBackfill } from "@/lib/jobs/reminder/rollup-handlers"; +import { + foldUserRollupHistory, + fullBackfillWindows, + ROLLUP_FULL_BACKFILL_QUEUE, + ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + type RollupFullBackfillPayload, +} from "@/lib/rollups/full-backfill"; +import { ROLLUP_FOLD_WINDOW_MS } from "@/lib/rollups/measurement-rollups"; + +const NOW = new Date("2026-10-29T10:00:00.000Z"); + +function job( + data: Partial = {}, +): Job { + return { + id: "job-1", + name: ROLLUP_FULL_BACKFILL_QUEUE, + data: { userId: "user-1", enqueuedAt: NOW.toISOString(), ...data }, + expireInSeconds: 3600, + signal: new AbortController().signal, + } as unknown as Job; +} + +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(NOW); + mocks.recomputeUserRollups.mockReset(); + mocks.recomputeUserRollups.mockResolvedValue({ + rowsUpserted: 10, + durationMs: 1, + }); + mocks.bossSend.mockReset(); + mocks.bossSend.mockResolvedValue("job-next"); + mocks.getGlobalBoss.mockReset(); + mocks.getGlobalBoss.mockReturnValue({ send: mocks.bossSend }); +}); + +describe("fullBackfillWindows", () => { + it("cuts the span at UTC New Year, newest first, and covers it exactly", () => { + const from = new Date("2021-10-30T10:00:00.000Z"); + const windows = fullBackfillWindows(from, NOW); + expect( + windows.map((w) => [w.from.toISOString(), w.to.toISOString()]), + ).toEqual([ + ["2026-01-01T00:00:00.000Z", NOW.toISOString()], + ["2025-01-01T00:00:00.000Z", "2026-01-01T00:00:00.000Z"], + ["2024-01-01T00:00:00.000Z", "2025-01-01T00:00:00.000Z"], + ["2023-01-01T00:00:00.000Z", "2024-01-01T00:00:00.000Z"], + ["2022-01-01T00:00:00.000Z", "2023-01-01T00:00:00.000Z"], + [from.toISOString(), "2022-01-01T00:00:00.000Z"], + ]); + for (const w of windows) { + expect(w.to.getTime() - w.from.getTime()).toBeLessThanOrEqual( + 366 * 86_400_000, + ); + } + }); + + it("treats an end on New Year as the end of the year before", () => { + const windows = fullBackfillWindows( + new Date("2024-06-01T00:00:00.000Z"), + new Date("2025-01-01T00:00:00.000Z"), + ); + expect(windows).toEqual([ + { + from: new Date("2024-06-01T00:00:00.000Z"), + to: new Date("2025-01-01T00:00:00.000Z"), + }, + ]); + }); + + it("is empty for an empty span", () => { + expect(fullBackfillWindows(NOW, NOW)).toEqual([]); + }); +}); + +describe("foldUserRollupHistory", () => { + it("folds every window of the fold window under the given statement limit", async () => { + const done = await foldUserRollupHistory("user-1", { + statementTimeoutMs: 1234, + }); + const expected = fullBackfillWindows( + new Date(NOW.getTime() - ROLLUP_FOLD_WINDOW_MS), + NOW, + ); + expect(mocks.recomputeUserRollups).toHaveBeenCalledTimes(expected.length); + expected.forEach((w, i) => { + expect(mocks.recomputeUserRollups).toHaveBeenNthCalledWith( + i + 1, + "user-1", + expect.objectContaining({ + from: w.from, + to: w.to, + statementTimeoutMs: 1234, + }), + ); + }); + expect(done).toMatchObject({ + rowsUpserted: 10 * expected.length, + windowsFolded: expected.length, + }); + expect(done.resumeBefore).toBeUndefined(); + }); + + it("stops before the next window and names where the rest begins", async () => { + const done = await foldUserRollupHistory("user-1", { + shouldStop: () => mocks.recomputeUserRollups.mock.calls.length >= 1, + }); + expect(mocks.recomputeUserRollups).toHaveBeenCalledTimes(1); + expect(done.windowsFolded).toBe(1); + expect(done.resumeBefore).toEqual(new Date("2026-01-01T00:00:00.000Z")); + }); + + it("reports a window the fold stopped inside as still to fold", async () => { + mocks.recomputeUserRollups + .mockResolvedValueOnce({ rowsUpserted: 5, durationMs: 1 }) + .mockResolvedValueOnce({ rowsUpserted: 2, durationMs: 1, stopped: true }); + const done = await foldUserRollupHistory("user-1", { + shouldStop: () => false, + }); + expect(done.windowsFolded).toBe(1); + expect(done.resumeBefore).toEqual(new Date("2026-01-01T00:00:00.000Z")); + }); + + it("resumes with only the windows that end at or before `before`", async () => { + await foldUserRollupHistory("user-1", { + before: new Date("2025-01-01T00:00:00.000Z"), + }); + const tos = mocks.recomputeUserRollups.mock.calls.map( + (call) => (call[1] as { to: Date }).to, + ); + expect(tos[0]).toEqual(new Date("2025-01-01T00:00:00.000Z")); + for (const to of tos) { + expect(to.getTime()).toBeLessThanOrEqual( + new Date("2025-01-01T00:00:00.000Z").getTime(), + ); + } + }); +}); + +describe("handleRollupFullBackfill", () => { + it("runs every statement of the fold under the full backfill's own limit", async () => { + await handleRollupFullBackfill([job()]); + expect(mocks.recomputeUserRollups).toHaveBeenCalled(); + for (const call of mocks.recomputeUserRollups.mock.calls) { + expect(call[1]).toMatchObject({ + statementTimeoutMs: ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + }); + } + expect(mocks.bossSend).not.toHaveBeenCalled(); + }); + + it("hands the years it did not reach to a follow-up and completes", async () => { + mocks.recomputeUserRollups + .mockResolvedValueOnce({ rowsUpserted: 5, durationMs: 1 }) + .mockResolvedValueOnce({ rowsUpserted: 0, durationMs: 1, stopped: true }); + const outcome = await handleRollupFullBackfill([job()]); + expect(outcome.ok).toBe(true); + expect(mocks.bossSend).toHaveBeenCalledTimes(1); + const [queue, payload, options] = mocks.bossSend.mock.calls[0]; + expect(queue).toBe(ROLLUP_FULL_BACKFILL_QUEUE); + expect(payload).toMatchObject({ + userId: "user-1", + resumeBefore: "2026-01-01T00:00:00.000Z", + }); + // Not the boot key: the running job still holds that one. + expect(options.singletonKey).toBe( + "boot-backfill|user-1|2026-01-01T00:00:00.000Z", + ); + expect(options.startAfter).toBeGreaterThan(0); + }); + + it("resumes a follow-up where the run before it stopped", async () => { + await handleRollupFullBackfill([ + job({ resumeBefore: "2024-01-01T00:00:00.000Z" }), + ]); + expect( + (mocks.recomputeUserRollups.mock.calls[0][1] as { to: Date }).to, + ).toEqual(new Date("2024-01-01T00:00:00.000Z")); + }); + + it("fails a run that could not finish one window, so its retry starts it again", async () => { + mocks.recomputeUserRollups.mockResolvedValueOnce({ + rowsUpserted: 0, + durationMs: 1, + stopped: true, + }); + await expect(handleRollupFullBackfill([job()])).rejects.toThrow( + /before it folded a window/, + ); + expect(mocks.bossSend).not.toHaveBeenCalled(); + }); + + it("fails the run when the follow-up cannot be queued", async () => { + mocks.getGlobalBoss.mockReturnValue(null); + mocks.recomputeUserRollups + .mockResolvedValueOnce({ rowsUpserted: 5, durationMs: 1 }) + .mockResolvedValueOnce({ rowsUpserted: 0, durationMs: 1, stopped: true }); + await expect(handleRollupFullBackfill([job()])).rejects.toThrow(/no queue/); + }); +}); diff --git a/src/lib/rollups/__tests__/measurement-rollups.test.ts b/src/lib/rollups/__tests__/measurement-rollups.test.ts index 69161ada6..88c499569 100644 --- a/src/lib/rollups/__tests__/measurement-rollups.test.ts +++ b/src/lib/rollups/__tests__/measurement-rollups.test.ts @@ -29,6 +29,7 @@ const mocks = vi.hoisted(() => ({ findMany: vi.fn(), findFirstMeasurement: vi.fn(), txExecuteRawUnsafe: vi.fn(), + txExecuteRaw: vi.fn(), bossSend: vi.fn(), getGlobalBossMock: vi.fn(), })); @@ -87,10 +88,12 @@ void queryRaw; import { ROLLUP_FULL_BACKFILL_QUEUE, + enqueueBootTimeRollupBackfill, +} from "../full-backfill"; +import { ROLLUP_RECOMPUTE_QUEUE, _resetEnsureUserRollupsFreshInFlightForTests, collapseToTypeDayKeys, - enqueueBootTimeRollupBackfill, enqueueRollupRecompute, ensureUserRollupsFresh, recomputeBucketsForMeasurement, @@ -127,6 +130,8 @@ beforeEach(() => { createMany: mocks.createMany, }, $executeRawUnsafe: mocks.txExecuteRawUnsafe, + $executeRaw: mocks.txExecuteRaw, + $queryRaw: mocks.queryRaw, }); } return Promise.all(arg as unknown[]); @@ -996,10 +1001,10 @@ describe("enqueueBootTimeRollupBackfill", () => { const sqlParts = queryRaw.mock.calls[0][0] as TemplateStringsArray; const sqlText = Array.isArray(sqlParts) ? sqlParts.join("?") : ""; - // Inner DISTINCT widens to `(user_id, type, bucket_start)`. - expect(sqlText).toContain("date_trunc('day', m.\"measured_at\")"); - // LEFT JOIN now compares bucket_start on both sides. - expect(sqlText).toContain('r."bucket_start" = mt."bucket_start"'); + // The day walk steps from each day's first reading to the next UTC day. + expect(sqlText).toContain("date_trunc('day', d.at) + interval '1 day'"); + // The coverage check compares the walked day with the DAY bucket. + expect(sqlText).toContain(`r."bucket_start" = date_trunc('day', d.at)`); }); // v1.16.10 — discovery and fold share one trailing window. Unbounded @@ -1017,12 +1022,14 @@ describe("enqueueBootTimeRollupBackfill", () => { const after = Date.now(); const sqlParts = queryRaw.mock.calls[0][0] as TemplateStringsArray; const sqlText = Array.isArray(sqlParts) ? sqlParts.join("?") : ""; - // The inner scan filters on measured_at >= . + // The day walk starts at measured_at >= . expect(sqlText).toContain('m."measured_at" >='); // The bound parameter is now − ROLLUP_FOLD_WINDOW_MS (5 years), // matching the `recomputeUserRollups` default the backfill worker // runs with. - const boundArg = queryRaw.mock.calls[0][1] as Date; + const boundArg = queryRaw.mock.calls[0].find( + (arg: unknown) => arg instanceof Date, + ) as Date; expect(boundArg).toBeInstanceOf(Date); const fiveYearsMs = 5 * 365 * 24 * 60 * 60 * 1000; expect(boundArg.getTime()).toBeGreaterThanOrEqual(before - fiveYearsMs); diff --git a/src/lib/rollups/full-backfill.ts b/src/lib/rollups/full-backfill.ts new file mode 100644 index 000000000..1d42b7a33 --- /dev/null +++ b/src/lib/rollups/full-backfill.ts @@ -0,0 +1,362 @@ +/** + * The full fold of one account's rollup tier, and the boot discovery that + * decides which accounts need one. + * + * The write hooks keep the DAY bucket of each touched day current and the + * read path warms the last 90 days, but neither folds the deep WEEK / MONTH / + * YEAR history, and neither refills a column a migration adds to rows that + * already exist (the accumulators of 0190, the reading extremes of 0417). The + * full backfill does both, once per account the discovery finds. + * + * Sized for large accounts (#1298). The fold used to be four aggregates over + * the whole five-year window, each held to the connection's statement limit + * (60 s unless `DATABASE_STATEMENT_TIMEOUT_MS` says otherwise). On an account + * with tens of millions of readings no aggregate finished in time, the job + * failed on every boot, and the columns 0417 added never filled, so every + * reader that needs them stayed on the raw table and timed out as well. The + * fold now works through the window one UTC calendar year at a time, newest + * first, and every statement of the full backfill runs under its own limit + * (`ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS`), the shape the pulse day-mean + * rebuild already had. Each window commits on its own, so a run that reaches + * its job budget hands the remaining years to a follow-up job instead of + * starting over on the retry. + */ +import { activeAccountSql } from "@/lib/account-deletion/active-account"; +import { prisma } from "@/lib/db"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { + applyStatementBudget, + recomputeUserRollups, + ROLLUP_FOLD_WINDOW_MS, +} from "@/lib/rollups/measurement-rollups"; + +/** + * The boot-time full-fold queue: one job per account the discovery finds, so + * a self-hosted instance converges on the rollup tier without an operator. + */ +export const ROLLUP_FULL_BACKFILL_QUEUE = "rollup-full-backfill"; + +/** Boot-fold worker concurrency. Serial so we don't crowd the pool. */ +export const ROLLUP_FULL_BACKFILL_CONCURRENCY = 1; + +/** + * The full fold's expiry: an hour. A run stops itself at three quarters of it + * (`jobBudget`), between two windows or two granularities, and hands what is + * left to a follow-up, so the expiry bounds one run, not the whole account. + * The margin above the budget covers the statement in flight, which can run + * for `ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS`. + */ +export const ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS = 60 * 60; + +/** + * The limit of every statement the full backfill and its discovery run, in + * place of the connection's request limit. Five minutes, as the pulse + * day-mean rebuild uses: one calendar year of one account's readings, + * aggregated per granularity, stays well inside it on the largest accounts + * reported, while a statement that would run longer still ends. + */ +export const ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS = 5 * 60 * 1000; + +/** Seconds a follow-up waits before it picks up the remaining years. */ +export const ROLLUP_FULL_BACKFILL_CONTINUATION_DELAY_SECONDS = 60; + +/** Payload of one full-fold job. */ +export interface RollupFullBackfillPayload { + userId: string; + /** Wall-clock kick-off for debugging only. */ + enqueuedAt: string; + /** + * ISO instant. A follow-up folds only the windows that end at or before + * it: the run that sent it folded everything newer. + */ + resumeBefore?: string; +} + +/** The retry policy of every full-fold send. */ +const RETRY_OPTIONS = { + retryLimit: 3, + retryDelay: 60, + retryBackoff: true, +} as const; + +/** + * `[from, to)` windows covering `[from, to)`, cut at UTC New Year and + * returned newest first. Every DAY, MONTH and YEAR bucket lies inside one + * window, and a WEEK that spans New Year is folded whole by both of its + * windows (the fold snaps each window out to whole buckets), so the windows + * write exactly the buckets one fold over the whole span writes. Pure. + */ +export function fullBackfillWindows( + from: Date, + to: Date, +): Array<{ from: Date; to: Date }> { + const out: Array<{ from: Date; to: Date }> = []; + let end = to.getTime(); + const floor = from.getTime(); + while (end > floor) { + const year = new Date(end).getUTCFullYear(); + const newYear = Date.UTC(year, 0, 1); + const start = Math.max( + floor, + newYear === end ? Date.UTC(year - 1, 0, 1) : newYear, + ); + out.push({ from: new Date(start), to: new Date(end) }); + end = start; + } + return out; +} + +export interface RollupHistoryFold { + rowsUpserted: number; + durationMs: number; + /** Windows folded completely by this call. */ + windowsFolded: number; + /** + * Set when `shouldStop` ended the fold: every window that ends at or + * before this instant is still to fold, and nothing newer is. + */ + resumeBefore?: Date; +} + +/** + * Fold one account's rollup tier over the five-year window, every type and + * granularity, one calendar year at a time, newest first. The same buckets + * `recomputeUserRollups(userId)` writes in one pass; see + * `fullBackfillWindows` for why the windows change nothing about them. + * + * `before` resumes a fold that stopped: only the windows ending at or before + * it run. `shouldStop` is asked before each window and between the + * granularities of a window; a window it interrupts is folded again whole by + * the follow-up, so `resumeBefore` is always a window's end. + */ +export async function foldUserRollupHistory( + userId: string, + opts: { + before?: Date; + shouldStop?: () => boolean; + statementTimeoutMs?: number; + now?: Date; + } = {}, +): Promise { + const startedAt = Date.now(); + const now = opts.now ?? new Date(); + const floor = new Date(now.getTime() - ROLLUP_FOLD_WINDOW_MS); + const ceiling = opts.before && opts.before < now ? opts.before : now; + let rowsUpserted = 0; + let windowsFolded = 0; + const result = (resumeBefore?: Date): RollupHistoryFold => ({ + rowsUpserted, + durationMs: Date.now() - startedAt, + windowsFolded, + ...(resumeBefore ? { resumeBefore } : {}), + }); + for (const window of fullBackfillWindows(floor, ceiling)) { + if (opts.shouldStop?.()) return result(window.to); + const done = await recomputeUserRollups(userId, { + from: window.from, + to: window.to, + statementTimeoutMs: opts.statementTimeoutMs, + shouldStop: opts.shouldStop, + }); + rowsUpserted += done.rowsUpserted; + if (done.stopped) return result(window.to); + windowsFolded += 1; + } + return result(); +} + +/** + * Hand the windows a stopped fold did not reach to a follow-up job. Keyed on + * the account and the resume point: the running job still holds the account's + * boot key, and a second follow-up for the same point is the same work. + * Throws when no queue is attached or the send fails, so the run fails and + * its retry folds the same windows again rather than leaving them. + */ +export async function sendRollupFullBackfillContinuation( + userId: string, + resumeBefore: Date, +): Promise { + const boss = getGlobalBoss(); + if (!boss) { + throw new Error("rollup-full-backfill: no queue to hand the rest to"); + } + const payload: RollupFullBackfillPayload = { + userId, + enqueuedAt: new Date().toISOString(), + resumeBefore: resumeBefore.toISOString(), + }; + const jobId = await boss.send(ROLLUP_FULL_BACKFILL_QUEUE, payload, { + ...RETRY_OPTIONS, + expireInSeconds: ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS, + singletonKey: `boot-backfill|${userId}|${payload.resumeBefore}`, + startAfter: ROLLUP_FULL_BACKFILL_CONTINUATION_DELAY_SECONDS, + }); + return jobId !== null; +} + +/** + * Whether a full fold of the account is queued or running. The dense fold + * leaves such an account alone (`ConsolidationOptions.deferUser`): it deletes + * the raw samples a fold still has to read, and a year the full backfill + * reached after the dense fold would take its day figures from hourly means. + * False when the queue schema cannot be read, as a web-only deployment has + * no full backfill either. + */ +export async function isRollupFullBackfillPending( + userId: string, +): Promise { + try { + const rows = await prisma.$queryRaw>` + SELECT count(*)::int AS n + FROM pgboss.job + WHERE name = ${ROLLUP_FULL_BACKFILL_QUEUE} + AND state IN ('created', 'retry', 'active') + AND data->>'userId' = ${userId} + `; + return (rows[0]?.n ?? 0) > 0; + } catch { + return false; + } +} + +/** + * Boot-time discovery. Finds the accounts whose rollup tier misses something + * inside the fold window and enqueues one full fold per account. + * + * Two arms: + * + * 1. Missing coverage per day: a (user, type, UTC day) with a live reading + * and no DAY rollup row. Per day rather than per type, because a sync + * or a restore that writes without the write hook leaves gaps in a type + * that already has buckets. + * 2. A DAY rollup row inside the window whose regression accumulators + * (0190) or reading extremes (0417) are NULL: a row folded before the + * column existed. New folds write them, so the arm drains to empty. + * + * Both are bounded to the fold window. The fold writes nothing older, so an + * older gap would flag the account on every boot and never close. + * + * Arm 1 used to read every live reading in the window, which on an account + * with tens of millions of rows is the same scan that kept the fold itself + * from finishing. It now walks the days instead: per account and type, the + * first live reading at or after the start of each next UTC day, read off + * the live `(user_id, type, measured_at)` index, so the cost follows the + * number of days with readings, not the number of readings. The statement + * also runs under the full backfill's own limit. + * + * Idempotent across reboots: an account drops off once it is covered, and + * the per-account `singletonKey` coalesces a send while a fold is queued. + * Best-effort: errors come back in the result so the worker boot never fails. + */ +export async function enqueueBootTimeRollupBackfill( + // Optional boot-storm stagger, seconds. Default 0 keeps immediate semantics + // for any non-boot caller. + startAfterSeconds: number = 0, +): Promise<{ + enqueued: number; + skipped: number; + error: string | null; +}> { + const boss = getGlobalBoss(); + if (!boss) { + return { enqueued: 0, skipped: 0, error: null }; + } + + try { + const foldWindowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); + const users = await prisma.$transaction( + async (tx) => { + await applyStatementBudget( + tx, + ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + ); + return tx.$queryRaw>` + WITH RECURSIVE pairs AS ( + SELECT u."id" AS user_id, t.type + FROM users u + CROSS JOIN unnest(enum_range(NULL::"measurement_type")) AS t(type) + WHERE ${activeAccountSql('u."id"')} + ), + days AS ( + SELECT p.user_id, p.type, f.at + FROM pairs p + CROSS JOIN LATERAL ( + SELECT min(m."measured_at") AS at + FROM measurements m + WHERE m."user_id" = p.user_id + AND m."type" = p.type + AND m."deleted_at" IS NULL + AND m."measured_at" >= ${foldWindowStart} + ) f + WHERE f.at IS NOT NULL + UNION ALL + SELECT d.user_id, d.type, n.at + FROM days d + CROSS JOIN LATERAL ( + SELECT min(m."measured_at") AS at + FROM measurements m + WHERE m."user_id" = d.user_id + AND m."type" = d.type + AND m."deleted_at" IS NULL + AND m."measured_at" >= date_trunc('day', d.at) + interval '1 day' + ) n + WHERE n.at IS NOT NULL + ) + SELECT DISTINCT id FROM ( + SELECT d.user_id AS id + FROM days d + WHERE NOT EXISTS ( + SELECT 1 FROM measurement_rollups r + WHERE r."user_id" = d.user_id + AND r."type" = d.type + AND r."granularity" = 'DAY' + AND r."bucket_start" = date_trunc('day', d.at) + ) + + UNION + + SELECT r."user_id" AS id + FROM measurement_rollups r + WHERE r."granularity" = 'DAY' + AND r."bucket_start" >= ${foldWindowStart} + AND (r."sum_xy" IS NULL OR r."reading_min_value" IS NULL) + AND ${activeAccountSql('r."user_id"')} + ) discovered + `; + }, + { + maxWait: 10_000, + timeout: ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS + 30_000, + }, + ); + + let enqueued = 0; + let skipped = 0; + for (const { id } of users) { + const payload: RollupFullBackfillPayload = { + userId: id, + enqueuedAt: new Date().toISOString(), + }; + const jobId = await boss.send(ROLLUP_FULL_BACKFILL_QUEUE, payload, { + ...RETRY_OPTIONS, + expireInSeconds: ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS, + // Coalesce: if a backfill for this user is already queued and + // we restart, pg-boss returns null instead of duplicating. + singletonKey: `boot-backfill|${id}`, + ...(startAfterSeconds > 0 ? { startAfter: startAfterSeconds } : {}), + }); + if (jobId) { + enqueued += 1; + } else { + skipped += 1; + } + } + return { enqueued, skipped, error: null }; + } catch (err) { + return { + enqueued: 0, + skipped: 0, + error: err instanceof Error ? err.message : String(err), + }; + } +} diff --git a/src/lib/rollups/measurement-read-wmy.ts b/src/lib/rollups/measurement-read-wmy.ts index 7ba0e5648..026b79f7d 100644 --- a/src/lib/rollups/measurement-read-wmy.ts +++ b/src/lib/rollups/measurement-read-wmy.ts @@ -116,12 +116,16 @@ export async function readBestGranularityRollups( type: MeasurementType, windowDays: number, userPriorityJson?: unknown, + now: Date = new Date(), ): Promise<{ granularity: RollupGranularity; rows: RollupBucketRow[]; } | null> { if (!Number.isFinite(windowDays) || windowDays <= 0) return null; - const since = new Date(Date.now() - windowDays * 24 * 60 * 60 * 1000); + // The window trails the caller's `now`, so a read anchored on another + // moment than the wall clock sees the same days through the rollups as + // through the live fallback. + const since = new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000); // v1.11.1 — load the source-priority blob once and thread it into every // granularity probe so the collapse never re-queries the user per floor. const priority = diff --git a/src/lib/rollups/measurement-rollups.ts b/src/lib/rollups/measurement-rollups.ts index ad330a2e9..0bda33e66 100644 --- a/src/lib/rollups/measurement-rollups.ts +++ b/src/lib/rollups/measurement-rollups.ts @@ -34,9 +34,7 @@ * took. Nothing here reads both tiers; the rollup replaces the legacy read and * falls back to live SQL only on a coverage miss. */ -import { activeAccountSql } from "@/lib/account-deletion/active-account"; import { prisma } from "@/lib/db"; -import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { enqueueWhenBossReady } from "@/lib/jobs/boss-ready"; import { annotate } from "@/lib/logging/context"; import { @@ -113,39 +111,6 @@ export interface RollupRecomputePayload { enqueuedAt: string; } -/** - * v1.4.35.1 — boot-time full-fold queue. The write-path hooks keep - * the DAY bucket current per measurement, and `ensureUserRollupsFresh` - * warm-on-read covers the trailing 90-day DAY window on cold mount, - * but neither path folds the deep WEEK / MONTH / YEAR history for an - * account that has measurements pre-dating the v1.4.35 deploy. This - * queue exists so the worker boot can enqueue one full - * `recomputeUserRollups(userId)` per uncovered user — no operator - * action needed on self-hosted instances. - */ -export const ROLLUP_FULL_BACKFILL_QUEUE = "rollup-full-backfill"; - -/** Boot-fold worker concurrency. Serial so we don't crowd the pool. */ -export const ROLLUP_FULL_BACKFILL_CONCURRENCY = 1; - -/** - * The full fold's expiry: an hour. One fold is four granularities, each an - * aggregate held to the connection's statement timeout (60 s by default) and - * a persist transaction of up to 120 s, so the worst case under the defaults - * is close to thirteen minutes, too near pg-boss's fifteen-minute default to - * rely on, and an operator who raises `DATABASE_STATEMENT_TIMEOUT_MS` (or - * sets it to 0) lifts that bound altogether. Past the expiry pg-boss retries - * the job, and the retry folded the same account a second time. - */ -export const ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS = 60 * 60; - -/** Payload the boot-time backfill helper enqueues. */ -export interface RollupFullBackfillPayload { - userId: string; - /** Wall-clock kick-off for debugging only. */ - enqueuedAt: string; -} - /** * Trailing fold window for `recomputeUserRollups` defaults AND the * boot-time discovery scan. The two MUST share this bound: the fold @@ -639,7 +604,7 @@ export async function readRollupBuckets( * comprehensive-aggregator + summaries-slice convention. */ /** Give the statements of a transaction their own limit (SET LOCAL). */ -async function applyStatementBudget( +export async function applyStatementBudget( tx: Pick, ms: number, ): Promise { @@ -1316,165 +1281,6 @@ async function ensureUserRollupsFreshImpl( } } -/** - * v1.4.35.1 — boot-time enqueue helper. Finds users with measurements - * but zero rollup coverage and enqueues one full-fold job per account - * onto `ROLLUP_FULL_BACKFILL_QUEUE`. Designed to be called once at - * worker boot so every self-hosted instance auto-converges on the - * persistent rollup tier without operator intervention. - * - * Idempotent across reboots: - * - the discovery query matches accounts that either have zero - * rollup coverage at all OR have at least one measurement type - * whose DAY-rollup partition is empty. Once every type the user - * has logged has at least one DAY bucket, the user drops off the - * list and subsequent boots skip them. The v1.4.35.1 shape (zero- - * rollups only) silently stranded users who logged a brand-new - * type after the initial fold — their `isFullyCovered` probe stays - * `false`, every read fans out to the live SQL aggregator, and - * `/api/analytics` cold-mounts in tens of seconds instead of - * 1.5-3 s. v1.4.38.5 widens discovery to per-type missing coverage. - * - pg-boss `singletonKey` coalesces duplicate sends within the - * queue, so a fast restart while a backfill is queued doesn't - * double up - * - * Best-effort: errors are swallowed and reported through the return - * value so the worker boot never fails because of a backfill miss. - */ -export async function enqueueBootTimeRollupBackfill( - // Optional boot-storm stagger. When > 0 the per-user sends carry a - // `startAfter` delay (seconds) so this full-history fold does not drain in - // parallel with the other boot backfills onto one heavy tenant. Default 0 - // keeps immediate semantics for any non-boot caller. - startAfterSeconds: number = 0, -): Promise<{ - enqueued: number; - skipped: number; - error: string | null; -}> { - const boss = getGlobalBoss(); - if (!boss) { - return { enqueued: 0, skipped: 0, error: null }; - } - - try { - // Users where at least one (type, day) pair in `measurements` - // has no matching DAY rollup row. - // - // v1.4.39.1 — tightened from per-type to per-day. The legacy shape - // matched users where ANY type had zero rollup rows. That stranded - // accounts whose Withings sync / `/api/import` / admin restore - // wrote measurements without firing the rollup write hook: as soon - // as ONE rollup row existed for the type (even from a prior boot - // backfill), the LEFT JOIN found a match and the user dropped off - // the discovery list — leaving the dashboard chart's - // `source=rollup` fast-path under-counting the missing days. - // Switching the join to `(user, type, bucketStart)` surfaces every - // per-day gap. Cost: the inner DISTINCT widens from - // `(user_id, type)` (≈15 rows/user) to - // `(user_id, type, day)` (≈5 k rows/user on a 5-year power user), - // which is still bounded by the `(user_id, type, measured_at)` - // index path. The LEFT JOIN walks `measurement_rollups` against - // its composite primary key. - // - // v1.4.41 — the v1.4.39 W-SUM legacy-NULL `sum_value` UNION arm - // has been retired. The maintainer's tenant converged at v1.4.40, so the - // arm was a permanent no-op seq scan over `measurement_rollups`. - // Per-day missing coverage remains the sole discovery anchor. - // - // v1.16.10 — discovery is bounded to the same trailing window the - // fold writes (`ROLLUP_FOLD_WINDOW_MS`). Unbounded, any account - // with measurements older than the window was re-flagged on every - // boot: the fold can never produce buckets for those days, so the - // per-day gap re-surfaced forever and the full backfill re-ran on - // every restart. Days outside the window are not foldable by - // design and must not anchor discovery. - const foldWindowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); - // Two discovery arms, UNION-ed: - // - // 1. Per-day MISSING coverage — a (user, type, day) pair in - // `measurements` with no matching DAY rollup row (v1.4.39.1). - // - // 2. v1.20.0 F6 NULL-accumulator arm — a user whose DAY rollup rows - // predate migration 0190 (regression accumulators are NULL) needs a - // one-time re-fold so the windowed slope / r² / sd readers compose - // from the accumulators instead of falling back to the live REGR_* - // scan. Anchored on `sum_xy IS NULL` (the column the read tier - // treats as the coverage marker). New writes self-populate the - // accumulators, so this arm drains to empty once the boot re-fold - // catches up — mirrors the v1.4.39 sum_value rollout, which the - // v1.4.41 note confirms converged via this same boot queue. Bounded - // to the fold window so days outside it (un-foldable by design) - // never re-flag a user on every boot. The same arm refills the - // reading extremes of migration 0417 (`reading_min_value IS NULL`); - // the extremes reader treats a NULL as uncovered until it lands. - const users = await prisma.$queryRaw>` - SELECT DISTINCT id FROM ( - SELECT DISTINCT mt."user_id" AS id - FROM ( - SELECT DISTINCT - m."user_id", - m."type", - date_trunc('day', m."measured_at") AS bucket_start - FROM measurements m - WHERE m."deleted_at" IS NULL - AND m."measured_at" >= ${foldWindowStart} - ) mt - LEFT JOIN measurement_rollups r - ON r."user_id" = mt."user_id" - AND r."type" = mt."type" - AND r."granularity" = 'DAY' - AND r."bucket_start" = mt."bucket_start" - WHERE r."bucket_start" IS NULL - - UNION - - SELECT DISTINCT r."user_id" AS id - FROM measurement_rollups r - WHERE r."granularity" = 'DAY' - AND r."bucket_start" >= ${foldWindowStart} - AND (r."sum_xy" IS NULL OR r."reading_min_value" IS NULL) - ) discovered - WHERE ${activeAccountSql('discovered."id"')} - `; - - if (users.length === 0) { - return { enqueued: 0, skipped: 0, error: null }; - } - - let enqueued = 0; - let skipped = 0; - for (const { id } of users) { - const payload: RollupFullBackfillPayload = { - userId: id, - enqueuedAt: new Date().toISOString(), - }; - const jobId = await boss.send(ROLLUP_FULL_BACKFILL_QUEUE, payload, { - expireInSeconds: ROLLUP_FULL_BACKFILL_EXPIRE_SECONDS, - retryLimit: 3, - retryDelay: 60, - retryBackoff: true, - // Coalesce: if a backfill for this user is already queued and - // we restart, pg-boss returns null instead of duplicating. - singletonKey: `boot-backfill|${id}`, - ...(startAfterSeconds > 0 ? { startAfter: startAfterSeconds } : {}), - }); - if (jobId) { - enqueued += 1; - } else { - skipped += 1; - } - } - return { enqueued, skipped, error: null }; - } catch (err) { - return { - enqueued: 0, - skipped: 0, - error: err instanceof Error ? err.message : String(err), - }; - } -} - /** * Collapse a list of `(type, day)` keys to the set of distinct * `(type, dayStart)` pairs. Used by the batch-ingest hooks so a 500- diff --git a/tests/integration/__golden__/queue-registration.json b/tests/integration/__golden__/queue-registration.json index 5521dd6b6..016678537 100644 --- a/tests/integration/__golden__/queue-registration.json +++ b/tests/integration/__golden__/queue-registration.json @@ -1,5 +1,5 @@ { - "registeredQueueCount": 125, + "registeredQueueCount": 126, "registeredQueues": [ "account-deletion-purge", "achievement-unlock-sweep", @@ -112,6 +112,7 @@ "strava-sync", "stress-score-compute", "supplement-nutrient-recompute", + "system-job-alert", "tls-pin-monitor", "whoop-backfill", "whoop-cycle-sync", @@ -1793,6 +1794,21 @@ "retry_delay_max": null, "retry_limit": 2 }, + { + "dead_letter": null, + "deletion_seconds": 604800, + "expire_seconds": 900, + "heartbeat_seconds": null, + "name": "system-job-alert", + "notify": false, + "partition": false, + "policy": "standard", + "retention_seconds": 1209600, + "retry_backoff": false, + "retry_delay": 0, + "retry_delay_max": null, + "retry_limit": 2 + }, { "dead_letter": null, "deletion_seconds": 604800, @@ -2825,6 +2841,18 @@ }, "timezone": "Europe/Berlin" }, + { + "cron": "43 * * * *", + "data": {}, + "key": "", + "kind": "cron", + "name": "system-job-alert", + "options": { + "retryLimit": 0, + "tz": "Europe/Berlin" + }, + "timezone": "Europe/Berlin" + }, { "cron": "7 */6 * * *", "data": {}, @@ -3682,6 +3710,13 @@ "localConcurrency": 1 } }, + { + "queue": "system-job-alert", + "options": { + "localConcurrency": 1, + "pollingIntervalSeconds": 30 + } + }, { "queue": "tls-pin-monitor", "options": { diff --git a/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts b/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts index f31ce755c..cbbadf508 100644 --- a/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts +++ b/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts @@ -12,10 +12,8 @@ import { beforeAll, describe, expect, it, vi } from "vitest"; import { getPrismaClient, truncateAllTables } from "./setup"; import type { Prisma } from "@/generated/prisma/client"; -import { - enqueueBootTimeRollupBackfill, - recomputeUserRollups, -} from "@/lib/rollups/measurement-rollups"; +import { enqueueBootTimeRollupBackfill } from "@/lib/rollups/full-backfill"; +import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; vi.mock("@/lib/db-compat", () => ({ ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), diff --git a/tests/integration/large-account-google-health.test.ts b/tests/integration/large-account-google-health.test.ts new file mode 100644 index 000000000..630a2898b --- /dev/null +++ b/tests/integration/large-account-google-health.test.ts @@ -0,0 +1,295 @@ +/** + * A large Google Health account end to end (#1297, #1298). + * + * One account with a dense heart-rate history, written through the real + * Google Health backfill (Google's API replaced at the `safeFetch` seam, as in + * `google-health-dense-backfill-memory.test.ts`), then taken through what a + * self-hosted instance with such an account went through: + * + * 1. The account's DAY rollups lose their reading extremes, as rows folded + * before migration 0417 have them. The boot discovery finds the account, + * and the full backfill, a calendar year per pass with every statement + * under its own limit, refills every bucket inside the fold window. + * 2. The dense fold thins the heart rate older than the retention window to + * hourly `stats:` rows of Google Health, and the DAY rollups keep the + * figures they had from the raw samples. + * 3. A second full-history backfill over the same samples inserts nothing: + * the folded hours are not written again. + * + * Each step reports its duration on stderr (`[large-account]`); the size is + * `POINTS` readings, one every five minutes, about four years. + */ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import type { Job } from "pg-boss"; + +import { Prisma } from "@/generated/prisma/client"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const USER = "large-account-google-health"; +const POINTS = 420_000; +const STEP_MS = 5 * 60_000; +const DAY_MS = 86_400_000; + +const fake = vi.hoisted(() => ({ total: 0, now: 0 })); + +vi.mock("@/lib/safe-fetch", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + safeFetch: async (url: string) => { + const u = new URL(url); + let body: unknown = {}; + if (u.pathname.endsWith("/dataTypes/heart-rate/dataPoints")) { + const pageSize = Number(u.searchParams.get("pageSize") ?? 1000); + const page = Number(u.searchParams.get("pageToken") ?? 0); + const end = Math.min(fake.total, (page + 1) * pageSize); + const dataPoints = []; + for (let g = page * pageSize; g < end; g++) { + dataPoints.push({ + name: `users/me/dataTypes/heart-rate/dataPoints/${8_000_000_000_000 + g}`, + dataSource: { + recordingMethod: "PASSIVE_MEASUREMENT", + device: { manufacturer: "Google", displayName: "Watch" }, + platform: "FITBIT", + }, + heartRate: { + beatsPerMinute: String(52 + ((g * 7) % 61)), + sampleTime: { + physicalTime: new Date(fake.now - g * STEP_MS).toISOString(), + }, + }, + }); + } + body = { + dataPoints, + ...(end < fake.total ? { nextPageToken: String(page + 1) } : {}), + }; + } + return new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }, + }; +}); + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: vi.fn(() => null), +})); + +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; + +const prisma = getPrismaClient(); + +function report(line: string): void { + process.stderr.write(`[large-account] ${line}\n`); +} + +async function timed(label: string, fn: () => Promise): Promise { + const started = Date.now(); + const result = await fn(); + report(`${label}: ${((Date.now() - started) / 1000).toFixed(1)} s`); + return result; +} + +async function backfillToTheEnd(): Promise { + const { runGoogleHealthBackfillForUser } = + await import("@/lib/jobs/google-health-backfill"); + let imported = 0; + for (let slice = 0; slice < 20; slice++) { + // One slice for the whole stream: Google's API is faked here, and + // the slice budget is not what this file is about. + const result = await runGoogleHealthBackfillForUser( + USER, + () => false, + slice, + { sliceMs: 1_800_000, slicePages: 100_000 }, + ); + imported += result.imported; + const connection = await prisma.googleHealthConnection.findUniqueOrThrow({ + where: { userId: USER }, + select: { backfillCompletedAt: true }, + }); + if (connection.backfillCompletedAt !== null) return imported; + } + throw new Error("the backfill never completed"); +} + +function pulseRows(where: Prisma.MeasurementWhereInput = {}) { + return prisma.measurement.count({ + where: { + userId: USER, + type: "PULSE", + source: "GOOGLE_HEALTH", + deletedAt: null, + ...where, + }, + }); +} + +async function dayPulseRollups() { + return prisma.measurementRollup.findMany({ + where: { userId: USER, type: "PULSE", granularity: "DAY" }, + select: { + bucketStart: true, + source: true, + count: true, + minValue: true, + maxValue: true, + mean: true, + readingMinValue: true, + readingMaxValue: true, + }, + orderBy: [{ bucketStart: "asc" }, { source: "asc" }], + }); +} + +beforeAll(async () => { + await truncateAllTables(prisma); + const { encrypt } = await import("@/lib/crypto"); + await prisma.user.create({ + data: { id: USER, username: USER, timezone: "Europe/Berlin" }, + }); + await prisma.googleHealthConnection.create({ + data: { + userId: USER, + googleUserId: `google-${USER}`, + accessToken: encrypt("access"), + refreshToken: encrypt("refresh"), + tokenExpiresAt: new Date(Date.now() + 86_400_000), + }, + }); + fake.total = POINTS; + fake.now = Date.now() - 3_600_000; +}, 120_000); + +afterAll(async () => { + await truncateAllTables(prisma); +}); + +describe("a large Google Health account (#1297, #1298)", () => { + it("backfills, refills the reading extremes, folds, and does not take folded samples back", async () => { + // ── The history arrives ─────────────────────────────────────────── + const imported = await timed("backfill", backfillToTheEnd); + expect(imported).toBe(POINTS); + expect(await pulseRows()).toBe(POINTS); + + // ── Rows folded before 0417 ─────────────────────────────────────── + await prisma.$executeRaw` + UPDATE measurement_rollups + SET reading_min_value = NULL, reading_max_value = NULL + WHERE user_id = ${USER} + `; + const { enqueueBootTimeRollupBackfill, ROLLUP_FULL_BACKFILL_QUEUE } = + await import("@/lib/rollups/full-backfill"); + const send = vi.fn(async () => "job-id"); + vi.mocked(getGlobalBoss).mockReturnValue({ send } as never); + const discovery = await timed("discovery", () => + enqueueBootTimeRollupBackfill(), + ); + vi.mocked(getGlobalBoss).mockReturnValue(null as never); + expect(discovery).toMatchObject({ enqueued: 1, error: null }); + + const { handleRollupFullBackfill } = + await import("@/lib/jobs/reminder/rollup-handlers"); + const job = { + id: "full-backfill", + name: ROLLUP_FULL_BACKFILL_QUEUE, + data: { userId: USER, enqueuedAt: new Date().toISOString() }, + expireInSeconds: 3600, + signal: new AbortController().signal, + } as unknown as Job<{ userId: string; enqueuedAt: string }>; + const outcome = await timed("full backfill", () => + handleRollupFullBackfill([job]), + ); + expect(outcome.ok).toBe(true); + + const { ROLLUP_FOLD_WINDOW_MS } = + await import("@/lib/rollups/measurement-rollups"); + const floor = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); + const before = await dayPulseRollups(); + expect(before.length).toBeGreaterThan(1000); + expect( + before.filter( + (r) => + r.bucketStart >= floor && + (r.readingMinValue === null || r.readingMaxValue === null), + ), + ).toEqual([]); + // The extremes are the raw readings' own. + const raw = await prisma.$queryRaw< + Array<{ day: Date; min: number; max: number }> + >` + SELECT date_trunc('day', measured_at) AS day, + MIN(value)::double precision AS min, + MAX(value)::double precision AS max + FROM measurements + WHERE user_id = ${USER} AND type = 'PULSE' AND deleted_at IS NULL + GROUP BY 1 + `; + const rawByDay = new Map(raw.map((r) => [r.day.getTime(), r])); + for (const row of before) { + const day = rawByDay.get(row.bucketStart.getTime()); + expect(day).toBeDefined(); + expect(row.readingMinValue).toBe(day!.min); + expect(row.readingMaxValue).toBe(day!.max); + } + + // ── The fold thins the old samples ──────────────────────────────── + const { runDenseIntradayRetention, DENSE_INTRADAY_RETENTION_DAYS } = + await import("@/lib/measurements/dense-intraday-retention"); + const summary = await timed("dense fold", () => + runDenseIntradayRetention(prisma, { userId: USER, log: () => {} }), + ); + expect(summary.stoppedEarly).toBe(false); + const retained = new Date( + Date.now() - (DENSE_INTRADAY_RETENTION_DAYS + 2) * DAY_MS, + ); + const rawOld = await pulseRows({ + measuredAt: { lt: retained }, + NOT: { externalId: { startsWith: "stats:" } }, + }); + const hourlyOld = await pulseRows({ + measuredAt: { lt: retained }, + externalId: { startsWith: "stats:" }, + }); + const afterFold = await pulseRows(); + report( + `rows: ${POINTS} raw → ${afterFold} after the fold (${hourlyOld} hourly rows before the window)`, + ); + expect(rawOld).toBe(0); + // At most one hourly row per hour, twelve samples an hour before. + expect(hourlyOld).toBeGreaterThan(0); + expect(afterFold).toBeLessThan(POINTS / 5); + + // The DAY tier keeps the figures it took from the raw samples. + const after = await dayPulseRollups(); + expect(after.map((r) => r.bucketStart.getTime())).toEqual( + before.map((r) => r.bucketStart.getTime()), + ); + after.forEach((row, i) => { + expect(row.count).toBe(before[i].count); + expect(row.minValue).toBe(before[i].minValue); + expect(row.maxValue).toBe(before[i].maxValue); + expect(row.readingMinValue).toBe(before[i].readingMinValue); + expect(row.mean).toBeCloseTo(before[i].mean, 6); + }); + + // ── A second full-history backfill over the same samples ────────── + await prisma.googleHealthConnection.update({ + where: { userId: USER }, + data: { + backfillCompletedAt: null, + backfillProgress: Prisma.DbNull, + }, + }); + const again = await timed("second backfill", backfillToTheEnd); + expect(again).toBe(0); + expect(await pulseRows()).toBe(afterFold); + }, 1_800_000); +}); diff --git a/tests/integration/measurement-rollups.test.ts b/tests/integration/measurement-rollups.test.ts index 2e382c9a5..c813fdebf 100644 --- a/tests/integration/measurement-rollups.test.ts +++ b/tests/integration/measurement-rollups.test.ts @@ -29,6 +29,8 @@ import { getPrismaClient, truncateAllTables } from "./setup"; import { ROLLUP_FULL_BACKFILL_QUEUE, enqueueBootTimeRollupBackfill, +} from "@/lib/rollups/full-backfill"; +import { recomputeBucketsForKeys, recomputeBucketsForMeasurement, recomputeUserRollups, diff --git a/tests/integration/provider-dense-fold.test.ts b/tests/integration/provider-dense-fold.test.ts new file mode 100644 index 000000000..d36c21a52 --- /dev/null +++ b/tests/integration/provider-dense-fold.test.ts @@ -0,0 +1,185 @@ +/** + * The dense hourly fold over the provider syncs that store heart rate one row + * per sample, and their ingest guard, against Postgres (#1297). + * + * - Google Health and Oura heart rate older than the retention window folds + * to hourly `stats:` rows under its own source, the raw rows go, and the + * DAY rollup keeps the figures it had from the raw samples. + * - A daily SpO2 value of the same sources is no intraday stream and stays. + * - Writing the same old samples again, as a backfill or a full sync does, + * inserts nothing and leaves every hourly mean as it was, including the + * hour whose anchor a sample fell on (Google Health's natural-key rescue + * used to re-key that row back into the sample). + * - Samples inside the window still write as before. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { upsertGoogleHealthMeasurements } from "@/lib/google-health/sync-core"; +import { runDenseIntradayRetention } from "@/lib/measurements/dense-intraday-retention"; +import { + TouchedDays, + writeOuraHeartRatePage, +} from "@/lib/oura/sync-heart-rate"; + +import { makeUser } from "../kit/factories"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const DAY_MS = 86_400_000; +const MIN_MS = 60_000; + +beforeEach(async () => { + await truncateAllTables(prisma); +}); + +/** Noon UTC, `days` ago: every sample below stays inside that UTC day. */ +function dayStart(days: number): number { + const now = Date.now(); + return Math.floor(now / DAY_MS) * DAY_MS - days * DAY_MS; +} + +/** Samples of 10:00-11:59 UTC, one every 10 minutes, 10:30 included. */ +function samples(base: number): Array<{ at: Date; bpm: number }> { + const out: Array<{ at: Date; bpm: number }> = []; + for (let m = 0; m < 120; m += 10) { + out.push({ + at: new Date(base + 10 * 60 * MIN_MS + m * MIN_MS), + bpm: 50 + ((m * 7) % 45), + }); + } + return out; +} + +const gh = (s: { at: Date; bpm: number }) => ({ + type: "PULSE", + value: s.bpm, + unit: "bpm", + measuredAt: s.at, + externalId: `${s.at.getTime()}:hr`, +}); + +async function liveRows(userId: string, source: "GOOGLE_HEALTH" | "OURA") { + return prisma.measurement.findMany({ + where: { userId, source, deletedAt: null }, + select: { type: true, value: true, externalId: true, measuredAt: true }, + orderBy: [{ type: "asc" }, { measuredAt: "asc" }], + }); +} + +describe("dense fold of provider heart rate (#1297)", () => { + it("folds old Google Health heart rate to hourly means, and a re-sync writes none of it back", async () => { + const user = await makeUser({ timezone: "UTC" }); + const old = samples(dayStart(120)); + const recent = samples(dayStart(5)); + await upsertGoogleHealthMeasurements(user.id, old.map(gh)); + // A nightly SpO2 average: one value a day, never folded. + await prisma.measurement.create({ + data: { + userId: user.id, + type: "OXYGEN_SATURATION", + value: 96, + unit: "%", + source: "GOOGLE_HEALTH", + measuredAt: new Date(dayStart(120) + 7 * 3_600_000), + externalId: "2026-01-01:spo2", + }, + }); + const dayBefore = await prisma.measurementRollup.findFirstOrThrow({ + where: { userId: user.id, type: "PULSE", granularity: "DAY" }, + }); + + const summary = await runDenseIntradayRetention(prisma, { + userId: user.id, + log: () => {}, + }); + expect(summary.totals.perSampleRowsSoftDeleted).toBeGreaterThan(0); + + const folded = await liveRows(user.id, "GOOGLE_HEALTH"); + const pulse = folded.filter((r) => r.type === "PULSE"); + // Two hours, two hourly means; no raw sample of them is left. + expect(pulse).toHaveLength(2); + for (const row of pulse) { + expect(row.externalId).toMatch( + /^stats:HKQuantityTypeIdentifierHeartRate:\d{4}-\d{2}-\d{2}T1[01]$/, + ); + } + const firstHour = old.slice(0, 6).map((s) => s.bpm); + expect(pulse[0].value).toBeCloseTo( + firstHour.reduce((a, b) => a + b, 0) / firstHour.length, + 6, + ); + expect(folded.filter((r) => r.type === "OXYGEN_SATURATION")).toHaveLength( + 1, + ); + // The DAY bucket keeps the raw figures. + const dayAfter = await prisma.measurementRollup.findFirstOrThrow({ + where: { userId: user.id, type: "PULSE", granularity: "DAY" }, + }); + expect(dayAfter.count).toBe(dayBefore.count); + expect(dayAfter.minValue).toBe(dayBefore.minValue); + expect(dayAfter.maxValue).toBe(dayBefore.maxValue); + expect(dayAfter.mean).toBeCloseTo(dayBefore.mean, 9); + + // A backfill walks the same history again. + const again = await upsertGoogleHealthMeasurements( + user.id, + [...old, ...recent].map(gh), + ); + expect(again.imported).toBe(recent.length); + const afterResync = await liveRows(user.id, "GOOGLE_HEALTH"); + expect(afterResync.filter((r) => r.type === "PULSE")).toHaveLength( + 2 + recent.length, + ); + // The hourly means are untouched, the 10:30 one included. + expect( + afterResync.filter((r) => r.externalId?.startsWith("stats:")), + ).toEqual(pulse); + }); + + it("folds old Oura heart rate the same way, and a re-read writes none of it back", async () => { + const user = await makeUser({ timezone: "UTC" }); + const old = samples(dayStart(150)).map((s) => ({ + bpm: s.bpm, + timestamp: s.at.toISOString(), + source: "awake", + })); + expect( + await writeOuraHeartRatePage(user.id, old, new TouchedDays(), { + emitArrivals: false, + }), + ).toBe(old.length); + + await runDenseIntradayRetention(prisma, { + userId: user.id, + log: () => {}, + }); + const folded = await liveRows(user.id, "OURA"); + expect(folded).toHaveLength(2); + expect(folded.every((r) => r.externalId?.startsWith("stats:"))).toBe(true); + + expect( + await writeOuraHeartRatePage(user.id, old, new TouchedDays(), { + emitArrivals: false, + }), + ).toBe(0); + expect(await liveRows(user.id, "OURA")).toEqual(folded); + }); + + it("leaves heart rate inside the window raw", async () => { + const user = await makeUser({ timezone: "UTC" }); + const recent = samples(dayStart(10)); + await upsertGoogleHealthMeasurements(user.id, recent.map(gh)); + await runDenseIntradayRetention(prisma, { + userId: user.id, + log: () => {}, + }); + expect(await liveRows(user.id, "GOOGLE_HEALTH")).toHaveLength( + recent.length, + ); + }); +}); diff --git a/tests/integration/rollup-full-backfill.test.ts b/tests/integration/rollup-full-backfill.test.ts new file mode 100644 index 000000000..cdf0742fa --- /dev/null +++ b/tests/integration/rollup-full-backfill.test.ts @@ -0,0 +1,347 @@ +/** + * The full backfill against Postgres (#1298). + * + * - The windowed fold (`foldUserRollupHistory`, a UTC calendar year per + * pass, newest first) writes exactly the rollup rows one pass of + * `recomputeUserRollups` over the whole five-year window writes: every + * granularity, every source, a week across New Year, the YEAR bucket the + * window's oldest edge snaps out to, and a reading older than the window. + * - A fold stopped part way and resumed with `before` ends in the same rows. + * - The boot discovery, rewritten to walk days instead of reading every + * reading in the window, finds the same accounts as the statement it + * replaced, kept verbatim below as the reference. + */ +import { PgBoss } from "pg-boss"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +import { Prisma, type MeasurementSource } from "@/generated/prisma/client"; +import { activeAccountSql } from "@/lib/account-deletion/active-account"; +import { + enqueueBootTimeRollupBackfill, + foldUserRollupHistory, + isRollupFullBackfillPending, + ROLLUP_FULL_BACKFILL_QUEUE, + ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, +} from "@/lib/rollups/full-backfill"; +import { + recomputeUserRollups, + ROLLUP_FOLD_WINDOW_MS, +} from "@/lib/rollups/measurement-rollups"; + +import { makeUser } from "../kit/factories"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: vi.fn(() => null), +})); + +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; + +const prisma = getPrismaClient(); +const DAY_MS = 86_400_000; +const HOUR_MS = 3_600_000; + +beforeEach(async () => { + await truncateAllTables(prisma); + vi.mocked(getGlobalBoss).mockReturnValue(null as never); +}); + +function utc(y: number, m: number, d: number, h = 12, min = 0): Date { + return new Date(Date.UTC(y, m - 1, d, h, min)); +} + +/** Readings that can tell a windowed fold from one pass. */ +async function seedHistory(userId: string): Promise { + const now = new Date(); + const year = now.getUTCFullYear(); + const floor = new Date(now.getTime() - ROLLUP_FOLD_WINDOW_MS); + const rows: Prisma.MeasurementCreateManyInput[] = []; + const add = ( + type: "WEIGHT" | "PULSE", + source: MeasurementSource, + at: Date, + value: number, + ) => + rows.push({ + userId, + type, + source, + value, + unit: type === "WEIGHT" ? "kg" : "bpm", + measuredAt: at, + externalId: `${source}:${type}:${at.toISOString()}`, + }); + const days: Date[] = []; + // A week across each of two New Years. + for (let d = 0; d < 9; d++) { + days.push(new Date(utc(year - 3, 12, 27).getTime() + d * DAY_MS)); + days.push(new Date(utc(year - 1, 12, 28).getTime() + d * DAY_MS)); + } + days.push(utc(year - 2, 6, 15), utc(year - 2, 6, 16)); + days.push(new Date(now.getTime() - 3 * DAY_MS)); + // Just inside the window, and inside the YEAR bucket its edge snaps to. + days.push(new Date(floor.getTime() + 2 * DAY_MS)); + days.push(new Date(floor.getTime() - 20 * DAY_MS)); + days.forEach((day, i) => { + const base = Date.UTC( + day.getUTCFullYear(), + day.getUTCMonth(), + day.getUTCDate(), + ); + add("WEIGHT", "MANUAL", new Date(base + 7 * HOUR_MS), 80 + (i % 5) / 3); + if (i % 2 === 0) { + add("WEIGHT", "WITHINGS", new Date(base + 8 * HOUR_MS), 81 + i / 7); + } + for (let k = 0; k < 6; k++) { + add( + "PULSE", + "APPLE_HEALTH", + new Date(base + (k < 4 ? 9 : 15) * HOUR_MS + k * 7 * 60_000), + 55 + ((i * 7 + k * 11) % 40), + ); + } + add("PULSE", "GOOGLE_HEALTH", new Date(base + 23 * HOUR_MS), 60 + i); + }); + await prisma.measurement.createMany({ data: rows }); +} + +async function snapshot(userId: string) { + const rows = await prisma.measurementRollup.findMany({ + where: { userId }, + orderBy: [ + { type: "asc" }, + { granularity: "asc" }, + { bucketStart: "asc" }, + { source: "asc" }, + ], + }); + return rows.map(({ computedAt, ...rest }) => { + void computedAt; + return rest; + }); +} + +function expectSameRows( + actual: Awaited>, + expected: Awaited>, +): void { + expect(actual.length).toBe(expected.length); + expected.forEach((want, i) => { + const got = actual[i] as Record; + for (const [key, value] of Object.entries(want)) { + if (typeof value === "number" && key !== "count") { + // A week across two windows is read in another range, so its sums + // may be added in another order: equal to rounding, not to the bit. + expect(got[key] as number).toBeCloseTo(value, 6); + } else { + expect(got[key]).toEqual(value); + } + } + }); +} + +describe("foldUserRollupHistory", () => { + it("writes the rows one pass over the whole window writes", async () => { + const user = await makeUser(); + await seedHistory(user.id); + + await recomputeUserRollups(user.id); + const onePass = await snapshot(user.id); + expect(onePass.length).toBeGreaterThan(50); + expect(onePass.some((r) => r.granularity === "WEEK")).toBe(true); + + await prisma.measurementRollup.deleteMany({ where: { userId: user.id } }); + const done = await foldUserRollupHistory(user.id, { + statementTimeoutMs: ROLLUP_FULL_BACKFILL_STATEMENT_BUDGET_MS, + }); + expect(done.windowsFolded).toBeGreaterThanOrEqual(5); + expect(done.resumeBefore).toBeUndefined(); + expectSameRows(await snapshot(user.id), onePass); + expect( + (await snapshot(user.id)).every( + (r) => r.readingMinValue !== null && r.readingMaxValue !== null, + ), + ).toBe(true); + }); + + it("ends in the same rows when stopped part way and resumed", async () => { + const user = await makeUser(); + await seedHistory(user.id); + await recomputeUserRollups(user.id); + const onePass = await snapshot(user.id); + await prisma.measurementRollup.deleteMany({ where: { userId: user.id } }); + + // Stop on the seventh question: inside the second window. + let asked = 0; + const first = await foldUserRollupHistory(user.id, { + shouldStop: () => ++asked >= 7, + }); + expect(first.windowsFolded).toBe(1); + expect(first.resumeBefore).toBeDefined(); + + const rest = await foldUserRollupHistory(user.id, { + before: first.resumeBefore, + }); + expect(rest.resumeBefore).toBeUndefined(); + expectSameRows(await snapshot(user.id), onePass); + }); +}); + +/** The discovery statement before the day walk, verbatim. */ +async function referenceDiscovery(): Promise { + const foldWindowStart = new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS); + const users = await prisma.$queryRaw>` + SELECT DISTINCT id FROM ( + SELECT DISTINCT mt."user_id" AS id + FROM ( + SELECT DISTINCT + m."user_id", + m."type", + date_trunc('day', m."measured_at") AS bucket_start + FROM measurements m + WHERE m."deleted_at" IS NULL + AND m."measured_at" >= ${foldWindowStart} + ) mt + LEFT JOIN measurement_rollups r + ON r."user_id" = mt."user_id" + AND r."type" = mt."type" + AND r."granularity" = 'DAY' + AND r."bucket_start" = mt."bucket_start" + WHERE r."bucket_start" IS NULL + + UNION + + SELECT DISTINCT r."user_id" AS id + FROM measurement_rollups r + WHERE r."granularity" = 'DAY' + AND r."bucket_start" >= ${foldWindowStart} + AND (r."sum_xy" IS NULL OR r."reading_min_value" IS NULL) + ) discovered + WHERE ${activeAccountSql('discovered."id"')} + `; + return users.map((u) => u.id).sort(); +} + +describe("enqueueBootTimeRollupBackfill", () => { + it("finds the accounts the statement it replaced found", async () => { + const covered = await makeUser(); + const missingDay = await makeUser(); + const nullExtremes = await makeUser(); + const tombstoneOnly = await makeUser(); + const tooOld = await makeUser(); + const pendingDeletion = await makeUser(); + for (const u of [covered, missingDay, nullExtremes, pendingDeletion]) { + await seedHistory(u.id); + await recomputeUserRollups(u.id); + } + // A reading on a day with no DAY bucket. + await prisma.measurement.create({ + data: { + userId: missingDay.id, + type: "BODY_FAT", + value: 20, + unit: "%", + source: "MANUAL", + measuredAt: new Date(Date.now() - 40 * DAY_MS), + }, + }); + await prisma.measurement.create({ + data: { + userId: pendingDeletion.id, + type: "BODY_FAT", + value: 20, + unit: "%", + source: "MANUAL", + measuredAt: new Date(Date.now() - 40 * DAY_MS), + }, + }); + await prisma.user.update({ + where: { id: pendingDeletion.id }, + data: { deletionRequestedAt: new Date() }, + }); + // A row folded before 0417 added the reading extremes. + await prisma.$executeRaw` + UPDATE measurement_rollups SET reading_min_value = NULL + WHERE user_id = ${nullExtremes.id} AND granularity = 'DAY' + AND bucket_start = ( + SELECT max(bucket_start) FROM measurement_rollups + WHERE user_id = ${nullExtremes.id} AND granularity = 'DAY' + ) + `; + // Only a tombstone inside the window: nothing to fold. + await prisma.measurement.create({ + data: { + userId: tombstoneOnly.id, + type: "WEIGHT", + value: 70, + unit: "kg", + source: "MANUAL", + measuredAt: new Date(Date.now() - 10 * DAY_MS), + deletedAt: new Date(), + }, + }); + // Only readings older than the window: the fold writes nothing there. + await prisma.measurement.create({ + data: { + userId: tooOld.id, + type: "WEIGHT", + value: 70, + unit: "kg", + source: "MANUAL", + measuredAt: new Date(Date.now() - ROLLUP_FOLD_WINDOW_MS - 5 * DAY_MS), + }, + }); + + const send = vi.fn(async () => "job-id"); + vi.mocked(getGlobalBoss).mockReturnValue({ send } as never); + const result = await enqueueBootTimeRollupBackfill(); + expect(result.error).toBeNull(); + const found = ( + send.mock.calls as unknown as Array<[string, { userId: string }]> + ) + .map((call) => call[1].userId) + .sort(); + + expect(found).toEqual([missingDay.id, nullExtremes.id].sort()); + expect(found).toEqual(await referenceDiscovery()); + }); +}); + +describe("isRollupFullBackfillPending", () => { + let boss: PgBoss; + + beforeAll(async () => { + boss = new PgBoss({ + connectionString: process.env.DATABASE_URL, + supervise: false, + schedule: false, + }); + await boss.start(); + await boss.createQueue(ROLLUP_FULL_BACKFILL_QUEUE); + }, 60_000); + + afterAll(async () => { + await boss?.stop({ graceful: false, close: true }); + await prisma.$executeRawUnsafe(`DROP SCHEMA IF EXISTS pgboss CASCADE`); + }); + + it("is true while the account's full fold is queued or running, and only for that account", async () => { + expect(await isRollupFullBackfillPending("user-a")).toBe(false); + await boss.send(ROLLUP_FULL_BACKFILL_QUEUE, { userId: "user-a" }); + expect(await isRollupFullBackfillPending("user-a")).toBe(true); + expect(await isRollupFullBackfillPending("user-b")).toBe(false); + const [job] = await boss.fetch(ROLLUP_FULL_BACKFILL_QUEUE); + expect(await isRollupFullBackfillPending("user-a")).toBe(true); + await boss.complete(ROLLUP_FULL_BACKFILL_QUEUE, job.id); + expect(await isRollupFullBackfillPending("user-a")).toBe(false); + }); +}); diff --git a/tests/integration/system-job-alert.test.ts b/tests/integration/system-job-alert.test.ts new file mode 100644 index 000000000..bb267bf76 --- /dev/null +++ b/tests/integration/system-job-alert.test.ts @@ -0,0 +1,178 @@ +/** + * The system job alert against a ledger pg-boss itself wrote (#1298). + * + * A background job of the instance that keeps failing pushes every admin + * once, a reminder at most once a day while it keeps failing, and once when it + * runs successfully again. Fewer failures than the streak, and queues that + * have an account-facing surface of their own, push nothing. + * + * pg-boss provisions its own schema here and the jobs are failed and completed + * through its own API, as in `job-failures.test.ts`, so the reader is proven + * against the table pg-boss writes rather than one the test wrote. + */ +import { PgBoss } from "pg-boss"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +const dispatched = vi.hoisted(() => ({ + calls: [] as Array<{ + userId: string; + titleKey: string; + messageKey: string; + params?: Record; + eventType?: string; + }>, +})); + +vi.mock("@/lib/notifications/dispatch-localised", () => ({ + dispatchLocalisedNotification: vi.fn( + async (opts: (typeof dispatched.calls)[number]) => { + dispatched.calls.push(opts); + }, + ), +})); + +import { + notifySystemJobFailures, + SYSTEM_JOB_FAILURE_STREAK, +} from "@/lib/jobs/system-job-alert"; + +import { makeUser } from "../kit/factories"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +const SYSTEM_QUEUE = "rollup-full-backfill"; +const ACCOUNT_QUEUE = "google-health-sync"; +const HOUR_MS = 3_600_000; + +let boss: PgBoss; + +async function runOnce(queue: string, outcome: "fail" | "complete") { + const id = await boss.send(queue, { userId: "u" }, { retryLimit: 0 }); + if (!id) throw new Error("queue rejected the job"); + const [job] = await boss.fetch(queue); + if (!job) throw new Error("nothing to fetch"); + if (outcome === "fail") { + await boss.fail( + queue, + job.id, + new Error("canceling statement due to statement timeout"), + ); + } else { + await boss.complete(queue, job.id, { ok: true }); + } +} + +beforeAll(async () => { + boss = new PgBoss({ + connectionString: process.env.DATABASE_URL, + supervise: false, + schedule: false, + }); + await boss.start(); + await boss.createQueue(SYSTEM_QUEUE); + await boss.createQueue(ACCOUNT_QUEUE); +}, 60_000); + +afterAll(async () => { + await boss?.stop({ graceful: false, close: true }); + await getPrismaClient().$executeRawUnsafe( + `DROP SCHEMA IF EXISTS pgboss CASCADE`, + ); +}); + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); + await getPrismaClient().$executeRawUnsafe(`DELETE FROM pgboss.job`); + dispatched.calls.length = 0; +}); + +describe("notifySystemJobFailures", () => { + it("tells every admin once, reminds a day later, and announces the recovery once", async () => { + const admin = await makeUser({ role: "ADMIN" }); + const second = await makeUser({ role: "ADMIN" }); + await makeUser({ role: "USER" }); + for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK; i++) { + await runOnce(SYSTEM_QUEUE, "fail"); + } + const now = new Date(); + + const first = await notifySystemJobFailures(now); + expect(first.failing).toBe(1); + expect(first.sent).toEqual([{ queue: SYSTEM_QUEUE, kind: "problem" }]); + expect(dispatched.calls.map((c) => c.userId).sort()).toEqual( + [admin.id, second.id].sort(), + ); + const call = dispatched.calls[0]; + expect(call.eventType).toBe("SYSTEM_ALERT"); + expect(call.titleKey).toBe("notifications.systemJobAlert.titleFailing"); + expect(call.params).toMatchObject({ + queue: SYSTEM_QUEUE, + failures: SYSTEM_JOB_FAILURE_STREAK, + }); + expect(String(call.params?.error)).toContain("statement timeout"); + + // The next hour: still failing, already told. + dispatched.calls.length = 0; + const again = await notifySystemJobFailures( + new Date(now.getTime() + HOUR_MS), + ); + expect(again.sent).toEqual([]); + expect(dispatched.calls).toEqual([]); + + // A day later: one reminder. + const reminder = await notifySystemJobFailures( + new Date(now.getTime() + 25 * HOUR_MS), + ); + expect(reminder.sent).toEqual([{ queue: SYSTEM_QUEUE, kind: "reminder" }]); + + // It runs again: one recovery notice, then silence. + await runOnce(SYSTEM_QUEUE, "complete"); + dispatched.calls.length = 0; + const recovered = await notifySystemJobFailures( + new Date(now.getTime() + 26 * HOUR_MS), + ); + expect(recovered.sent).toEqual([ + { queue: SYSTEM_QUEUE, kind: "recovered" }, + ]); + expect(dispatched.calls[0].titleKey).toBe( + "notifications.systemJobAlert.titleRecovered", + ); + const quiet = await notifySystemJobFailures( + new Date(now.getTime() + 27 * HOUR_MS), + ); + expect(quiet.sent).toEqual([]); + }); + + it("stays quiet below the streak and for queues with an account-facing surface", async () => { + await makeUser({ role: "ADMIN" }); + for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK - 1; i++) { + await runOnce(SYSTEM_QUEUE, "fail"); + } + for (let i = 0; i < SYSTEM_JOB_FAILURE_STREAK + 2; i++) { + await runOnce(ACCOUNT_QUEUE, "fail"); + } + const outcome = await notifySystemJobFailures(); + expect(outcome.failing).toBe(0); + expect(outcome.sent).toEqual([]); + expect(dispatched.calls).toEqual([]); + }); + + it("counts a success between failures as the end of the streak", async () => { + await makeUser({ role: "ADMIN" }); + await runOnce(SYSTEM_QUEUE, "fail"); + await runOnce(SYSTEM_QUEUE, "fail"); + await runOnce(SYSTEM_QUEUE, "complete"); + await runOnce(SYSTEM_QUEUE, "fail"); + const outcome = await notifySystemJobFailures(); + expect(outcome.failing).toBe(0); + expect(outcome.sent).toEqual([]); + }); +});