From c5d2053e5ad9292f149412ea4ad83058c950a1af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 12:00:20 +0200 Subject: [PATCH] fix(hrv): file Fitbit and Google Health RMSSD as HRV_RMSSD Fitbit's `dailyRmssd` and Google Health's daily HRV average are both RMSSD, but the mappers wrote them into HEART_RATE_VARIABILITY, the SDNN series. The `hrv` ladder ranks both above Apple Health, so an account with an Apple Watch and a Fitbit had a series that switched estimator from day to day. The mappers now write HRV_RMSSD, and the `hrvRmssd` ladder carries both sources where they sat on the SDNN one. Migration 0420 moves the rows already written (tombstones included, with a sync-version bump so the change reaches the phone), drops their SDNN rollup buckets and the RMSSD buckets of every account whose rows moved, so the boot-time rollup discovery folds those accounts again, and deletes the SDNN personal records those rows had set. One DO block, so the move and its cleanup land together. --- .../migration.sql | 70 +++++ src/lib/fitbit/__tests__/client.test.ts | 4 +- src/lib/fitbit/client.ts | 15 +- src/lib/fitbit/mapping.md | 16 +- src/lib/fitbit/sync-metrics.ts | 2 +- .../google-health/__tests__/mappers.test.ts | 4 +- src/lib/google-health/mappers.ts | 12 +- src/lib/google-health/mapping.md | 26 +- src/lib/google-health/sync-metrics.ts | 4 +- src/lib/validations/source-priority.ts | 13 +- .../hrv-rmssd-fitbit-google-migration.test.ts | 270 ++++++++++++++++++ 11 files changed, 393 insertions(+), 43 deletions(-) create mode 100644 prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql create mode 100644 tests/integration/hrv-rmssd-fitbit-google-migration.test.ts diff --git a/prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql b/prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql new file mode 100644 index 000000000..cc1ae0c14 --- /dev/null +++ b/prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql @@ -0,0 +1,70 @@ +-- Fitbit and Google Health HRV is RMSSD; file it as such. +-- +-- Both providers report a nightly RMSSD (`dailyRmssd`; Google documents its +-- daily average as RMSSD too), but their mappers wrote it as +-- `HEART_RATE_VARIABILITY`, the SDNN series. The `hrv` ladder ranks Fitbit and +-- Google Health above Apple Health, so an account with both had a series that +-- switched estimator from day to day, and every baseline, trend and +-- correlation over it measured the switch. The mappers now write `HRV_RMSSD`; +-- this moves the rows they wrote before. +-- +-- Every `HEART_RATE_VARIABILITY` row under source FITBIT or GOOGLE_HEALTH came +-- from that one daily-HRV mapper per provider (their externalIds end in +-- `:hrv`), so source and type identify them. Tombstones move too, so the sync +-- feed and a restore see the same type. A row whose RMSSD twin already exists +-- under the `(user, type, source, externalId)` key stays where it is. +-- +-- Derived state that would otherwise keep the old filing: +-- - the FITBIT / GOOGLE_HEALTH `HEART_RATE_VARIABILITY` rollup buckets are +-- dropped (no rows remain behind them); +-- - every `HRV_RMSSD` rollup bucket of an account whose rows moved is +-- dropped, so the per-type coverage probe reads that series live and the +-- boot-time rollup discovery (`enqueueBootTimeRollupBackfill`, which flags +-- any (user, type, day) with no DAY bucket) folds the account again on the +-- next worker start. Dropping only the moved days would not do: a day that +-- another source already covers would keep its bucket and never be +-- flagged, and the moved rows would be missing from it; +-- - personal records filed under `HEART_RATE_VARIABILITY` from those sources +-- are deleted. They ranked an RMSSD value as an SDNN best; `HRV_RMSSD` keeps +-- no records, and the detector re-derives the SDNN ones from what remains. +-- +-- One `DO` block, so the move and its cleanup commit or roll back together +-- (`migrate deploy` commits each top-level statement on its own). Running it +-- again finds nothing to move and drops nothing a fold has not rebuilt. +-- +-- Reversibility: the moved rows carry source FITBIT / GOOGLE_HEALTH and type +-- `HRV_RMSSD`, which nothing else writes for those sources; setting the type +-- back and dropping their rollups reverses it. + +DO $migration$ +BEGIN + WITH moved AS ( + UPDATE "measurements" m + SET "type" = 'HRV_RMSSD', + "updated_at" = CURRENT_TIMESTAMP, + "sync_version" = m."sync_version" + 1 + WHERE m."type" = 'HEART_RATE_VARIABILITY' + AND m."source" IN ('FITBIT', 'GOOGLE_HEALTH') + AND NOT EXISTS ( + SELECT 1 + FROM "measurements" twin + WHERE twin."user_id" = m."user_id" + AND twin."type" = 'HRV_RMSSD' + AND twin."source" = m."source" + AND twin."external_id" = m."external_id" + ) + RETURNING m."user_id" + ) + DELETE FROM "measurement_rollups" r + WHERE r."type" = 'HRV_RMSSD' + AND r."user_id" IN (SELECT DISTINCT "user_id" FROM moved); + + DELETE FROM "measurement_rollups" + WHERE "type" = 'HEART_RATE_VARIABILITY' + AND "source" IN ('FITBIT', 'GOOGLE_HEALTH'); + + DELETE FROM "personal_records" + WHERE "metric_type" = 'HEART_RATE_VARIABILITY' + AND "source" IN ('FITBIT', 'GOOGLE_HEALTH'); +END +$migration$; diff --git a/src/lib/fitbit/__tests__/client.test.ts b/src/lib/fitbit/__tests__/client.test.ts index 392c818ab..30668375c 100644 --- a/src/lib/fitbit/__tests__/client.test.ts +++ b/src/lib/fitbit/__tests__/client.test.ts @@ -332,13 +332,13 @@ describe("metric mappers (classic shapes)", () => { expect(out[0]!.fieldTag).toBe("2026-05-10:spo2"); }); - it("maps HRV dailyRmssd into the canonical HEART_RATE_VARIABILITY slot", () => { + it("maps HRV dailyRmssd into HRV_RMSSD, apart from the SDNN series", () => { const out = mapHeartRateVariability({ hrv: [ { dateTime: "2026-05-10", value: { dailyRmssd: 42.7, deepRmssd: 50 } }, ], }); - expect(out[0]!.type).toBe("HEART_RATE_VARIABILITY"); + expect(out[0]!.type).toBe("HRV_RMSSD"); expect(out[0]).toMatchObject({ value: 42.7, unit: "ms" }); expect(out[0]!.fieldTag).toBe("2026-05-10:hrv"); }); diff --git a/src/lib/fitbit/client.ts b/src/lib/fitbit/client.ts index cd8cb3b85..5f936c61d 100644 --- a/src/lib/fitbit/client.ts +++ b/src/lib/fitbit/client.ts @@ -741,15 +741,16 @@ export function mapHeartRateVariability( body: unknown, tz?: string, ): FitbitMappedMeasurement[] { - // Fitbit's HRV summary is an RMSSD estimator (`value.dailyRmssd`). It lands in - // the canonical `HEART_RATE_VARIABILITY` slot that FITBIT occupies in the - // source-priority `hrv` ladder (alongside Apple / Oura), keeping cross-source - // comparison on one axis. (`HRV_RMSSD` is reserved for the WHOOP-native slot.) + // Fitbit's HRV summary is an RMSSD estimator (`value.dailyRmssd`), so it lands + // in `HRV_RMSSD` beside WHOOP / Oura / Polar. `HEART_RATE_VARIABILITY` is the + // SDNN series; the two estimators are not interchangeable, and a Fitbit night + // ranked into the SDNN ladder used to splice RMSSD values into an Apple + // Watch SDNN series day by day. const rows = readArray(body, "hrv") as FitbitDailyRow[]; return mapDailySummary( rows, { - type: "HEART_RATE_VARIABILITY", + type: "HRV_RMSSD", unit: "ms", fieldTag: "hrv", leaves: ["dailyRmssd"], @@ -987,9 +988,9 @@ export const FITBIT_FIELD_MAP: Record< note: "spo2 summary value.avg", }, heartRateVariability: { - type: "HEART_RATE_VARIABILITY", + type: "HRV_RMSSD", unit: "ms", - note: "hrv summary value.dailyRmssd (RMSSD estimator); canonical HRV slot", + note: "hrv summary value.dailyRmssd (RMSSD estimator), apart from SDNN", }, restingHeartRate: { type: "RESTING_HEART_RATE", diff --git a/src/lib/fitbit/mapping.md b/src/lib/fitbit/mapping.md index 6b554c9c1..96a8ea5d6 100644 --- a/src/lib/fitbit/mapping.md +++ b/src/lib/fitbit/mapping.md @@ -26,14 +26,14 @@ metric (kg / km); the US default is imperial. Scope: `weight`, `oxygen_saturation`, `heartrate`, `respiratory_rate`. Verified against the dev.fitbit.com reference (response shapes quoted there). -| Endpoint | MeasurementType | Unit | Source field | Grain | Note | -| ---------------------------------------------- | ------------------------ | ----------- | ------------------------------------------- | ----- | ----------------------------------------------------------- | -| `/1/user/-/body/log/weight/date/{s}/{e}.json` | `WEIGHT` | kg | `weight[].weight` | log | externalId anchors on `logId`. Picker ranks Withings above. | -| `/1/user/-/body/log/fat/date/{s}/{e}.json` | `BODY_FAT` | % | `fat[].fat` | log | externalId anchors on `logId`. | -| `/1/user/-/spo2/date/{s}/{e}.json` | `OXYGEN_SATURATION` | % | `[].value.avg` (bare array) | daily | day-keyed externalId. | -| `/1/user/-/hrv/date/{s}/{e}.json` | `HEART_RATE_VARIABILITY` | ms | `hrv[].value.dailyRmssd` | daily | RMSSD estimator → canonical HRV slot (not `HRV_RMSSD`). | -| `/1/user/-/activities/heart/date/{s}/{e}.json` | `RESTING_HEART_RATE` | bpm | `activities-heart[].value.restingHeartRate` | daily | rows with no resting HR for the day are skipped. | -| `/1/user/-/br/date/{s}/{e}.json` | `RESPIRATORY_RATE` | breaths/min | `br[].value.breathingRate` | daily | | +| Endpoint | MeasurementType | Unit | Source field | Grain | Note | +| ---------------------------------------------- | -------------------- | ----------- | ------------------------------------------- | ----- | ------------------------------------------------------------------- | +| `/1/user/-/body/log/weight/date/{s}/{e}.json` | `WEIGHT` | kg | `weight[].weight` | log | externalId anchors on `logId`. Picker ranks Withings above. | +| `/1/user/-/body/log/fat/date/{s}/{e}.json` | `BODY_FAT` | % | `fat[].fat` | log | externalId anchors on `logId`. | +| `/1/user/-/spo2/date/{s}/{e}.json` | `OXYGEN_SATURATION` | % | `[].value.avg` (bare array) | daily | day-keyed externalId. | +| `/1/user/-/hrv/date/{s}/{e}.json` | `HRV_RMSSD` | ms | `hrv[].value.dailyRmssd` | daily | RMSSD estimator, kept apart from the SDNN `HEART_RATE_VARIABILITY`. | +| `/1/user/-/activities/heart/date/{s}/{e}.json` | `RESTING_HEART_RATE` | bpm | `activities-heart[].value.restingHeartRate` | daily | rows with no resting HR for the day are skipped. | +| `/1/user/-/br/date/{s}/{e}.json` | `RESPIRATORY_RATE` | breaths/min | `br[].value.breathingRate` | daily | | Each daily-summary value passes a finite + strictly-positive guard (a 0/NaN is a garbage/empty reading and is dropped). Each endpoint caps its own date range diff --git a/src/lib/fitbit/sync-metrics.ts b/src/lib/fitbit/sync-metrics.ts index 066bd4a6d..872fc1c35 100644 --- a/src/lib/fitbit/sync-metrics.ts +++ b/src/lib/fitbit/sync-metrics.ts @@ -7,7 +7,7 @@ * - body/log/weight → WEIGHT (kg) * - body/log/fat → BODY_FAT (%) * - spo2 summary → OXYGEN_SATURATION (%) - * - hrv summary → HEART_RATE_VARIABILITY (ms, dailyRmssd) + * - hrv summary → HRV_RMSSD (ms, dailyRmssd) * - activities/heart → RESTING_HEART_RATE (bpm) * - br summary → RESPIRATORY_RATE (breaths/min) * diff --git a/src/lib/google-health/__tests__/mappers.test.ts b/src/lib/google-health/__tests__/mappers.test.ts index 16d119f23..36be47ff2 100644 --- a/src/lib/google-health/__tests__/mappers.test.ts +++ b/src/lib/google-health/__tests__/mappers.test.ts @@ -148,7 +148,7 @@ describe("mapOxygenSaturation — daily-oxygen-saturation summary", () => { }); describe("mapHeartRateVariability — daily-heart-rate-variability summary", () => { - it("maps averageHeartRateVariabilityMilliseconds to the HRV slot in ms", () => { + it("maps the RMSSD daily average to HRV_RMSSD in ms", () => { const rows = mapHeartRateVariability({ dailyHeartRateVariability: { averageHeartRateVariabilityMilliseconds: 42.5, @@ -157,7 +157,7 @@ describe("mapHeartRateVariability — daily-heart-rate-variability summary", () }); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ - type: "HEART_RATE_VARIABILITY", + type: "HRV_RMSSD", unit: "ms", value: 42.5, fieldTag: "2026-06-01:hrv", diff --git a/src/lib/google-health/mappers.ts b/src/lib/google-health/mappers.ts index 0e6bfe02a..384d7d6d9 100644 --- a/src/lib/google-health/mappers.ts +++ b/src/lib/google-health/mappers.ts @@ -511,16 +511,16 @@ export function mapHeartRateVariability( tz?: string, ): GoogleHealthMappedMeasurement[] { const dt = GOOGLE_HEALTH_DATA_TYPES.heartRateVariability; - // The daily field is an unlabelled "average HRV ms". Per the design decision - // it lands in the SDNN-lineage `HEART_RATE_VARIABILITY` slot - // (Apple-comparable), NOT WHOOP's `HRV_RMSSD` (reserved for the WHOOP-native - // estimator). The per-sample type carries explicit RMSSD + SDNN fields — - // re-confirm the estimator against live data and revisit if warranted. + // Google documents the daily average as RMSSD ("average heart rate + // variability calculated using the root mean square of successive + // differences"), the same estimator the classic Fitbit API reports as + // `dailyRmssd`. It lands in `HRV_RMSSD`, apart from the SDNN + // `HEART_RATE_VARIABILITY` series. return mapSimple( point, dt, { - type: "HEART_RATE_VARIABILITY", + type: "HRV_RMSSD", unit: "ms", fieldTag: "hrv", valuePaths: [`${dt.key}.averageHeartRateVariabilityMilliseconds`], diff --git a/src/lib/google-health/mapping.md b/src/lib/google-health/mapping.md index 48e46beb3..eb958bf6b 100644 --- a/src/lib/google-health/mapping.md +++ b/src/lib/google-health/mapping.md @@ -81,19 +81,19 @@ The exercise civil bound is derived from the watermark in the USER'S zone. Scope: `googlehealth.health_metrics_and_measurements.readonly`. -| Data type (path) | Payload value read | MeasurementType | Unit | fieldTag | Grain | Note | -| ------------------------------------- | --------------------------------------------------------------------- | ------------------------ | ----------- | ------------ | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `weight` | `weight.weightGrams` ÷ 1000 | `WEIGHT` | kg | `weight` | sample | Grams on the wire. Picker ranks a real Withings scale above Google Health. | -| `body-fat` | `bodyFat.percentage` | `BODY_FAT` | % | `body_fat` | sample | Union key is camelCase `bodyFat`. | -| `daily-oxygen-saturation` | `dailyOxygenSaturation.averagePercentage` | `OXYGEN_SATURATION` | % | `spo2` | daily | The bare `oxygen-saturation` type is per-SAMPLE and rejects a `.date` filter. | -| `daily-heart-rate-variability` | `dailyHeartRateVariability.averageHeartRateVariabilityMilliseconds` | `HEART_RATE_VARIABILITY` | ms | `hrv` | daily | **Decision:** SDNN slot (Apple-comparable), NOT `HRV_RMSSD`. The daily field is an unlabelled "average HRV ms" — re-confirm estimator live. | -| `daily-resting-heart-rate` | `dailyRestingHeartRate.beatsPerMinute` (int64 string) | `RESTING_HEART_RATE` | bpm | `rhr` | daily | | -| `daily-respiratory-rate` | `dailyRespiratoryRate.breathsPerMinute` (number) | `RESPIRATORY_RATE` | breaths/min | `resp_rate` | daily | `respiratory-rate` does NOT exist in the catalogue. Schema is `{date, breathsPerMinute}` — the earlier `dailyRespiratoryRateBpm` leaf never existed. | -| `heart-rate` | `heartRate.beatsPerMinute` (int64 string) | `PULSE` | bpm | `hr` | sample | Intraday spot HR; per-minute volume — consider the 14-day-max rollup if PULSE should stay coarse. | -| `blood-glucose` | `bloodGlucose.bloodGlucoseMilligramsPerDeciliter` (number) | `BLOOD_GLUCOSE` | mg/dL | `glucose` | sample | Already in the canonical mg/dL storage unit. Meal/timing/specimen enums not stored. | -| `core-body-temperature` | `coreBodyTemperature.temperatureCelsius` (number) | `BODY_TEMPERATURE` | celsius | `core_temp` | sample | Core slot — distinct from SKIN_TEMPERATURE / WRIST_TEMPERATURE. | -| `daily-sleep-temperature-derivations` | `dailySleepTemperatureDerivations.nightlyTemperatureCelsius` (number) | `WRIST_TEMPERATURE` | celsius | `wrist_temp` | daily | ABSOLUTE nightly skin temperature (mean over the night's samples) — the schema refutes the earlier "deviation-only" premise. Baseline/stddev derivations not stored. | -| `height` | `height.heightMillimeters` (int64 string) ÷ 10 | `User.heightCm` | cm | — | sample | Profile seed — written ONLY when null; never a Measurement. List order is DESCENDING → the seed picks max(sampleTime) explicitly. The earlier `heightMeters` leaf never existed. | +| Data type (path) | Payload value read | MeasurementType | Unit | fieldTag | Grain | Note | +| ------------------------------------- | --------------------------------------------------------------------- | -------------------- | ----------- | ------------ | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `weight` | `weight.weightGrams` ÷ 1000 | `WEIGHT` | kg | `weight` | sample | Grams on the wire. Picker ranks a real Withings scale above Google Health. | +| `body-fat` | `bodyFat.percentage` | `BODY_FAT` | % | `body_fat` | sample | Union key is camelCase `bodyFat`. | +| `daily-oxygen-saturation` | `dailyOxygenSaturation.averagePercentage` | `OXYGEN_SATURATION` | % | `spo2` | daily | The bare `oxygen-saturation` type is per-SAMPLE and rejects a `.date` filter. | +| `daily-heart-rate-variability` | `dailyHeartRateVariability.averageHeartRateVariabilityMilliseconds` | `HRV_RMSSD` | ms | `hrv` | daily | Google documents the daily average as RMSSD, so it lands in `HRV_RMSSD`, apart from the SDNN `HEART_RATE_VARIABILITY`. | +| `daily-resting-heart-rate` | `dailyRestingHeartRate.beatsPerMinute` (int64 string) | `RESTING_HEART_RATE` | bpm | `rhr` | daily | | +| `daily-respiratory-rate` | `dailyRespiratoryRate.breathsPerMinute` (number) | `RESPIRATORY_RATE` | breaths/min | `resp_rate` | daily | `respiratory-rate` does NOT exist in the catalogue. Schema is `{date, breathsPerMinute}` — the earlier `dailyRespiratoryRateBpm` leaf never existed. | +| `heart-rate` | `heartRate.beatsPerMinute` (int64 string) | `PULSE` | bpm | `hr` | sample | Intraday spot HR; per-minute volume — consider the 14-day-max rollup if PULSE should stay coarse. | +| `blood-glucose` | `bloodGlucose.bloodGlucoseMilligramsPerDeciliter` (number) | `BLOOD_GLUCOSE` | mg/dL | `glucose` | sample | Already in the canonical mg/dL storage unit. Meal/timing/specimen enums not stored. | +| `core-body-temperature` | `coreBodyTemperature.temperatureCelsius` (number) | `BODY_TEMPERATURE` | celsius | `core_temp` | sample | Core slot — distinct from SKIN_TEMPERATURE / WRIST_TEMPERATURE. | +| `daily-sleep-temperature-derivations` | `dailySleepTemperatureDerivations.nightlyTemperatureCelsius` (number) | `WRIST_TEMPERATURE` | celsius | `wrist_temp` | daily | ABSOLUTE nightly skin temperature (mean over the night's samples) — the schema refutes the earlier "deviation-only" premise. Baseline/stddev derivations not stored. | +| `height` | `height.heightMillimeters` (int64 string) ÷ 10 | `User.heightCm` | cm | — | sample | Profile seed — written ONLY when null; never a Measurement. List order is DESCENDING → the seed picks max(sampleTime) explicitly. The earlier `heightMeters` leaf never existed. | Daily-summary `date` fields are `{year,month,day}` objects, anchored at UTC-midday so a timezone shift can't roll the civil day. Every value passes a diff --git a/src/lib/google-health/sync-metrics.ts b/src/lib/google-health/sync-metrics.ts index 95ed4a564..8115ac280 100644 --- a/src/lib/google-health/sync-metrics.ts +++ b/src/lib/google-health/sync-metrics.ts @@ -8,7 +8,7 @@ * - weight → WEIGHT * - body-fat → BODY_FAT * - daily-oxygen-saturation → OXYGEN_SATURATION - * - daily-heart-rate-variability → HEART_RATE_VARIABILITY (SDNN slot; see mapper) + * - daily-heart-rate-variability → HRV_RMSSD (RMSSD; see mapper) * - daily-resting-heart-rate → RESTING_HEART_RATE * - daily-respiratory-rate → RESPIRATORY_RATE * - heart-rate (intraday) → PULSE @@ -117,7 +117,7 @@ const METRIC_RESOURCES: MetricResource[] = [ dataType: GOOGLE_HEALTH_DATA_TYPES.heartRateVariability, map: mapHeartRateVariability, verb: "fetchHrv", - type: "HEART_RATE_VARIABILITY", + type: "HRV_RMSSD", }, { dataType: GOOGLE_HEALTH_DATA_TYPES.restingHeartRate, diff --git a/src/lib/validations/source-priority.ts b/src/lib/validations/source-priority.ts index 476daaed5..6ad587dbd 100644 --- a/src/lib/validations/source-priority.ts +++ b/src/lib/validations/source-priority.ts @@ -404,8 +404,17 @@ export const DEFAULT_SOURCE_PRIORITY: Required = { // on now. Mirrors the recovery ladder's native-above-computed ordering. sleepDebt: ["WHOOP", "OURA", "POLAR", "COMPUTED"], // v1.42 (#1110) — the straps and rings that measure RMSSD overnight lead; - // the HealthKit relay of a third-party reading is the fallback. - hrvRmssd: ["WHOOP", "OURA", "POLAR", "APPLE_HEALTH"], + // the HealthKit relay of a third-party reading is the fallback. Fitbit and + // Google Health report RMSSD too and take the places they hold on the SDNN + // ladder they used to be filed under. + hrvRmssd: [ + "WHOOP", + "FITBIT", + "GOOGLE_HEALTH", + "OURA", + "POLAR", + "APPLE_HEALTH", + ], }; /** diff --git a/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts b/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts new file mode 100644 index 000000000..a2b7e4bfe --- /dev/null +++ b/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts @@ -0,0 +1,270 @@ +/** + * Migration 0420 moves the RMSSD rows Fitbit and Google Health wrote into the + * SDNN series over to `HRV_RMSSD`, and clears the derived state that kept the + * old filing: their SDNN rollup buckets, the RMSSD buckets of the accounts + * whose rows moved (so the boot-time discovery folds them again), and the + * SDNN personal records those rows set. Apple Health SDNN and WHOOP RMSSD are + * left exactly as they were, and a rerun changes nothing. + */ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { Prisma } from "@/generated/prisma/client"; +import { + enqueueBootTimeRollupBackfill, + recomputeUserRollups, +} from "@/lib/rollups/measurement-rollups"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: vi.fn(() => null), +})); +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; + +const prisma = getPrismaClient(); +const USER = "hrv-rmssd-migration"; +const BYSTANDER = "hrv-rmssd-migration-apple-only"; + +const MIGRATION = readFileSync( + resolve( + process.cwd(), + "prisma", + "migrations", + "0420_hrv_rmssd_from_fitbit_and_google_health", + "migration.sql", + ), + "utf8", +); + +async function runMigration(): Promise { + // The file is one `DO` block; the comments around it are harmless. + await prisma.$executeRawUnsafe(MIGRATION); +} + +const DAY_MS = 86_400_000; +/** Ten days ago at 07:00 UTC, so every row sits inside the fold window. */ +const D0 = + Math.floor(Date.now() / DAY_MS) * DAY_MS - 10 * DAY_MS + 7 * 3_600_000; +const dayKey = (i: number) => + new Date(D0 + i * DAY_MS).toISOString().slice(0, 10); + +function rows(): Prisma.MeasurementCreateManyInput[] { + const out: Prisma.MeasurementCreateManyInput[] = []; + for (let i = 0; i < 4; i += 1) { + const at = new Date(D0 + i * DAY_MS); + out.push( + { + id: `fitbit-${i}`, + userId: USER, + type: "HEART_RATE_VARIABILITY", + source: "FITBIT", + unit: "ms", + value: 40 + i, + measuredAt: at, + externalId: `${dayKey(i)}:hrv`, + }, + { + id: `apple-${i}`, + userId: USER, + type: "HEART_RATE_VARIABILITY", + source: "APPLE_HEALTH", + unit: "ms", + value: 70 + i, + measuredAt: new Date(at.getTime() + 3_600_000), + }, + // WHOOP covers the same nights in the RMSSD series already: the moved + // Fitbit rows land on days that already have an RMSSD bucket. + { + id: `whoop-${i}`, + userId: USER, + type: "HRV_RMSSD", + source: "WHOOP", + unit: "ms", + value: 55 + i, + measuredAt: new Date(at.getTime() + 1_800_000), + externalId: `sleep-${i}:hrv_rmssd`, + }, + { + id: `bystander-${i}`, + userId: BYSTANDER, + type: "HEART_RATE_VARIABILITY", + source: "APPLE_HEALTH", + unit: "ms", + value: 80 + i, + measuredAt: at, + }, + ); + } + out.push( + { + // On a night WHOOP covers too: every live moved row lands on a day that + // already has an RMSSD bucket, so only the dropped buckets can get the + // account flagged for a fold. + id: "google-0", + userId: USER, + type: "HEART_RATE_VARIABILITY", + source: "GOOGLE_HEALTH", + unit: "ms", + value: 44, + measuredAt: new Date(D0 + DAY_MS + 7_200_000), + externalId: `${dayKey(1)}:hrv`, + }, + // A tombstone moves too, so the sync feed reports one type for the id. + { + id: "fitbit-deleted", + userId: USER, + type: "HEART_RATE_VARIABILITY", + source: "FITBIT", + unit: "ms", + value: 39, + measuredAt: new Date(D0 + 6 * DAY_MS), + externalId: `${dayKey(6)}:hrv`, + deletedAt: new Date(), + }, + ); + return out; +} + +type RollupRow = { type: string; source: string; granularity: string }; +const rollups = (userId: string) => + prisma.$queryRaw` + SELECT "type"::text AS type, "source"::text AS source, + "granularity"::text AS granularity + FROM measurement_rollups + WHERE "user_id" = ${userId} + ORDER BY 1, 2, 3 + `; +const pairs = (all: RollupRow[]) => + [...new Set(all.map((r) => `${r.type}/${r.source}`))].sort(); + +let bystanderBefore: RollupRow[] = []; + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.createMany({ + data: [ + { id: USER, username: USER, timezone: "UTC" }, + { id: BYSTANDER, username: BYSTANDER, timezone: "UTC" }, + ], + }); + await prisma.measurement.createMany({ data: rows() }); + await recomputeUserRollups(USER); + await recomputeUserRollups(BYSTANDER); + await prisma.personalRecord.createMany({ + data: [ + { + userId: USER, + metricType: "HEART_RATE_VARIABILITY", + direction: "MAX", + value: 43, + unit: "ms", + achievedAt: new Date(D0 + 3 * DAY_MS), + source: "FITBIT", + sourceMeasurementId: "fitbit-3", + }, + { + userId: USER, + metricType: "HEART_RATE_VARIABILITY", + direction: "MAX", + value: 73, + unit: "ms", + achievedAt: new Date(D0 + 3 * DAY_MS + 3_600_000), + source: "APPLE_HEALTH", + sourceMeasurementId: "apple-3", + }, + ], + }); + bystanderBefore = await rollups(BYSTANDER); + await runMigration(); +}, 120_000); + +describe("migration 0420", () => { + it("moves every Fitbit and Google Health SDNN row to HRV_RMSSD, tombstones included", async () => { + const moved = await prisma.measurement.findMany({ + where: { userId: USER, source: { in: ["FITBIT", "GOOGLE_HEALTH"] } }, + select: { id: true, type: true, syncVersion: true, value: true }, + orderBy: { id: "asc" }, + }); + expect(moved).toHaveLength(6); + for (const row of moved) { + expect(row.type, row.id).toBe("HRV_RMSSD"); + // A type change must reach the phone through the sync feed. + expect(row.syncVersion, row.id).toBe(2); + } + }); + + it("leaves Apple Health SDNN and WHOOP RMSSD rows alone", async () => { + const apple = await prisma.measurement.findMany({ + where: { source: "APPLE_HEALTH" }, + select: { type: true, syncVersion: true }, + }); + expect(apple).toHaveLength(8); + expect(apple.every((r) => r.type === "HEART_RATE_VARIABILITY")).toBe(true); + expect(apple.every((r) => r.syncVersion === 1)).toBe(true); + const whoop = await prisma.measurement.findMany({ + where: { source: "WHOOP" }, + select: { type: true, syncVersion: true }, + }); + expect(whoop.every((r) => r.type === "HRV_RMSSD")).toBe(true); + expect(whoop.every((r) => r.syncVersion === 1)).toBe(true); + }); + + it("drops the SDNN buckets of the moved sources and the account's RMSSD buckets", async () => { + expect(pairs(await rollups(USER))).toEqual([ + "HEART_RATE_VARIABILITY/APPLE_HEALTH", + ]); + expect(await rollups(BYSTANDER)).toEqual(bystanderBefore); + }); + + it("leaves the account where the boot-time discovery folds it again", async () => { + const send = vi.fn(async () => "job-id"); + vi.mocked(getGlobalBoss).mockReturnValue({ send } as never); + try { + const result = await enqueueBootTimeRollupBackfill(); + expect(result.error).toBeNull(); + const users = send.mock.calls.map( + (call) => (call as unknown as [string, { userId: string }])[1].userId, + ); + expect(users).toContain(USER); + expect(users).not.toContain(BYSTANDER); + } finally { + vi.mocked(getGlobalBoss).mockReturnValue(null as never); + } + }); + + it("refolds into an RMSSD series that carries both straps", async () => { + await recomputeUserRollups(USER); + expect(pairs(await rollups(USER))).toEqual([ + "HEART_RATE_VARIABILITY/APPLE_HEALTH", + "HRV_RMSSD/FITBIT", + "HRV_RMSSD/GOOGLE_HEALTH", + "HRV_RMSSD/WHOOP", + ]); + }); + + it("deletes the SDNN records the moved rows set and keeps the rest", async () => { + const records = await prisma.personalRecord.findMany({ + where: { userId: USER }, + select: { source: true, metricType: true }, + }); + expect(records).toEqual([ + { source: "APPLE_HEALTH", metricType: "HEART_RATE_VARIABILITY" }, + ]); + }); + + it("is idempotent", async () => { + const measurementsBefore = await prisma.measurement.findMany({ + orderBy: { id: "asc" }, + }); + const rollupsBefore = await rollups(USER); + await runMigration(); + expect( + await prisma.measurement.findMany({ orderBy: { id: "asc" } }), + ).toEqual(measurementsBefore); + expect(await rollups(USER)).toEqual(rollupsBefore); + }); +});