From fdecefd1d3c31dabc363401cb68dd27509c98439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 16:50:13 +0200 Subject: [PATCH] fix(hrv): match an RMSSD twin on both unique identities in 0420 and the restore Migration 0420 looked for an RMSSD twin by externalId only. A Fitbit or Google Health SDNN row with no externalId, or another one, on the same night as an RMSSD row would have broken the measured-at unique key and aborted the block at boot. The twin check now matches either identity, a row left behind by a live twin is tombstoned, and an RMSSD tombstone that would make a live row look twinned is removed so the row moves. The backup restore applies the same rule and no longer counts a tombstone as a twin. --- .../migration.sql | 64 +++++++++- src/lib/export/restore-backup.ts | 55 ++++++-- .../backup-legacy-rmssd-restore.test.ts | 23 +++- .../hrv-rmssd-fitbit-google-migration.test.ts | 117 ++++++++++++++++++ 4 files changed, 245 insertions(+), 14 deletions(-) diff --git a/prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql b/prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql index cc1ae0c14..dc1652c2d 100644 --- a/prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql +++ b/prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql @@ -12,7 +12,11 @@ -- from that one daily-HRV mapper per provider (their externalIds end in -- `:hrv`), so source and type identify them. Tombstones move too, so the sync -- feed and a restore see the same type. A row whose RMSSD twin already exists --- under the `(user, type, source, externalId)` key stays where it is. +-- under either unique identity, `(user, type, source, externalId)` or +-- `(user, type, measuredAt, source, sleepStage)`, cannot move without +-- breaking that key and aborting the block; it is tombstoned instead, since +-- the twin holds the night. The second identity matters for a row with no +-- externalId, or a different one, on the same night as an RMSSD row. -- -- Derived state that would otherwise keep the old filing: -- - the FITBIT / GOOGLE_HEALTH `HEART_RATE_VARIABILITY` rollup buckets are @@ -38,6 +42,44 @@ DO $migration$ BEGIN + -- An RMSSD tombstone on the night of a live legacy row with no live RMSSD + -- twin would make that row look twinned and lose the night. It holds + -- nothing, so it goes and the live row moves into its place. + DELETE FROM "measurements" dead + WHERE dead."type" = 'HRV_RMSSD' + AND dead."source" IN ('FITBIT', 'GOOGLE_HEALTH') + AND dead."deleted_at" IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM "measurements" m + WHERE m."user_id" = dead."user_id" + AND m."type" = 'HEART_RATE_VARIABILITY' + AND m."source" = dead."source" + AND m."deleted_at" IS NULL + AND ( + m."external_id" = dead."external_id" + OR ( + m."measured_at" = dead."measured_at" + AND m."sleep_stage" IS NOT DISTINCT FROM dead."sleep_stage" + ) + ) + AND NOT EXISTS ( + SELECT 1 + FROM "measurements" live + WHERE live."user_id" = m."user_id" + AND live."type" = 'HRV_RMSSD' + AND live."source" = m."source" + AND live."deleted_at" IS NULL + AND ( + live."external_id" = m."external_id" + OR ( + live."measured_at" = m."measured_at" + AND live."sleep_stage" IS NOT DISTINCT FROM m."sleep_stage" + ) + ) + ) + ); + WITH moved AS ( UPDATE "measurements" m SET "type" = 'HRV_RMSSD', @@ -51,7 +93,13 @@ BEGIN WHERE twin."user_id" = m."user_id" AND twin."type" = 'HRV_RMSSD' AND twin."source" = m."source" - AND twin."external_id" = m."external_id" + AND ( + twin."external_id" = m."external_id" + OR ( + twin."measured_at" = m."measured_at" + AND twin."sleep_stage" IS NOT DISTINCT FROM m."sleep_stage" + ) + ) ) RETURNING m."user_id" ) @@ -59,6 +107,18 @@ BEGIN WHERE r."type" = 'HRV_RMSSD' AND r."user_id" IN (SELECT DISTINCT "user_id" FROM moved); + -- What did not move had an RMSSD twin under one of the two identities. + -- The twin holds the night; the row left behind would keep an RMSSD value + -- in the SDNN series, so it becomes a tombstone (the sync feed then tells + -- every client to drop it). + UPDATE "measurements" + SET "deleted_at" = CURRENT_TIMESTAMP, + "updated_at" = CURRENT_TIMESTAMP, + "sync_version" = "sync_version" + 1 + WHERE "type" = 'HEART_RATE_VARIABILITY' + AND "source" IN ('FITBIT', 'GOOGLE_HEALTH') + AND "deleted_at" IS NULL; + DELETE FROM "measurement_rollups" WHERE "type" = 'HEART_RATE_VARIABILITY' AND "source" IN ('FITBIT', 'GOOGLE_HEALTH'); diff --git a/src/lib/export/restore-backup.ts b/src/lib/export/restore-backup.ts index 7338d4ba5..81c0e0c34 100644 --- a/src/lib/export/restore-backup.ts +++ b/src/lib/export/restore-backup.ts @@ -465,25 +465,58 @@ export async function restoreBackup( // providers report a nightly RMSSD, which their mappers had written as the // SDNN series. A file written before that release still carries those rows // as `HEART_RATE_VARIABILITY`, and restoring it verbatim would undo the - // migration for the account. The restore applies the same move, and skips a - // legacy row whose RMSSD twin the file also holds (the migration leaves that - // row behind; a restore must not write it under either type, since writing - // it as RMSSD would collide with the twin on the unique key and writing it - // as SDNN would put RMSSD back into the SDNN series). The twins are gathered - // on the first read, under both unique identities of a measurement. - const rmssdTwinKeys = new Set(); + // migration for the account. The restore applies the migration's rule: + // + // - a legacy row moves to RMSSD, unless a LIVE RMSSD row of its source + // holds the same night under either unique identity (external id, or + // measured-at and sleep stage). Then the twin holds the night and the + // legacy row is not written under either type: as RMSSD it would break + // the unique key, as SDNN it would put RMSSD back into the SDNN series; + // - a legacy row whose only twin is a tombstone wins: the tombstone is the + // row left out, since it shares the key and holds nothing; + // - a legacy tombstone with any RMSSD twin is left out. + // + // The keys are gathered on the first read. + const rmssdLiveKeys = new Set(); + const rmssdAnyKeys = new Set(); + const legacyLiveKeys = new Set(); const noteRmssdTwin = (row: BackupMeasurement) => { + if (isLegacyRmssdRow(row)) { + if (row.deletedAt == null) { + for (const key of rmssdTwinKeysOf(row)) legacyLiveKeys.add(key); + } + return; + } if ( row.type === "HRV_RMSSD" && row.source !== undefined && LEGACY_RMSSD_SOURCES.has(row.source) ) { - for (const key of rmssdTwinKeysOf(row)) rmssdTwinKeys.add(key); + for (const key of rmssdTwinKeysOf(row)) { + rmssdAnyKeys.add(key); + if (row.deletedAt == null) rmssdLiveKeys.add(key); + } } }; - const isShadowedLegacyRmssd = (row: BackupMeasurement) => - isLegacyRmssdRow(row) && - rmssdTwinKeysOf(row).some((key) => rmssdTwinKeys.has(key)); + const anyKeyIn = (row: BackupMeasurement, keys: ReadonlySet) => + rmssdTwinKeysOf(row).some((key) => keys.has(key)); + const isShadowedLegacyRmssd = (row: BackupMeasurement): boolean => { + if (isLegacyRmssdRow(row)) { + return anyKeyIn( + row, + row.deletedAt == null ? rmssdLiveKeys : rmssdAnyKeys, + ); + } + // An RMSSD tombstone on the night a live legacy row is about to take. + return ( + row.type === "HRV_RMSSD" && + row.deletedAt != null && + row.source !== undefined && + LEGACY_RMSSD_SOURCES.has(row.source) && + anyKeyIn(row, legacyLiveKeys) && + !anyKeyIn(row, rmssdLiveKeys) + ); + }; // Set inside the transaction when the restore takes the account's fold // repair marker away; the repair is queued once the transaction commits. let foldRepairReset = false; diff --git a/tests/integration/backup-legacy-rmssd-restore.test.ts b/tests/integration/backup-legacy-rmssd-restore.test.ts index d65c888cb..25a8ad4c2 100644 --- a/tests/integration/backup-legacy-rmssd-restore.test.ts +++ b/tests/integration/backup-legacy-rmssd-restore.test.ts @@ -94,6 +94,19 @@ describe("restoring a backup written before migration 0420", () => { row("HRV_RMSSD", "FITBIT", "2026-09-03", 44), // A real SDNN reading stays where it is. row("HEART_RATE_VARIABILITY", "APPLE_HEALTH", "2026-09-01", 52), + // No externalId: twinned with a live RMSSD row only by night. + { + ...row("HEART_RATE_VARIABILITY", "GOOGLE_HEALTH", "2026-09-04", 77), + externalId: null, + }, + row("HRV_RMSSD", "GOOGLE_HEALTH", "2026-09-04", 36), + // Twinned only with a tombstone: the live row wins the night. + row("HEART_RATE_VARIABILITY", "FITBIT", "2026-09-05", 45), + { + ...row("HRV_RMSSD", "FITBIT", "2026-09-05", 1), + // Recent, so the tombstone horizon does not drop it first. + deletedAt: new Date(Date.now() - 86_400_000), + }, ], }); const legacyBest = await prisma.measurement.findFirstOrThrow({ @@ -137,7 +150,7 @@ describe("restoring a backup written before migration 0420", () => { expect(response.status).toBe(200); const restored = await prisma.measurement.findMany({ - where: { userId: owner.id }, + where: { userId: owner.id, deletedAt: null }, select: { type: true, source: true, value: true }, orderBy: [{ source: "asc" }, { measuredAt: "asc" }], }); @@ -145,8 +158,16 @@ describe("restoring a backup written before migration 0420", () => { { type: "HEART_RATE_VARIABILITY", source: "APPLE_HEALTH", value: 52 }, { type: "HRV_RMSSD", source: "FITBIT", value: 41 }, { type: "HRV_RMSSD", source: "FITBIT", value: 44 }, + { type: "HRV_RMSSD", source: "FITBIT", value: 45 }, { type: "HRV_RMSSD", source: "GOOGLE_HEALTH", value: 38 }, + { type: "HRV_RMSSD", source: "GOOGLE_HEALTH", value: 36 }, ]); + // The tombstone that shared the live row's night is the one left out. + expect( + await prisma.measurement.count({ + where: { userId: owner.id, deletedAt: { not: null } }, + }), + ).toBe(0); // The SDNN best an RMSSD reading set does not come back. expect( await prisma.personalRecord.count({ where: { userId: owner.id } }), diff --git a/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts b/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts index a2b7e4bfe..f31ce755c 100644 --- a/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts +++ b/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts @@ -268,3 +268,120 @@ describe("migration 0420", () => { expect(await rollups(USER)).toEqual(rollupsBefore); }); }); + +describe("migration 0420 with an RMSSD twin on the same night", () => { + const TWIN_USER = "hrv-rmssd-migration-twins"; + const at = new Date(D0 + 2 * DAY_MS); + + it("tombstones the SDNN row instead of breaking the measured-at key", async () => { + await prisma.user.create({ + data: { id: TWIN_USER, username: TWIN_USER, timezone: "UTC" }, + }); + await prisma.measurement.createMany({ + data: [ + // No externalId: only the (user, type, measuredAt, source, + // sleepStage) identity can see the twin. Moving it would violate + // that key and abort the whole block at boot. + { + id: "twin-sdnn-null-ext", + userId: TWIN_USER, + type: "HEART_RATE_VARIABILITY", + source: "FITBIT", + unit: "ms", + value: 41, + measuredAt: at, + externalId: null, + }, + { + id: "twin-rmssd", + userId: TWIN_USER, + type: "HRV_RMSSD", + source: "FITBIT", + unit: "ms", + value: 42, + measuredAt: at, + externalId: `${dayKey(2)}:hrv`, + }, + // A different externalId on the same night of Google Health. + { + id: "twin-sdnn-other-ext", + userId: TWIN_USER, + type: "HEART_RATE_VARIABILITY", + source: "GOOGLE_HEALTH", + unit: "ms", + value: 50, + measuredAt: at, + externalId: "older-id:hrv", + }, + { + id: "twin-rmssd-google", + userId: TWIN_USER, + type: "HRV_RMSSD", + source: "GOOGLE_HEALTH", + unit: "ms", + value: 51, + measuredAt: at, + externalId: `${dayKey(2)}:hrv`, + }, + ], + }); + + await runMigration(); + + const after = await prisma.measurement.findMany({ + where: { userId: TWIN_USER }, + select: { id: true, type: true, value: true, deletedAt: true }, + orderBy: { id: "asc" }, + }); + expect( + after.map((r) => [r.id, r.type, r.value, r.deletedAt !== null]), + ).toEqual([ + ["twin-rmssd", "HRV_RMSSD", 42, false], + ["twin-rmssd-google", "HRV_RMSSD", 51, false], + ["twin-sdnn-null-ext", "HEART_RATE_VARIABILITY", 41, true], + ["twin-sdnn-other-ext", "HEART_RATE_VARIABILITY", 50, true], + ]); + }); + + it("moves a live row whose only twin is a tombstone, dropping the tombstone", async () => { + const user = "hrv-rmssd-migration-dead-twin"; + await prisma.user.create({ + data: { id: user, username: user, timezone: "UTC" }, + }); + await prisma.measurement.createMany({ + data: [ + { + id: "dead-twin-sdnn", + userId: user, + type: "HEART_RATE_VARIABILITY", + source: "FITBIT", + unit: "ms", + value: 47, + measuredAt: at, + externalId: null, + }, + { + id: "dead-twin-rmssd", + userId: user, + type: "HRV_RMSSD", + source: "FITBIT", + unit: "ms", + value: 46, + measuredAt: at, + externalId: `${dayKey(2)}:hrv`, + deletedAt: new Date(), + }, + ], + }); + + await runMigration(); + + const after = await prisma.measurement.findMany({ + where: { userId: user }, + select: { id: true, type: true, deletedAt: true }, + }); + expect(after).toEqual([ + { id: "dead-twin-sdnn", type: "HRV_RMSSD", deletedAt: null }, + ]); + }); +});