From d21d29572da4e3a8df1932856e3a25355d8042b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 11:42:53 +0200 Subject: [PATCH 1/9] fix(whoop): keep the night's sleep when its recovery is deleted The v2 recovery webhook carries the id of the associated sleep, and a recovery's rows are keyed off that same sleep id. The delete branch matched on the `:` prefix, so a `recovery.deleted` event also soft-deleted every sleep row of that night, and a `sleep.deleted` event took the recovery with it. Recovery now names its exact ids (pinned against the mapper's field tags); sleep keeps the prefix minus those ids. The targeted refresh for `recovery.updated` passed the same sleep UUID into `/v2/cycle/{cycleId}/recovery`, which answered 404 and booked a sync failure on every recovery webhook. It now reads the sleep for its `cycle_id` first. --- .../webhook/[token]/__tests__/route.test.ts | 16 ++- .../__tests__/sync-recovery-by-id.test.ts | 85 +++++++++++++ .../__tests__/webhook-notification.test.ts | 119 +++++++++++++++++- src/lib/whoop/client.ts | 27 +++- src/lib/whoop/sync-recovery.ts | 33 +++-- src/lib/whoop/webhook-handler.ts | 38 +++++- 6 files changed, 295 insertions(+), 23 deletions(-) create mode 100644 src/lib/whoop/__tests__/sync-recovery-by-id.test.ts diff --git a/src/app/api/whoop/webhook/[token]/__tests__/route.test.ts b/src/app/api/whoop/webhook/[token]/__tests__/route.test.ts index ddf3950fe..93f212a60 100644 --- a/src/app/api/whoop/webhook/[token]/__tests__/route.test.ts +++ b/src/app/api/whoop/webhook/[token]/__tests__/route.test.ts @@ -191,14 +191,16 @@ describe("POST /api/whoop/webhook/[token]", () => { expect(getGlobalBoss).not.toHaveBeenCalled(); }); - it("soft-deletes matching rows on a `*.deleted` event", async () => { + it("soft-deletes the recovery's own rows on a `recovery.deleted` event", async () => { vi.mocked(prisma.whoopConnection.findUnique).mockResolvedValue({ userId: "user-1", } as never); + // The v2 recovery webhook id is the associated sleep's UUID, which also + // prefixes the night's sleep rows — the match names the recovery's ids. const req = signedRequest({ user_id: 42, - id: "rec-uuid", + id: "sleep-uuid", type: "recovery.deleted", }); const res = await POST(req, ctx(SECRET)); @@ -208,7 +210,15 @@ describe("POST /api/whoop/webhook/[token]", () => { where: { userId: "user-1", source: "WHOOP", - externalId: { startsWith: "rec-uuid:" }, + externalId: { + in: [ + "sleep-uuid:recovery", + "sleep-uuid:hrv_rmssd", + "sleep-uuid:rhr", + "sleep-uuid:spo2", + "sleep-uuid:skin_temp", + ], + }, deletedAt: null, }, data: { deletedAt: expect.any(Date), syncVersion: { increment: 1 } }, diff --git a/src/lib/whoop/__tests__/sync-recovery-by-id.test.ts b/src/lib/whoop/__tests__/sync-recovery-by-id.test.ts new file mode 100644 index 000000000..31176c597 --- /dev/null +++ b/src/lib/whoop/__tests__/sync-recovery-by-id.test.ts @@ -0,0 +1,85 @@ +/** + * The v2 recovery webhook carries the id of the associated SLEEP, and WHOOP + * reads a recovery only through its cycle. The targeted refresh must resolve + * sleep -> cycle -> recovery; passing the sleep UUID straight into the cycle + * route answered 404 on every recovery webhook and booked a sync failure. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { fetchSleepById, fetchRecoveryByCycleId, upsertMock, fetchErrorMock } = + vi.hoisted(() => ({ + fetchSleepById: vi.fn(), + fetchRecoveryByCycleId: vi.fn(), + upsertMock: vi.fn(async (_u: string, rows: unknown[]) => rows.length), + fetchErrorMock: vi.fn(async () => 0), + })); + +vi.mock("@/lib/db", () => ({ prisma: {} })); + +vi.mock("../sync-core", async (importOriginal) => ({ + ...(await importOriginal()), + getValidToken: vi.fn(async () => ({ accessToken: "tok" })), + upsertWhoopMeasurements: upsertMock, + handleCollectionFetchError: fetchErrorMock, +})); + +vi.mock("../client", async (importOriginal) => ({ + ...(await importOriginal()), + fetchSleepById, + fetchRecoveryByCycleId, +})); + +import { syncWhoopRecoveryById } from "../sync-recovery"; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("syncWhoopRecoveryById", () => { + it("reads the sleep for its cycle id, then the recovery through the cycle", async () => { + fetchSleepById.mockResolvedValue({ id: "sleep-uuid", cycle_id: 991 }); + fetchRecoveryByCycleId.mockResolvedValue({ + cycle_id: 991, + sleep_id: "sleep-uuid", + user_id: 42, + created_at: "2026-06-01T07:00:00.000Z", + updated_at: "2026-06-01T07:05:00.000Z", + score_state: "SCORED", + score: { + user_calibrating: false, + recovery_score: 70, + resting_heart_rate: 52, + hrv_rmssd_milli: 61, + }, + }); + + const imported = await syncWhoopRecoveryById("user-1", "sleep-uuid"); + + expect(fetchSleepById).toHaveBeenCalledWith("tok", "sleep-uuid"); + expect(fetchRecoveryByCycleId).toHaveBeenCalledWith("tok", "991"); + expect(fetchErrorMock).not.toHaveBeenCalled(); + expect(imported).toBe(3); + const rows = upsertMock.mock.calls[0][1] as Array<{ externalId: string }>; + expect(rows.map((r) => r.externalId)).toEqual([ + "sleep-uuid:recovery", + "sleep-uuid:hrv_rmssd", + "sleep-uuid:rhr", + ]); + }); + + it("imports nothing when the sleep carries no cycle id", async () => { + fetchSleepById.mockResolvedValue({ id: "sleep-uuid" }); + + expect(await syncWhoopRecoveryById("user-1", "sleep-uuid")).toBe(0); + expect(fetchRecoveryByCycleId).not.toHaveBeenCalled(); + expect(upsertMock).not.toHaveBeenCalled(); + }); + + it("routes a failed sleep read through the shared fetch-error path", async () => { + const err = new Error("boom"); + fetchSleepById.mockRejectedValue(err); + + await syncWhoopRecoveryById("user-1", "sleep-uuid"); + expect(fetchErrorMock).toHaveBeenCalledWith("recovery", "user-1", err); + }); +}); diff --git a/src/lib/whoop/__tests__/webhook-notification.test.ts b/src/lib/whoop/__tests__/webhook-notification.test.ts index f118932bb..ba08d8caa 100644 --- a/src/lib/whoop/__tests__/webhook-notification.test.ts +++ b/src/lib/whoop/__tests__/webhook-notification.test.ts @@ -54,7 +54,18 @@ vi.mock("@/lib/logging/context", () => ({ getEvent: () => ({ setAuth: vi.fn(), addWarning: vi.fn() }), })); -import { processWhoopNotification } from "../webhook-handler"; +import { + deletedResourceExternalIdFilter, + processWhoopNotification, +} from "../webhook-handler"; +import { + mapRecovery, + mapSleep, + WHOOP_RECOVERY_FIELD_TAGS, + type WhoopRecovery, + type WhoopSleep, +} from "../client"; +import type { Prisma } from "@/generated/prisma/client"; beforeEach(() => { vi.clearAllMocks(); @@ -79,7 +90,7 @@ describe("processWhoopNotification — fetch-by-id enqueue", () => { }); }); - it("carries a numeric resource id (cycle-based recovery) as a string", async () => { + it("carries a numeric resource id as a string", async () => { await processWhoopNotification({ user_id: 42, id: 9981, @@ -148,3 +159,107 @@ describe("processWhoopNotification — fetch-by-id enqueue", () => { ); }); }); + +// One night as WHOOP v2 models it: a sleep and the recovery scored from it. +// Both key their rows off the sleep UUID, which is also the id the recovery +// webhook carries. +const SLEEP: WhoopSleep = { + id: "sleep-uuid", + cycle_id: 991, + user_id: 42, + created_at: "2026-06-01T05:00:00.000Z", + updated_at: "2026-06-01T07:00:00.000Z", + start: "2026-05-31T23:00:00.000Z", + end: "2026-06-01T07:00:00.000Z", + nap: false, + score_state: "SCORED", + score: { + stage_summary: { + total_in_bed_time_milli: 8 * 3_600_000, + total_awake_time_milli: 30 * 60_000, + total_light_sleep_time_milli: 4 * 3_600_000, + total_slow_wave_sleep_time_milli: 90 * 60_000, + total_rem_sleep_time_milli: 2 * 3_600_000, + disturbance_count: 4, + }, + sleep_needed: { + baseline_milli: 8 * 3_600_000, + need_from_sleep_debt_milli: 0, + need_from_recent_strain_milli: 0, + need_from_recent_nap_milli: 0, + }, + respiratory_rate: 15, + sleep_performance_percentage: 90, + sleep_consistency_percentage: 80, + sleep_efficiency_percentage: 92, + }, +}; +const RECOVERY: WhoopRecovery = { + cycle_id: 991, + sleep_id: "sleep-uuid", + user_id: 42, + created_at: "2026-06-01T07:00:00.000Z", + updated_at: "2026-06-01T07:05:00.000Z", + score_state: "SCORED", + score: { + user_calibrating: false, + recovery_score: 70, + resting_heart_rate: 52, + hrv_rmssd_milli: 61, + spo2_percentage: 96, + skin_temp_celsius: 33.4, + }, +}; +const SLEEP_IDS = mapSleep(SLEEP).map( + (m) => m.externalId ?? `${SLEEP.id}:${m.fieldTag}`, +); +const RECOVERY_IDS = mapRecovery(RECOVERY).map( + (m) => `${RECOVERY.sleep_id}:${m.fieldTag}`, +); + +/** The subset of Prisma's StringFilter semantics the filter uses. */ +function matches(filter: Prisma.StringFilter, id: string): boolean { + if (filter.in && !(filter.in as string[]).includes(id)) return false; + if (filter.notIn && (filter.notIn as string[]).includes(id)) return false; + if (filter.startsWith && !id.startsWith(filter.startsWith as string)) { + return false; + } + return true; +} + +describe("deletedResourceExternalIdFilter — one night, two resources", () => { + it("pins the recovery field-tag list to what the mapper emits", () => { + expect(new Set(mapRecovery(RECOVERY).map((m) => m.fieldTag))).toEqual( + new Set(WHOOP_RECOVERY_FIELD_TAGS), + ); + }); + + it("a recovery deletion removes the recovery and leaves the night's sleep", () => { + const filter = deletedResourceExternalIdFilter("recovery", "sleep-uuid"); + expect(SLEEP_IDS.length).toBeGreaterThan(5); + expect(RECOVERY_IDS.filter((id) => matches(filter, id))).toEqual( + RECOVERY_IDS, + ); + expect(SLEEP_IDS.filter((id) => matches(filter, id))).toEqual([]); + }); + + it("a sleep deletion removes the sleep and leaves its recovery", () => { + const filter = deletedResourceExternalIdFilter("sleep", "sleep-uuid"); + expect(SLEEP_IDS.filter((id) => matches(filter, id))).toEqual(SLEEP_IDS); + expect(RECOVERY_IDS.filter((id) => matches(filter, id))).toEqual([]); + }); + + it("a recovery.deleted webhook passes the exact-id match to both reads", async () => { + updateMany.mockResolvedValue({ count: 5 }); + await processWhoopNotification({ + user_id: 42, + id: "sleep-uuid", + type: "recovery.deleted", + }); + const expected = { + in: WHOOP_RECOVERY_FIELD_TAGS.map((t) => `sleep-uuid:${t}`), + }; + expect(findMany.mock.calls[0][0].where.externalId).toEqual(expected); + expect(updateMany.mock.calls[0][0].where.externalId).toEqual(expected); + }); +}); diff --git a/src/lib/whoop/client.ts b/src/lib/whoop/client.ts index f6b76aa69..17d2aa529 100644 --- a/src/lib/whoop/client.ts +++ b/src/lib/whoop/client.ts @@ -180,6 +180,12 @@ export interface WhoopRecovery { /** Sleep activity record. */ export interface WhoopSleep { id: string; + /** + * The cycle this sleep belongs to. A v2 recovery is read through its cycle, + * and the recovery webhook carries the SLEEP id, so the targeted recovery + * refresh resolves sleep -> cycle -> recovery through this field. + */ + cycle_id?: number; user_id: number; created_at: string; updated_at: string; @@ -449,8 +455,10 @@ export function fetchCycleById( /** * WHOOP v2 has no recovery-by-recovery-id route — a recovery is read through - * its cycle (`GET /v2/cycle/{cycleId}/recovery`). The recovery webhook carries - * the cycle id, so the targeted refresh resolves the recovery from there. + * its cycle (`GET /v2/cycle/{cycleId}/recovery`). The v2 recovery webhook + * carries the id of the associated SLEEP, not the cycle, so the targeted + * refresh first reads the sleep for its `cycle_id` (see + * `syncWhoopRecoveryById`). */ export function fetchRecoveryByCycleId( accessToken: string, @@ -523,6 +531,21 @@ export interface MappedMeasurement { reconstructed?: boolean; } +/** + * Every field-tag `mapRecovery` can emit. A recovery's rows are keyed + * `:` and share the `:` prefix with the rows of + * the sleep it belongs to, so a recovery deletion must name these exact ids + * rather than match on the prefix (which would take the night's sleep rows + * with it). Pinned against the mapper by a test. + */ +export const WHOOP_RECOVERY_FIELD_TAGS = [ + "recovery", + "hrv_rmssd", + "rhr", + "spo2", + "skin_temp", +] as const; + /** * Map one WHOOP recovery record into Measurement readings. An unscored record * (`score === null`) yields nothing. `recovery.score.recovery_score` → diff --git a/src/lib/whoop/sync-recovery.ts b/src/lib/whoop/sync-recovery.ts index 617d1eebf..496371ed7 100644 --- a/src/lib/whoop/sync-recovery.ts +++ b/src/lib/whoop/sync-recovery.ts @@ -9,7 +9,12 @@ * sleep UUID (v2 recovery carries `sleep_id`, not a stable recovery id) so the * externalId is stable across re-scores. */ -import { fetchRecoveries, fetchRecoveryByCycleId, mapRecovery } from "./client"; +import { + fetchRecoveries, + fetchRecoveryByCycleId, + fetchSleepById, + mapRecovery, +} from "./client"; import { getValidToken, incrementalStart, @@ -71,24 +76,32 @@ export async function syncUserRecovery( } /** - * Webhook-driven targeted refresh: resolve ONE recovery by its cycle id and - * upsert its readings, instead of re-walking the whole collection. WHOOP v2 - * reads a recovery through its cycle (`/v2/cycle/{cycleId}/recovery`), and the - * recovery webhook carries the cycle id. An unscored / since-deleted record - * yields nothing. Does NOT advance the resource cursor — a single-id refresh - * proves nothing about records between the cursor and now, so the cron/overlap - * path stays responsible for moving the cursor. + * Webhook-driven targeted refresh: resolve ONE recovery and upsert its + * readings, instead of re-walking the whole collection. The v2 recovery + * webhook carries the id of the associated SLEEP (a UUID), but WHOOP reads a + * recovery only through its cycle (`/v2/cycle/{cycleId}/recovery`), so the + * sleep is read first for its `cycle_id`. Passing the sleep UUID as a cycle id + * answered 404 on every recovery webhook and booked each one as a sync + * failure. An unscored / since-deleted record yields nothing. Does NOT advance + * the resource cursor — a single-id refresh proves nothing about records + * between the cursor and now, so the cron/overlap path stays responsible for + * moving the cursor. */ export async function syncWhoopRecoveryById( userId: string, - cycleId: string, + sleepId: string, ): Promise { const tokenInfo = await getValidToken(userId); if (!tokenInfo) return 0; let record: Awaited>; try { - record = await fetchRecoveryByCycleId(tokenInfo.accessToken, cycleId); + const sleep = await fetchSleepById(tokenInfo.accessToken, sleepId); + if (sleep.cycle_id == null) return 0; + record = await fetchRecoveryByCycleId( + tokenInfo.accessToken, + String(sleep.cycle_id), + ); } catch (err) { return handleCollectionFetchError("recovery", userId, err); } diff --git a/src/lib/whoop/webhook-handler.ts b/src/lib/whoop/webhook-handler.ts index 6fe2cb8a5..5cdaff6ff 100644 --- a/src/lib/whoop/webhook-handler.ts +++ b/src/lib/whoop/webhook-handler.ts @@ -32,7 +32,8 @@ import { getClientIp } from "@/lib/api-response"; import { annotate, getEvent } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; -import type { MeasurementType } from "@/generated/prisma/client"; +import type { MeasurementType, Prisma } from "@/generated/prisma/client"; +import { WHOOP_RECOVERY_FIELD_TAGS } from "./client"; /** * Maximum age of a webhook timestamp before it is treated as a replay. Five @@ -119,6 +120,31 @@ export function verifyWhoopSignature(args: { return timingSafeEqual(a, b); } +/** + * The `externalId` match for the rows a `.deleted` event removes. + * + * Rows are keyed `:`, but the v2 recovery webhook + * carries the id of the associated SLEEP, and a recovery's rows are keyed off + * that same sleep id (`:hrv_rmssd`, ...). The sleep's own rows share + * the prefix (`:sleep_eff`, `:seg:REM`, ...). A prefix + * match on a recovery deletion therefore took the whole night's sleep with it, + * and a sleep deletion took the recovery. Recovery names its exact ids; sleep + * keeps the prefix minus those ids; every other resource keeps the prefix. + */ +export function deletedResourceExternalIdFilter( + resource: string | undefined, + resourceId: string, +): Prisma.StringFilter { + const recoveryIds = WHOOP_RECOVERY_FIELD_TAGS.map( + (tag) => `${resourceId}:${tag}`, + ); + if (resource === "recovery") return { in: recoveryIds }; + if (resource === "sleep") { + return { startsWith: `${resourceId}:`, notIn: recoveryIds }; + } + return { startsWith: `${resourceId}:` }; +} + interface WhoopWebhookBody { user_id?: number | string; id?: number | string; @@ -160,11 +186,11 @@ export async function processWhoopNotification( const queue = WHOOP_RESOURCE_QUEUE[resource ?? ""]; if (verb === "deleted") { - // Soft-delete every matching row for this user + WHOOP resource id. - // `externalId` is `:` so a `startsWith` - // match catches every measurement derived from the deleted resource. + // Soft-delete every row derived from the deleted resource for this user. + // See `deletedResourceExternalIdFilter` for why the match is per resource. if (resourceId) { const now = new Date(); + const externalId = deletedResourceExternalIdFilter(resource, resourceId); // Collect the affected `(type, measuredAt)` pairs BEFORE the // soft-delete — the rollup tail below needs to know which // `(type, day)` buckets lose rows, and after the updateMany the @@ -173,7 +199,7 @@ export async function processWhoopNotification( where: { userId: connection.userId, source: "WHOOP", - externalId: { startsWith: `${resourceId}:` }, + externalId, deletedAt: null, }, select: { type: true, measuredAt: true }, @@ -182,7 +208,7 @@ export async function processWhoopNotification( where: { userId: connection.userId, source: "WHOOP", - externalId: { startsWith: `${resourceId}:` }, + externalId, deletedAt: null, }, data: { deletedAt: now, syncVersion: { increment: 1 } }, From c5d2053e5ad9292f149412ea4ad83058c950a1af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 12:00:20 +0200 Subject: [PATCH 2/9] fix(hrv): file Fitbit and Google Health RMSSD as HRV_RMSSD Fitbit's `dailyRmssd` and Google Health's daily HRV average are both RMSSD, but the mappers wrote them into HEART_RATE_VARIABILITY, the SDNN series. The `hrv` ladder ranks both above Apple Health, so an account with an Apple Watch and a Fitbit had a series that switched estimator from day to day. The mappers now write HRV_RMSSD, and the `hrvRmssd` ladder carries both sources where they sat on the SDNN one. Migration 0420 moves the rows already written (tombstones included, with a sync-version bump so the change reaches the phone), drops their SDNN rollup buckets and the RMSSD buckets of every account whose rows moved, so the boot-time rollup discovery folds those accounts again, and deletes the SDNN personal records those rows had set. One DO block, so the move and its cleanup land together. --- .../migration.sql | 70 +++++ src/lib/fitbit/__tests__/client.test.ts | 4 +- src/lib/fitbit/client.ts | 15 +- src/lib/fitbit/mapping.md | 16 +- src/lib/fitbit/sync-metrics.ts | 2 +- .../google-health/__tests__/mappers.test.ts | 4 +- src/lib/google-health/mappers.ts | 12 +- src/lib/google-health/mapping.md | 26 +- src/lib/google-health/sync-metrics.ts | 4 +- src/lib/validations/source-priority.ts | 13 +- .../hrv-rmssd-fitbit-google-migration.test.ts | 270 ++++++++++++++++++ 11 files changed, 393 insertions(+), 43 deletions(-) create mode 100644 prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql create mode 100644 tests/integration/hrv-rmssd-fitbit-google-migration.test.ts diff --git a/prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql b/prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql new file mode 100644 index 000000000..cc1ae0c14 --- /dev/null +++ b/prisma/migrations/0420_hrv_rmssd_from_fitbit_and_google_health/migration.sql @@ -0,0 +1,70 @@ +-- Fitbit and Google Health HRV is RMSSD; file it as such. +-- +-- Both providers report a nightly RMSSD (`dailyRmssd`; Google documents its +-- daily average as RMSSD too), but their mappers wrote it as +-- `HEART_RATE_VARIABILITY`, the SDNN series. The `hrv` ladder ranks Fitbit and +-- Google Health above Apple Health, so an account with both had a series that +-- switched estimator from day to day, and every baseline, trend and +-- correlation over it measured the switch. The mappers now write `HRV_RMSSD`; +-- this moves the rows they wrote before. +-- +-- Every `HEART_RATE_VARIABILITY` row under source FITBIT or GOOGLE_HEALTH came +-- from that one daily-HRV mapper per provider (their externalIds end in +-- `:hrv`), so source and type identify them. Tombstones move too, so the sync +-- feed and a restore see the same type. A row whose RMSSD twin already exists +-- under the `(user, type, source, externalId)` key stays where it is. +-- +-- Derived state that would otherwise keep the old filing: +-- - the FITBIT / GOOGLE_HEALTH `HEART_RATE_VARIABILITY` rollup buckets are +-- dropped (no rows remain behind them); +-- - every `HRV_RMSSD` rollup bucket of an account whose rows moved is +-- dropped, so the per-type coverage probe reads that series live and the +-- boot-time rollup discovery (`enqueueBootTimeRollupBackfill`, which flags +-- any (user, type, day) with no DAY bucket) folds the account again on the +-- next worker start. Dropping only the moved days would not do: a day that +-- another source already covers would keep its bucket and never be +-- flagged, and the moved rows would be missing from it; +-- - personal records filed under `HEART_RATE_VARIABILITY` from those sources +-- are deleted. They ranked an RMSSD value as an SDNN best; `HRV_RMSSD` keeps +-- no records, and the detector re-derives the SDNN ones from what remains. +-- +-- One `DO` block, so the move and its cleanup commit or roll back together +-- (`migrate deploy` commits each top-level statement on its own). Running it +-- again finds nothing to move and drops nothing a fold has not rebuilt. +-- +-- Reversibility: the moved rows carry source FITBIT / GOOGLE_HEALTH and type +-- `HRV_RMSSD`, which nothing else writes for those sources; setting the type +-- back and dropping their rollups reverses it. + +DO $migration$ +BEGIN + WITH moved AS ( + UPDATE "measurements" m + SET "type" = 'HRV_RMSSD', + "updated_at" = CURRENT_TIMESTAMP, + "sync_version" = m."sync_version" + 1 + WHERE m."type" = 'HEART_RATE_VARIABILITY' + AND m."source" IN ('FITBIT', 'GOOGLE_HEALTH') + AND NOT EXISTS ( + SELECT 1 + FROM "measurements" twin + WHERE twin."user_id" = m."user_id" + AND twin."type" = 'HRV_RMSSD' + AND twin."source" = m."source" + AND twin."external_id" = m."external_id" + ) + RETURNING m."user_id" + ) + DELETE FROM "measurement_rollups" r + WHERE r."type" = 'HRV_RMSSD' + AND r."user_id" IN (SELECT DISTINCT "user_id" FROM moved); + + DELETE FROM "measurement_rollups" + WHERE "type" = 'HEART_RATE_VARIABILITY' + AND "source" IN ('FITBIT', 'GOOGLE_HEALTH'); + + DELETE FROM "personal_records" + WHERE "metric_type" = 'HEART_RATE_VARIABILITY' + AND "source" IN ('FITBIT', 'GOOGLE_HEALTH'); +END +$migration$; diff --git a/src/lib/fitbit/__tests__/client.test.ts b/src/lib/fitbit/__tests__/client.test.ts index 392c818ab..30668375c 100644 --- a/src/lib/fitbit/__tests__/client.test.ts +++ b/src/lib/fitbit/__tests__/client.test.ts @@ -332,13 +332,13 @@ describe("metric mappers (classic shapes)", () => { expect(out[0]!.fieldTag).toBe("2026-05-10:spo2"); }); - it("maps HRV dailyRmssd into the canonical HEART_RATE_VARIABILITY slot", () => { + it("maps HRV dailyRmssd into HRV_RMSSD, apart from the SDNN series", () => { const out = mapHeartRateVariability({ hrv: [ { dateTime: "2026-05-10", value: { dailyRmssd: 42.7, deepRmssd: 50 } }, ], }); - expect(out[0]!.type).toBe("HEART_RATE_VARIABILITY"); + expect(out[0]!.type).toBe("HRV_RMSSD"); expect(out[0]).toMatchObject({ value: 42.7, unit: "ms" }); expect(out[0]!.fieldTag).toBe("2026-05-10:hrv"); }); diff --git a/src/lib/fitbit/client.ts b/src/lib/fitbit/client.ts index cd8cb3b85..5f936c61d 100644 --- a/src/lib/fitbit/client.ts +++ b/src/lib/fitbit/client.ts @@ -741,15 +741,16 @@ export function mapHeartRateVariability( body: unknown, tz?: string, ): FitbitMappedMeasurement[] { - // Fitbit's HRV summary is an RMSSD estimator (`value.dailyRmssd`). It lands in - // the canonical `HEART_RATE_VARIABILITY` slot that FITBIT occupies in the - // source-priority `hrv` ladder (alongside Apple / Oura), keeping cross-source - // comparison on one axis. (`HRV_RMSSD` is reserved for the WHOOP-native slot.) + // Fitbit's HRV summary is an RMSSD estimator (`value.dailyRmssd`), so it lands + // in `HRV_RMSSD` beside WHOOP / Oura / Polar. `HEART_RATE_VARIABILITY` is the + // SDNN series; the two estimators are not interchangeable, and a Fitbit night + // ranked into the SDNN ladder used to splice RMSSD values into an Apple + // Watch SDNN series day by day. const rows = readArray(body, "hrv") as FitbitDailyRow[]; return mapDailySummary( rows, { - type: "HEART_RATE_VARIABILITY", + type: "HRV_RMSSD", unit: "ms", fieldTag: "hrv", leaves: ["dailyRmssd"], @@ -987,9 +988,9 @@ export const FITBIT_FIELD_MAP: Record< note: "spo2 summary value.avg", }, heartRateVariability: { - type: "HEART_RATE_VARIABILITY", + type: "HRV_RMSSD", unit: "ms", - note: "hrv summary value.dailyRmssd (RMSSD estimator); canonical HRV slot", + note: "hrv summary value.dailyRmssd (RMSSD estimator), apart from SDNN", }, restingHeartRate: { type: "RESTING_HEART_RATE", diff --git a/src/lib/fitbit/mapping.md b/src/lib/fitbit/mapping.md index 6b554c9c1..96a8ea5d6 100644 --- a/src/lib/fitbit/mapping.md +++ b/src/lib/fitbit/mapping.md @@ -26,14 +26,14 @@ metric (kg / km); the US default is imperial. Scope: `weight`, `oxygen_saturation`, `heartrate`, `respiratory_rate`. Verified against the dev.fitbit.com reference (response shapes quoted there). -| Endpoint | MeasurementType | Unit | Source field | Grain | Note | -| ---------------------------------------------- | ------------------------ | ----------- | ------------------------------------------- | ----- | ----------------------------------------------------------- | -| `/1/user/-/body/log/weight/date/{s}/{e}.json` | `WEIGHT` | kg | `weight[].weight` | log | externalId anchors on `logId`. Picker ranks Withings above. | -| `/1/user/-/body/log/fat/date/{s}/{e}.json` | `BODY_FAT` | % | `fat[].fat` | log | externalId anchors on `logId`. | -| `/1/user/-/spo2/date/{s}/{e}.json` | `OXYGEN_SATURATION` | % | `[].value.avg` (bare array) | daily | day-keyed externalId. | -| `/1/user/-/hrv/date/{s}/{e}.json` | `HEART_RATE_VARIABILITY` | ms | `hrv[].value.dailyRmssd` | daily | RMSSD estimator → canonical HRV slot (not `HRV_RMSSD`). | -| `/1/user/-/activities/heart/date/{s}/{e}.json` | `RESTING_HEART_RATE` | bpm | `activities-heart[].value.restingHeartRate` | daily | rows with no resting HR for the day are skipped. | -| `/1/user/-/br/date/{s}/{e}.json` | `RESPIRATORY_RATE` | breaths/min | `br[].value.breathingRate` | daily | | +| Endpoint | MeasurementType | Unit | Source field | Grain | Note | +| ---------------------------------------------- | -------------------- | ----------- | ------------------------------------------- | ----- | ------------------------------------------------------------------- | +| `/1/user/-/body/log/weight/date/{s}/{e}.json` | `WEIGHT` | kg | `weight[].weight` | log | externalId anchors on `logId`. Picker ranks Withings above. | +| `/1/user/-/body/log/fat/date/{s}/{e}.json` | `BODY_FAT` | % | `fat[].fat` | log | externalId anchors on `logId`. | +| `/1/user/-/spo2/date/{s}/{e}.json` | `OXYGEN_SATURATION` | % | `[].value.avg` (bare array) | daily | day-keyed externalId. | +| `/1/user/-/hrv/date/{s}/{e}.json` | `HRV_RMSSD` | ms | `hrv[].value.dailyRmssd` | daily | RMSSD estimator, kept apart from the SDNN `HEART_RATE_VARIABILITY`. | +| `/1/user/-/activities/heart/date/{s}/{e}.json` | `RESTING_HEART_RATE` | bpm | `activities-heart[].value.restingHeartRate` | daily | rows with no resting HR for the day are skipped. | +| `/1/user/-/br/date/{s}/{e}.json` | `RESPIRATORY_RATE` | breaths/min | `br[].value.breathingRate` | daily | | Each daily-summary value passes a finite + strictly-positive guard (a 0/NaN is a garbage/empty reading and is dropped). Each endpoint caps its own date range diff --git a/src/lib/fitbit/sync-metrics.ts b/src/lib/fitbit/sync-metrics.ts index 066bd4a6d..872fc1c35 100644 --- a/src/lib/fitbit/sync-metrics.ts +++ b/src/lib/fitbit/sync-metrics.ts @@ -7,7 +7,7 @@ * - body/log/weight → WEIGHT (kg) * - body/log/fat → BODY_FAT (%) * - spo2 summary → OXYGEN_SATURATION (%) - * - hrv summary → HEART_RATE_VARIABILITY (ms, dailyRmssd) + * - hrv summary → HRV_RMSSD (ms, dailyRmssd) * - activities/heart → RESTING_HEART_RATE (bpm) * - br summary → RESPIRATORY_RATE (breaths/min) * diff --git a/src/lib/google-health/__tests__/mappers.test.ts b/src/lib/google-health/__tests__/mappers.test.ts index 16d119f23..36be47ff2 100644 --- a/src/lib/google-health/__tests__/mappers.test.ts +++ b/src/lib/google-health/__tests__/mappers.test.ts @@ -148,7 +148,7 @@ describe("mapOxygenSaturation — daily-oxygen-saturation summary", () => { }); describe("mapHeartRateVariability — daily-heart-rate-variability summary", () => { - it("maps averageHeartRateVariabilityMilliseconds to the HRV slot in ms", () => { + it("maps the RMSSD daily average to HRV_RMSSD in ms", () => { const rows = mapHeartRateVariability({ dailyHeartRateVariability: { averageHeartRateVariabilityMilliseconds: 42.5, @@ -157,7 +157,7 @@ describe("mapHeartRateVariability — daily-heart-rate-variability summary", () }); expect(rows).toHaveLength(1); expect(rows[0]).toMatchObject({ - type: "HEART_RATE_VARIABILITY", + type: "HRV_RMSSD", unit: "ms", value: 42.5, fieldTag: "2026-06-01:hrv", diff --git a/src/lib/google-health/mappers.ts b/src/lib/google-health/mappers.ts index 0e6bfe02a..384d7d6d9 100644 --- a/src/lib/google-health/mappers.ts +++ b/src/lib/google-health/mappers.ts @@ -511,16 +511,16 @@ export function mapHeartRateVariability( tz?: string, ): GoogleHealthMappedMeasurement[] { const dt = GOOGLE_HEALTH_DATA_TYPES.heartRateVariability; - // The daily field is an unlabelled "average HRV ms". Per the design decision - // it lands in the SDNN-lineage `HEART_RATE_VARIABILITY` slot - // (Apple-comparable), NOT WHOOP's `HRV_RMSSD` (reserved for the WHOOP-native - // estimator). The per-sample type carries explicit RMSSD + SDNN fields — - // re-confirm the estimator against live data and revisit if warranted. + // Google documents the daily average as RMSSD ("average heart rate + // variability calculated using the root mean square of successive + // differences"), the same estimator the classic Fitbit API reports as + // `dailyRmssd`. It lands in `HRV_RMSSD`, apart from the SDNN + // `HEART_RATE_VARIABILITY` series. return mapSimple( point, dt, { - type: "HEART_RATE_VARIABILITY", + type: "HRV_RMSSD", unit: "ms", fieldTag: "hrv", valuePaths: [`${dt.key}.averageHeartRateVariabilityMilliseconds`], diff --git a/src/lib/google-health/mapping.md b/src/lib/google-health/mapping.md index 48e46beb3..eb958bf6b 100644 --- a/src/lib/google-health/mapping.md +++ b/src/lib/google-health/mapping.md @@ -81,19 +81,19 @@ The exercise civil bound is derived from the watermark in the USER'S zone. Scope: `googlehealth.health_metrics_and_measurements.readonly`. -| Data type (path) | Payload value read | MeasurementType | Unit | fieldTag | Grain | Note | -| ------------------------------------- | --------------------------------------------------------------------- | ------------------------ | ----------- | ------------ | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `weight` | `weight.weightGrams` ÷ 1000 | `WEIGHT` | kg | `weight` | sample | Grams on the wire. Picker ranks a real Withings scale above Google Health. | -| `body-fat` | `bodyFat.percentage` | `BODY_FAT` | % | `body_fat` | sample | Union key is camelCase `bodyFat`. | -| `daily-oxygen-saturation` | `dailyOxygenSaturation.averagePercentage` | `OXYGEN_SATURATION` | % | `spo2` | daily | The bare `oxygen-saturation` type is per-SAMPLE and rejects a `.date` filter. | -| `daily-heart-rate-variability` | `dailyHeartRateVariability.averageHeartRateVariabilityMilliseconds` | `HEART_RATE_VARIABILITY` | ms | `hrv` | daily | **Decision:** SDNN slot (Apple-comparable), NOT `HRV_RMSSD`. The daily field is an unlabelled "average HRV ms" — re-confirm estimator live. | -| `daily-resting-heart-rate` | `dailyRestingHeartRate.beatsPerMinute` (int64 string) | `RESTING_HEART_RATE` | bpm | `rhr` | daily | | -| `daily-respiratory-rate` | `dailyRespiratoryRate.breathsPerMinute` (number) | `RESPIRATORY_RATE` | breaths/min | `resp_rate` | daily | `respiratory-rate` does NOT exist in the catalogue. Schema is `{date, breathsPerMinute}` — the earlier `dailyRespiratoryRateBpm` leaf never existed. | -| `heart-rate` | `heartRate.beatsPerMinute` (int64 string) | `PULSE` | bpm | `hr` | sample | Intraday spot HR; per-minute volume — consider the 14-day-max rollup if PULSE should stay coarse. | -| `blood-glucose` | `bloodGlucose.bloodGlucoseMilligramsPerDeciliter` (number) | `BLOOD_GLUCOSE` | mg/dL | `glucose` | sample | Already in the canonical mg/dL storage unit. Meal/timing/specimen enums not stored. | -| `core-body-temperature` | `coreBodyTemperature.temperatureCelsius` (number) | `BODY_TEMPERATURE` | celsius | `core_temp` | sample | Core slot — distinct from SKIN_TEMPERATURE / WRIST_TEMPERATURE. | -| `daily-sleep-temperature-derivations` | `dailySleepTemperatureDerivations.nightlyTemperatureCelsius` (number) | `WRIST_TEMPERATURE` | celsius | `wrist_temp` | daily | ABSOLUTE nightly skin temperature (mean over the night's samples) — the schema refutes the earlier "deviation-only" premise. Baseline/stddev derivations not stored. | -| `height` | `height.heightMillimeters` (int64 string) ÷ 10 | `User.heightCm` | cm | — | sample | Profile seed — written ONLY when null; never a Measurement. List order is DESCENDING → the seed picks max(sampleTime) explicitly. The earlier `heightMeters` leaf never existed. | +| Data type (path) | Payload value read | MeasurementType | Unit | fieldTag | Grain | Note | +| ------------------------------------- | --------------------------------------------------------------------- | -------------------- | ----------- | ------------ | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `weight` | `weight.weightGrams` ÷ 1000 | `WEIGHT` | kg | `weight` | sample | Grams on the wire. Picker ranks a real Withings scale above Google Health. | +| `body-fat` | `bodyFat.percentage` | `BODY_FAT` | % | `body_fat` | sample | Union key is camelCase `bodyFat`. | +| `daily-oxygen-saturation` | `dailyOxygenSaturation.averagePercentage` | `OXYGEN_SATURATION` | % | `spo2` | daily | The bare `oxygen-saturation` type is per-SAMPLE and rejects a `.date` filter. | +| `daily-heart-rate-variability` | `dailyHeartRateVariability.averageHeartRateVariabilityMilliseconds` | `HRV_RMSSD` | ms | `hrv` | daily | Google documents the daily average as RMSSD, so it lands in `HRV_RMSSD`, apart from the SDNN `HEART_RATE_VARIABILITY`. | +| `daily-resting-heart-rate` | `dailyRestingHeartRate.beatsPerMinute` (int64 string) | `RESTING_HEART_RATE` | bpm | `rhr` | daily | | +| `daily-respiratory-rate` | `dailyRespiratoryRate.breathsPerMinute` (number) | `RESPIRATORY_RATE` | breaths/min | `resp_rate` | daily | `respiratory-rate` does NOT exist in the catalogue. Schema is `{date, breathsPerMinute}` — the earlier `dailyRespiratoryRateBpm` leaf never existed. | +| `heart-rate` | `heartRate.beatsPerMinute` (int64 string) | `PULSE` | bpm | `hr` | sample | Intraday spot HR; per-minute volume — consider the 14-day-max rollup if PULSE should stay coarse. | +| `blood-glucose` | `bloodGlucose.bloodGlucoseMilligramsPerDeciliter` (number) | `BLOOD_GLUCOSE` | mg/dL | `glucose` | sample | Already in the canonical mg/dL storage unit. Meal/timing/specimen enums not stored. | +| `core-body-temperature` | `coreBodyTemperature.temperatureCelsius` (number) | `BODY_TEMPERATURE` | celsius | `core_temp` | sample | Core slot — distinct from SKIN_TEMPERATURE / WRIST_TEMPERATURE. | +| `daily-sleep-temperature-derivations` | `dailySleepTemperatureDerivations.nightlyTemperatureCelsius` (number) | `WRIST_TEMPERATURE` | celsius | `wrist_temp` | daily | ABSOLUTE nightly skin temperature (mean over the night's samples) — the schema refutes the earlier "deviation-only" premise. Baseline/stddev derivations not stored. | +| `height` | `height.heightMillimeters` (int64 string) ÷ 10 | `User.heightCm` | cm | — | sample | Profile seed — written ONLY when null; never a Measurement. List order is DESCENDING → the seed picks max(sampleTime) explicitly. The earlier `heightMeters` leaf never existed. | Daily-summary `date` fields are `{year,month,day}` objects, anchored at UTC-midday so a timezone shift can't roll the civil day. Every value passes a diff --git a/src/lib/google-health/sync-metrics.ts b/src/lib/google-health/sync-metrics.ts index 95ed4a564..8115ac280 100644 --- a/src/lib/google-health/sync-metrics.ts +++ b/src/lib/google-health/sync-metrics.ts @@ -8,7 +8,7 @@ * - weight → WEIGHT * - body-fat → BODY_FAT * - daily-oxygen-saturation → OXYGEN_SATURATION - * - daily-heart-rate-variability → HEART_RATE_VARIABILITY (SDNN slot; see mapper) + * - daily-heart-rate-variability → HRV_RMSSD (RMSSD; see mapper) * - daily-resting-heart-rate → RESTING_HEART_RATE * - daily-respiratory-rate → RESPIRATORY_RATE * - heart-rate (intraday) → PULSE @@ -117,7 +117,7 @@ const METRIC_RESOURCES: MetricResource[] = [ dataType: GOOGLE_HEALTH_DATA_TYPES.heartRateVariability, map: mapHeartRateVariability, verb: "fetchHrv", - type: "HEART_RATE_VARIABILITY", + type: "HRV_RMSSD", }, { dataType: GOOGLE_HEALTH_DATA_TYPES.restingHeartRate, diff --git a/src/lib/validations/source-priority.ts b/src/lib/validations/source-priority.ts index 476daaed5..6ad587dbd 100644 --- a/src/lib/validations/source-priority.ts +++ b/src/lib/validations/source-priority.ts @@ -404,8 +404,17 @@ export const DEFAULT_SOURCE_PRIORITY: Required = { // on now. Mirrors the recovery ladder's native-above-computed ordering. sleepDebt: ["WHOOP", "OURA", "POLAR", "COMPUTED"], // v1.42 (#1110) — the straps and rings that measure RMSSD overnight lead; - // the HealthKit relay of a third-party reading is the fallback. - hrvRmssd: ["WHOOP", "OURA", "POLAR", "APPLE_HEALTH"], + // the HealthKit relay of a third-party reading is the fallback. Fitbit and + // Google Health report RMSSD too and take the places they hold on the SDNN + // ladder they used to be filed under. + hrvRmssd: [ + "WHOOP", + "FITBIT", + "GOOGLE_HEALTH", + "OURA", + "POLAR", + "APPLE_HEALTH", + ], }; /** diff --git a/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts b/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts new file mode 100644 index 000000000..a2b7e4bfe --- /dev/null +++ b/tests/integration/hrv-rmssd-fitbit-google-migration.test.ts @@ -0,0 +1,270 @@ +/** + * Migration 0420 moves the RMSSD rows Fitbit and Google Health wrote into the + * SDNN series over to `HRV_RMSSD`, and clears the derived state that kept the + * old filing: their SDNN rollup buckets, the RMSSD buckets of the accounts + * whose rows moved (so the boot-time discovery folds them again), and the + * SDNN personal records those rows set. Apple Health SDNN and WHOOP RMSSD are + * left exactly as they were, and a rerun changes nothing. + */ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { Prisma } from "@/generated/prisma/client"; +import { + enqueueBootTimeRollupBackfill, + recomputeUserRollups, +} from "@/lib/rollups/measurement-rollups"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: vi.fn(() => null), +})); +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; + +const prisma = getPrismaClient(); +const USER = "hrv-rmssd-migration"; +const BYSTANDER = "hrv-rmssd-migration-apple-only"; + +const MIGRATION = readFileSync( + resolve( + process.cwd(), + "prisma", + "migrations", + "0420_hrv_rmssd_from_fitbit_and_google_health", + "migration.sql", + ), + "utf8", +); + +async function runMigration(): Promise { + // The file is one `DO` block; the comments around it are harmless. + await prisma.$executeRawUnsafe(MIGRATION); +} + +const DAY_MS = 86_400_000; +/** Ten days ago at 07:00 UTC, so every row sits inside the fold window. */ +const D0 = + Math.floor(Date.now() / DAY_MS) * DAY_MS - 10 * DAY_MS + 7 * 3_600_000; +const dayKey = (i: number) => + new Date(D0 + i * DAY_MS).toISOString().slice(0, 10); + +function rows(): Prisma.MeasurementCreateManyInput[] { + const out: Prisma.MeasurementCreateManyInput[] = []; + for (let i = 0; i < 4; i += 1) { + const at = new Date(D0 + i * DAY_MS); + out.push( + { + id: `fitbit-${i}`, + userId: USER, + type: "HEART_RATE_VARIABILITY", + source: "FITBIT", + unit: "ms", + value: 40 + i, + measuredAt: at, + externalId: `${dayKey(i)}:hrv`, + }, + { + id: `apple-${i}`, + userId: USER, + type: "HEART_RATE_VARIABILITY", + source: "APPLE_HEALTH", + unit: "ms", + value: 70 + i, + measuredAt: new Date(at.getTime() + 3_600_000), + }, + // WHOOP covers the same nights in the RMSSD series already: the moved + // Fitbit rows land on days that already have an RMSSD bucket. + { + id: `whoop-${i}`, + userId: USER, + type: "HRV_RMSSD", + source: "WHOOP", + unit: "ms", + value: 55 + i, + measuredAt: new Date(at.getTime() + 1_800_000), + externalId: `sleep-${i}:hrv_rmssd`, + }, + { + id: `bystander-${i}`, + userId: BYSTANDER, + type: "HEART_RATE_VARIABILITY", + source: "APPLE_HEALTH", + unit: "ms", + value: 80 + i, + measuredAt: at, + }, + ); + } + out.push( + { + // On a night WHOOP covers too: every live moved row lands on a day that + // already has an RMSSD bucket, so only the dropped buckets can get the + // account flagged for a fold. + id: "google-0", + userId: USER, + type: "HEART_RATE_VARIABILITY", + source: "GOOGLE_HEALTH", + unit: "ms", + value: 44, + measuredAt: new Date(D0 + DAY_MS + 7_200_000), + externalId: `${dayKey(1)}:hrv`, + }, + // A tombstone moves too, so the sync feed reports one type for the id. + { + id: "fitbit-deleted", + userId: USER, + type: "HEART_RATE_VARIABILITY", + source: "FITBIT", + unit: "ms", + value: 39, + measuredAt: new Date(D0 + 6 * DAY_MS), + externalId: `${dayKey(6)}:hrv`, + deletedAt: new Date(), + }, + ); + return out; +} + +type RollupRow = { type: string; source: string; granularity: string }; +const rollups = (userId: string) => + prisma.$queryRaw` + SELECT "type"::text AS type, "source"::text AS source, + "granularity"::text AS granularity + FROM measurement_rollups + WHERE "user_id" = ${userId} + ORDER BY 1, 2, 3 + `; +const pairs = (all: RollupRow[]) => + [...new Set(all.map((r) => `${r.type}/${r.source}`))].sort(); + +let bystanderBefore: RollupRow[] = []; + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.createMany({ + data: [ + { id: USER, username: USER, timezone: "UTC" }, + { id: BYSTANDER, username: BYSTANDER, timezone: "UTC" }, + ], + }); + await prisma.measurement.createMany({ data: rows() }); + await recomputeUserRollups(USER); + await recomputeUserRollups(BYSTANDER); + await prisma.personalRecord.createMany({ + data: [ + { + userId: USER, + metricType: "HEART_RATE_VARIABILITY", + direction: "MAX", + value: 43, + unit: "ms", + achievedAt: new Date(D0 + 3 * DAY_MS), + source: "FITBIT", + sourceMeasurementId: "fitbit-3", + }, + { + userId: USER, + metricType: "HEART_RATE_VARIABILITY", + direction: "MAX", + value: 73, + unit: "ms", + achievedAt: new Date(D0 + 3 * DAY_MS + 3_600_000), + source: "APPLE_HEALTH", + sourceMeasurementId: "apple-3", + }, + ], + }); + bystanderBefore = await rollups(BYSTANDER); + await runMigration(); +}, 120_000); + +describe("migration 0420", () => { + it("moves every Fitbit and Google Health SDNN row to HRV_RMSSD, tombstones included", async () => { + const moved = await prisma.measurement.findMany({ + where: { userId: USER, source: { in: ["FITBIT", "GOOGLE_HEALTH"] } }, + select: { id: true, type: true, syncVersion: true, value: true }, + orderBy: { id: "asc" }, + }); + expect(moved).toHaveLength(6); + for (const row of moved) { + expect(row.type, row.id).toBe("HRV_RMSSD"); + // A type change must reach the phone through the sync feed. + expect(row.syncVersion, row.id).toBe(2); + } + }); + + it("leaves Apple Health SDNN and WHOOP RMSSD rows alone", async () => { + const apple = await prisma.measurement.findMany({ + where: { source: "APPLE_HEALTH" }, + select: { type: true, syncVersion: true }, + }); + expect(apple).toHaveLength(8); + expect(apple.every((r) => r.type === "HEART_RATE_VARIABILITY")).toBe(true); + expect(apple.every((r) => r.syncVersion === 1)).toBe(true); + const whoop = await prisma.measurement.findMany({ + where: { source: "WHOOP" }, + select: { type: true, syncVersion: true }, + }); + expect(whoop.every((r) => r.type === "HRV_RMSSD")).toBe(true); + expect(whoop.every((r) => r.syncVersion === 1)).toBe(true); + }); + + it("drops the SDNN buckets of the moved sources and the account's RMSSD buckets", async () => { + expect(pairs(await rollups(USER))).toEqual([ + "HEART_RATE_VARIABILITY/APPLE_HEALTH", + ]); + expect(await rollups(BYSTANDER)).toEqual(bystanderBefore); + }); + + it("leaves the account where the boot-time discovery folds it again", async () => { + const send = vi.fn(async () => "job-id"); + vi.mocked(getGlobalBoss).mockReturnValue({ send } as never); + try { + const result = await enqueueBootTimeRollupBackfill(); + expect(result.error).toBeNull(); + const users = send.mock.calls.map( + (call) => (call as unknown as [string, { userId: string }])[1].userId, + ); + expect(users).toContain(USER); + expect(users).not.toContain(BYSTANDER); + } finally { + vi.mocked(getGlobalBoss).mockReturnValue(null as never); + } + }); + + it("refolds into an RMSSD series that carries both straps", async () => { + await recomputeUserRollups(USER); + expect(pairs(await rollups(USER))).toEqual([ + "HEART_RATE_VARIABILITY/APPLE_HEALTH", + "HRV_RMSSD/FITBIT", + "HRV_RMSSD/GOOGLE_HEALTH", + "HRV_RMSSD/WHOOP", + ]); + }); + + it("deletes the SDNN records the moved rows set and keeps the rest", async () => { + const records = await prisma.personalRecord.findMany({ + where: { userId: USER }, + select: { source: true, metricType: true }, + }); + expect(records).toEqual([ + { source: "APPLE_HEALTH", metricType: "HEART_RATE_VARIABILITY" }, + ]); + }); + + it("is idempotent", async () => { + const measurementsBefore = await prisma.measurement.findMany({ + orderBy: { id: "asc" }, + }); + const rollupsBefore = await rollups(USER); + await runMigration(); + expect( + await prisma.measurement.findMany({ orderBy: { id: "asc" } }), + ).toEqual(measurementsBefore); + expect(await rollups(USER)).toEqual(rollupsBefore); + }); +}); From a66c54a1f18def921e95a13c7ab429dfddab9964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 12:13:39 +0200 Subject: [PATCH 3/9] feat(insights): scan the nightly RMSSD series for correlations and illness HRV_RMSSD was read nowhere HRV is analysed: the correlation scan and the illness engine only knew the SDNN series, so a strap or ring wearer's HRV never took part in either. RMSSD is now its own outcome channel (owned by the recovery module, like the summary tile) and its own illness scan type with a drop read as adverse. It is not merged into the SDNN channel: the estimators differ in scale, and one series of both would correlate the change of device rather than the body. --- messages/de.json | 1 + messages/en.json | 1 + messages/es.json | 1 + messages/fr.json | 1 + messages/it.json | 1 + messages/ko.json | 1 + messages/nl.json | 1 + messages/pl.json | 1 + src/__tests__/i18n-english-leak-guard.test.ts | 1 + src/lib/illness/__tests__/correlation.test.ts | 7 +++++++ src/lib/illness/correlation.ts | 4 ++++ .../__tests__/discovery-matrix-modules.test.ts | 15 +++++++++++++++ src/lib/insights/correlation-discovery.ts | 6 ++++++ src/lib/modules/surface.ts | 2 ++ 14 files changed, 43 insertions(+) diff --git a/messages/de.json b/messages/de.json index 9a2c80392..90d17c3c7 100644 --- a/messages/de.json +++ b/messages/de.json @@ -10392,6 +10392,7 @@ "vital": { "RESTING_HEART_RATE": "Ruheherzfrequenz", "HEART_RATE_VARIABILITY": "Herzfrequenzvariabilität", + "HRV_RMSSD": "HRV (RMSSD)", "RECOVERY_SCORE": "Recovery", "OXYGEN_SATURATION": "Sauerstoffsättigung", "BODY_TEMPERATURE": "Körpertemperatur", diff --git a/messages/en.json b/messages/en.json index 8d6ab2cd4..eba85e21c 100644 --- a/messages/en.json +++ b/messages/en.json @@ -10392,6 +10392,7 @@ "vital": { "RESTING_HEART_RATE": "Resting heart rate", "HEART_RATE_VARIABILITY": "Heart-rate variability", + "HRV_RMSSD": "HRV (RMSSD)", "RECOVERY_SCORE": "Recovery", "OXYGEN_SATURATION": "Oxygen saturation", "BODY_TEMPERATURE": "Body temperature", diff --git a/messages/es.json b/messages/es.json index f86b42a52..f89ebd220 100644 --- a/messages/es.json +++ b/messages/es.json @@ -10392,6 +10392,7 @@ "vital": { "RESTING_HEART_RATE": "Frecuencia cardíaca en reposo", "HEART_RATE_VARIABILITY": "Variabilidad de la frecuencia cardíaca", + "HRV_RMSSD": "VFC (RMSSD)", "RECOVERY_SCORE": "Recovery", "OXYGEN_SATURATION": "Saturación de oxígeno", "BODY_TEMPERATURE": "Temperatura corporal", diff --git a/messages/fr.json b/messages/fr.json index 3b3a92f2f..eada370a5 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -10392,6 +10392,7 @@ "vital": { "RESTING_HEART_RATE": "Fréquence cardiaque au repos", "HEART_RATE_VARIABILITY": "Variabilité de la fréquence cardiaque", + "HRV_RMSSD": "VFC (RMSSD)", "RECOVERY_SCORE": "Recovery", "OXYGEN_SATURATION": "Saturation en oxygène", "BODY_TEMPERATURE": "Température corporelle", diff --git a/messages/it.json b/messages/it.json index 85e7d2cf8..42929075e 100644 --- a/messages/it.json +++ b/messages/it.json @@ -10392,6 +10392,7 @@ "vital": { "RESTING_HEART_RATE": "Frequenza cardiaca a riposo", "HEART_RATE_VARIABILITY": "Variabilità della frequenza cardiaca", + "HRV_RMSSD": "HRV (RMSSD)", "RECOVERY_SCORE": "Recovery", "OXYGEN_SATURATION": "Saturazione di ossigeno", "BODY_TEMPERATURE": "Temperatura corporea", diff --git a/messages/ko.json b/messages/ko.json index cc848901c..ffd1b4c6f 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -10392,6 +10392,7 @@ "vital": { "RESTING_HEART_RATE": "안정 시 심박수", "HEART_RATE_VARIABILITY": "심박변이도", + "HRV_RMSSD": "심박변이도 (RMSSD)", "RECOVERY_SCORE": "회복", "OXYGEN_SATURATION": "산소포화도", "BODY_TEMPERATURE": "체온", diff --git a/messages/nl.json b/messages/nl.json index 745094620..d42032ded 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -10392,6 +10392,7 @@ "vital": { "RESTING_HEART_RATE": "Rusthartslag", "HEART_RATE_VARIABILITY": "Hartslagvariabiliteit", + "HRV_RMSSD": "HRV (RMSSD)", "RECOVERY_SCORE": "Herstel", "OXYGEN_SATURATION": "Zuurstofverzadiging", "BODY_TEMPERATURE": "Lichaamstemperatuur", diff --git a/messages/pl.json b/messages/pl.json index d87da90db..6985f25f3 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -10392,6 +10392,7 @@ "vital": { "RESTING_HEART_RATE": "Spoczynkowe tętno", "HEART_RATE_VARIABILITY": "Zmienność rytmu serca", + "HRV_RMSSD": "HRV (RMSSD)", "RECOVERY_SCORE": "Recovery", "OXYGEN_SATURATION": "Saturacja tlenem", "BODY_TEMPERATURE": "Temperatura ciała", diff --git a/src/__tests__/i18n-english-leak-guard.test.ts b/src/__tests__/i18n-english-leak-guard.test.ts index 451ab270f..b091256da 100644 --- a/src/__tests__/i18n-english-leak-guard.test.ts +++ b/src/__tests__/i18n-english-leak-guard.test.ts @@ -98,6 +98,7 @@ const LEGIT_IDENTICAL = new Set([ "insights.derived.composite.READINESS.component.hrv", "labs.catalog.hs-crp", "measurements.typeHrvRmssd", + "illness.vital.HRV_RMSSD", "settings.sections.sources.metrics.hrvRmssd", "measurements.typeVo2Max", // "{hours} h {minutes} min": the SI symbols for hour and minute. diff --git a/src/lib/illness/__tests__/correlation.test.ts b/src/lib/illness/__tests__/correlation.test.ts index 4c7e5cbd9..754026c7a 100644 --- a/src/lib/illness/__tests__/correlation.test.ts +++ b/src/lib/illness/__tests__/correlation.test.ts @@ -17,6 +17,7 @@ import { computeIllnessCorrelation, dayDiff, FUNCTIONAL_IMPACT_RETURN_KEY, + ILLNESS_SCAN_TYPES, isAdverseDeviation, MIN_BASELINE_DAYS, MIN_EPISODE_COVERAGE_DAYS, @@ -803,6 +804,12 @@ describe("isAdverseDeviation", () => { expect(isAdverseDeviation("HEART_RATE_VARIABILITY", "above")).toBe(false); }); + it("reads a drop in the nightly RMSSD series as adverse, like SDNN", () => { + expect(ILLNESS_SCAN_TYPES).toContain("HRV_RMSSD"); + expect(isAdverseDeviation("HRV_RMSSD", "below")).toBe(true); + expect(isAdverseDeviation("HRV_RMSSD", "above")).toBe(false); + }); + it("treats neutral-direction vitals (WEIGHT) as never adverse", () => { expect(isAdverseDeviation("WEIGHT", "above")).toBe(false); expect(isAdverseDeviation("WEIGHT", "below")).toBe(false); diff --git a/src/lib/illness/correlation.ts b/src/lib/illness/correlation.ts index 95f47e60b..2633ff279 100644 --- a/src/lib/illness/correlation.ts +++ b/src/lib/illness/correlation.ts @@ -118,6 +118,7 @@ export const ADVERSE_DIRECTION: Partial< BLOOD_PRESSURE_SYS: "up", BLOOD_PRESSURE_DIA: "up", HEART_RATE_VARIABILITY: "down", + HRV_RMSSD: "down", RECOVERY_SCORE: "down", OXYGEN_SATURATION: "down", }; @@ -130,6 +131,9 @@ export const ADVERSE_DIRECTION: Partial< export const ILLNESS_SCAN_TYPES: MeasurementType[] = [ "RESTING_HEART_RATE", "HEART_RATE_VARIABILITY", + // RMSSD, scanned on its own baseline: a nightly strap or ring reading drops + // during illness the same way SDNN does. + "HRV_RMSSD", "RECOVERY_SCORE", "OXYGEN_SATURATION", "BODY_TEMPERATURE", diff --git a/src/lib/insights/__tests__/discovery-matrix-modules.test.ts b/src/lib/insights/__tests__/discovery-matrix-modules.test.ts index 92d0d96b4..8f53be251 100644 --- a/src/lib/insights/__tests__/discovery-matrix-modules.test.ts +++ b/src/lib/insights/__tests__/discovery-matrix-modules.test.ts @@ -127,6 +127,21 @@ describe("assembleDiscoveryMatrix — switched-off modules", () => { expect(keys).toContain("CUSTOM_METRIC:abc"); }); + it("scans the nightly RMSSD series as an outcome owned by recovery", async () => { + const on = await assembleDiscoveryMatrix("u1", OPTS); + expect( + on.series.find((s) => s.key === "HRV_RMSSD" && s.role === "outcome"), + ).toBeDefined(); + const off = await assembleDiscoveryMatrix("u1", { + ...OPTS, + modules: { recovery: false }, + }); + const keys = keysOf(off.series); + expect(keys).not.toContain("HRV_RMSSD"); + // SDNN is a plain vital and stays. + expect(keys).toContain("HEART_RATE_VARIABILITY"); + }); + it("does not read the weather at all with the environment module off", async () => { const read = vi.mocked(fetchEnvironmentSeries); read.mockClear(); diff --git a/src/lib/insights/correlation-discovery.ts b/src/lib/insights/correlation-discovery.ts index db16e66c0..c08f20f70 100644 --- a/src/lib/insights/correlation-discovery.ts +++ b/src/lib/insights/correlation-discovery.ts @@ -662,6 +662,12 @@ export const DISCOVERY_BEHAVIOURS = [ export const DISCOVERY_OUTCOMES = [ "SLEEP_DURATION", "HEART_RATE_VARIABILITY", + // The nightly RMSSD series (WHOOP, Oura, Polar, Fitbit, Google Health) as + // its own outcome. It is not merged into the SDNN channel: the estimators + // differ in scale, so one series of both would correlate the switch between + // devices rather than the body. Neither HRV channel is a behaviour, so the + // two never pair with each other. + "HRV_RMSSD", "RESTING_HEART_RATE", "WEIGHT", // v1.22 — blood pressure as an OUTCOME (it was behaviour-only). This makes diff --git a/src/lib/modules/surface.ts b/src/lib/modules/surface.ts index 21f670926..a11e848a2 100644 --- a/src/lib/modules/surface.ts +++ b/src/lib/modules/surface.ts @@ -212,6 +212,8 @@ const STATIC_SURFACE_MODULE = { "correlation:SLEEP_DURATION": "sleep", "correlation:BLOOD_GLUCOSE": "glucose", "correlation:RESTING_HEART_RATE": "recovery", + // The RMSSD series is the recovery module's (see `summary:HRV_RMSSD`). + "correlation:HRV_RMSSD": "recovery", "correlation:MEDICATION_COMPLIANCE": "medications", // Symptom burden is read from the illness journal. "correlation:SYMPTOM_SEVERITY": "illness", From 1fc0268b3d185b46babed2a5024d675a4438d235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 12:23:57 +0200 Subject: [PATCH 4/9] fix(polar): clear each leg on its own success, read a lone 403 as refused The workouts leg recorded its failures under its own leg but left the success to the vitals leg. A leg-scoped success clears only its own leg, so one workouts failure stayed on the row for good and kept the connection in error however healthy the exercises read became. Each leg now records its own success. AccessLink documents its 403 as missing consent for that data. A 403 on one collection while a sibling succeeded with the same token was classified reauth_required and put a working connection behind a reconnect prompt no reconnect clears. Like Oura's scope refusals, such a collection (or the exercises read, once the vitals leg proved the token) is now noted on the wide event and not counted as a failure. A 403 on every collection, or any 401, stays one. --- .../api/polar/sync/__tests__/route.test.ts | 17 +++- .../__tests__/polar-sync-legs.test.ts | 40 +++++++-- src/lib/jobs/reminder/polar-sync.ts | 9 +- src/lib/polar/__tests__/sync-workouts.test.ts | 60 ++++++++++++-- src/lib/polar/__tests__/sync.test.ts | 82 ++++++++++++++++--- src/lib/polar/response-classifier.ts | 11 +++ src/lib/polar/sync-workouts.ts | 41 +++++++--- src/lib/polar/sync.ts | 76 +++++++++++++---- 8 files changed, 277 insertions(+), 59 deletions(-) diff --git a/src/app/api/polar/sync/__tests__/route.test.ts b/src/app/api/polar/sync/__tests__/route.test.ts index a3f870aed..654fa82f6 100644 --- a/src/app/api/polar/sync/__tests__/route.test.ts +++ b/src/app/api/polar/sync/__tests__/route.test.ts @@ -67,13 +67,21 @@ beforeEach(() => { vi.clearAllMocks(); vi.mocked(checkRateLimit).mockResolvedValue({ allowed: true } as never); vi.mocked(requireAuth).mockResolvedValue({ user: { id: "u1" } } as never); - syncUserPolarMock.mockResolvedValue({ imported: 0, failed: false }); + syncUserPolarMock.mockResolvedValue({ + imported: 0, + failed: false, + tokenProven: true, + }); syncUserPolarWorkoutsMock.mockResolvedValue(0); }); describe("POST /api/polar/sync", () => { it("runs both Polar legs and totals what they imported", async () => { - syncUserPolarMock.mockResolvedValue({ imported: 5, failed: false }); + syncUserPolarMock.mockResolvedValue({ + imported: 5, + failed: false, + tokenProven: true, + }); syncUserPolarWorkoutsMock.mockResolvedValue(2); const response = await post(request()); @@ -84,7 +92,10 @@ describe("POST /api/polar/sync", () => { error: null, }); expect(syncUserPolarMock).toHaveBeenCalledWith("u1"); - expect(syncUserPolarWorkoutsMock).toHaveBeenCalledWith("u1"); + // The vitals leg tells the workouts leg the token just worked. + expect(syncUserPolarWorkoutsMock).toHaveBeenCalledWith("u1", { + tokenProven: true, + }); }); it("does not call a run whose vitals collections all failed a success", async () => { diff --git a/src/lib/jobs/reminder/__tests__/polar-sync-legs.test.ts b/src/lib/jobs/reminder/__tests__/polar-sync-legs.test.ts index 9ea8874ce..4752702bf 100644 --- a/src/lib/jobs/reminder/__tests__/polar-sync-legs.test.ts +++ b/src/lib/jobs/reminder/__tests__/polar-sync-legs.test.ts @@ -17,20 +17,30 @@ import { syncUserPolarLegs } from "../polar-sync"; beforeEach(() => { vi.clearAllMocks(); - syncUserPolar.mockResolvedValue({ imported: 0, failed: false }); + syncUserPolar.mockResolvedValue({ + imported: 0, + failed: false, + tokenProven: true, + }); syncUserPolarWorkouts.mockResolvedValue(0); }); describe("syncUserPolarLegs — composite independence on the shared polar ledger", () => { it("runs both legs and returns the summed count when both pass", async () => { - syncUserPolar.mockResolvedValue({ imported: 5, failed: false }); + syncUserPolar.mockResolvedValue({ + imported: 5, + failed: false, + tokenProven: true, + }); syncUserPolarWorkouts.mockResolvedValue(2); await expect(syncUserPolarLegs("u1")).resolves.toEqual({ imported: 7, failed: false, }); expect(syncUserPolar).toHaveBeenCalledWith("u1"); - expect(syncUserPolarWorkouts).toHaveBeenCalledWith("u1"); + expect(syncUserPolarWorkouts).toHaveBeenCalledWith("u1", { + tokenProven: true, + }); }); it("sums both counts without throwing when the vitals leg returns a partial-failure count", async () => { @@ -38,7 +48,11 @@ describe("syncUserPolarLegs — composite independence on the shared polar ledge // its healthy-collections import count rather than throwing. The wrapper // must sum it with the workout leg and complete cleanly — a partial vitals // failure never starves the user's success accounting at the cohort level. - syncUserPolar.mockResolvedValue({ imported: 4, failed: true }); + syncUserPolar.mockResolvedValue({ + imported: 4, + failed: true, + tokenProven: true, + }); syncUserPolarWorkouts.mockResolvedValue(2); // The partial is carried, not swallowed: the wrapper completes with the // summed count AND says a leg did not settle, so the card cannot paint a @@ -48,7 +62,9 @@ describe("syncUserPolarLegs — composite independence on the shared polar ledge failed: true, }); expect(syncUserPolar).toHaveBeenCalledWith("u1"); - expect(syncUserPolarWorkouts).toHaveBeenCalledWith("u1"); + expect(syncUserPolarWorkouts).toHaveBeenCalledWith("u1", { + tokenProven: true, + }); }); it("still attempts the workout leg when the vitals leg fails (vice-versa isolation)", async () => { @@ -57,11 +73,17 @@ describe("syncUserPolarLegs — composite independence on the shared polar ledge // The vitals error is rethrown (attributable), but only AFTER the workout // leg was attempted — a vitals failure never starves the workout leg. await expect(syncUserPolarLegs("u1")).rejects.toThrow("vitals down"); - expect(syncUserPolarWorkouts).toHaveBeenCalledWith("u1"); + expect(syncUserPolarWorkouts).toHaveBeenCalledWith("u1", { + tokenProven: false, + }); }); it("leaves the vitals leg's result intact when the workout leg fails", async () => { - syncUserPolar.mockResolvedValue({ imported: 4, failed: false }); + syncUserPolar.mockResolvedValue({ + imported: 4, + failed: false, + tokenProven: true, + }); syncUserPolarWorkouts.mockRejectedValue(new Error("workouts down")); // The vitals leg ran to completion (its own success already recorded on // the shared ledger); the workout error is surfaced without undoing it. @@ -73,6 +95,8 @@ describe("syncUserPolarLegs — composite independence on the shared polar ledge syncUserPolar.mockRejectedValue(new Error("vitals first")); syncUserPolarWorkouts.mockRejectedValue(new Error("workouts second")); await expect(syncUserPolarLegs("u1")).rejects.toThrow("vitals first"); - expect(syncUserPolarWorkouts).toHaveBeenCalledWith("u1"); + expect(syncUserPolarWorkouts).toHaveBeenCalledWith("u1", { + tokenProven: false, + }); }); }); diff --git a/src/lib/jobs/reminder/polar-sync.ts b/src/lib/jobs/reminder/polar-sync.ts index a89cb7a6c..b8c34c2f5 100644 --- a/src/lib/jobs/reminder/polar-sync.ts +++ b/src/lib/jobs/reminder/polar-sync.ts @@ -23,8 +23,9 @@ export type PolarSyncPayload = PollCohortPayload; /** * Run both Polar legs for one user with INDEPENDENT attribution on the shared * `polar` ledger. Sequentially awaited (not `Promise.all`) so a workouts error - * stays attributable. Each leg records its own failure; the vitals leg owns the - * single success write. A failure in either leg is isolated — both legs are + * stays attributable, and so the vitals leg can tell the workouts leg whether + * the token just worked (a 403 on exercises is then a per-type refusal). Each + * leg records its own failure and its own success. A failure in either leg is isolated — both legs are * always attempted, so a workouts-leg failure can never mask a vitals success * and vice versa — then the FIRST error is rethrown so the cohort handler warns * and skips this user's success accounting without losing the error. @@ -45,18 +46,20 @@ export async function syncUserPolarLegs( // it has to be carried explicitly — otherwise a run in which four of five // collections failed reads exactly like a clean one. let partial = false; + let tokenProven = false; try { const vitals = await syncUserPolar(userId); total += vitals.imported; partial = vitals.failed; + tokenProven = vitals.tokenProven; } catch (err) { firstError = err; hadError = true; } try { - total += await syncUserPolarWorkouts(userId); + total += await syncUserPolarWorkouts(userId, { tokenProven }); } catch (err) { if (!hadError) { firstError = err; diff --git a/src/lib/polar/__tests__/sync-workouts.test.ts b/src/lib/polar/__tests__/sync-workouts.test.ts index 12b6d3fd1..893a4a509 100644 --- a/src/lib/polar/__tests__/sync-workouts.test.ts +++ b/src/lib/polar/__tests__/sync-workouts.test.ts @@ -9,6 +9,7 @@ const { getPolarConnection, fetchExercises, recordSyncFailure, + recordSyncSuccess, } = vi.hoisted(() => ({ createManyAndReturn: vi.fn(), update: vi.fn(), @@ -16,6 +17,7 @@ const { getPolarConnection: vi.fn(), fetchExercises: vi.fn(), recordSyncFailure: vi.fn(async () => {}), + recordSyncSuccess: vi.fn(async () => {}), })); vi.mock("@/lib/db", () => ({ @@ -32,6 +34,7 @@ vi.mock("../credentials", () => ({ getPolarConnection })); vi.mock("@/lib/integrations/status", async (importOriginal) => ({ ...(await importOriginal()), recordSyncFailure, + recordSyncSuccess, })); vi.mock("../client", async (importOriginal) => { const actual = await importOriginal(); @@ -181,9 +184,9 @@ describe("syncUserPolarWorkouts", () => { ); }); - // Assumption #4: a 403 scope gap is a soft/recoverable ledger error, not a - // hard crash — recorded as reauth_required and rethrown for cohort isolation. - it("records a 403 scope error as a soft reauth_required failure", async () => { + // Without a read that proved the token, a 403 may be the account-wide + // consent gap: recorded as reauth_required and rethrown for cohort isolation. + it("records a 403 as a reauth_required failure when nothing proved the token", async () => { fetchExercises.mockRejectedValue( new PolarApiError({ verb: "fetchExercises", @@ -204,11 +207,54 @@ describe("syncUserPolarWorkouts", () => { ); }); - it("never records success on the shared ledger (vitals leg owns the single success write)", async () => { - const status = await import("@/lib/integrations/status"); - const spy = vi.spyOn(status, "recordSyncSuccess"); + it("reads a 403 as a refused data type once the vitals leg proved the token", async () => { + fetchExercises.mockRejectedValue( + new PolarApiError({ + verb: "fetchExercises", + classification: "reauth_required", + httpStatus: 403, + reason: "http_403", + }), + ); + await expect( + syncUserPolarWorkouts("u1", { tokenProven: true }), + ).resolves.toBe(0); + expect(recordSyncFailure).not.toHaveBeenCalled(); + expect(recordSyncSuccess).toHaveBeenCalledWith("u1", "polar", { + leg: "workouts", + }); + }); + + it("keeps a 401 a failure even after the vitals leg proved the token", async () => { + fetchExercises.mockRejectedValue( + new PolarApiError({ + verb: "fetchExercises", + classification: "reauth_required", + httpStatus: 401, + reason: "http_401", + }), + ); + await expect( + syncUserPolarWorkouts("u1", { tokenProven: true }), + ).rejects.toBeInstanceOf(PolarApiError); + expect(recordSyncFailure).toHaveBeenCalledTimes(1); + expect(recordSyncSuccess).not.toHaveBeenCalled(); + }); + + // A leg-scoped success clears only its own leg, so this leg has to write + // its own: left to the vitals leg, one workouts failure stayed on the row + // for good. + it("records its own leg's success after a clean read", async () => { fetchExercises.mockResolvedValue([]); await syncUserPolarWorkouts("u1"); - expect(spy).not.toHaveBeenCalled(); + expect(recordSyncSuccess).toHaveBeenCalledWith("u1", "polar", { + leg: "workouts", + }); + }); + + it("records no success when the read failed", async () => { + fetchExercises.mockRejectedValue(new Error("boom")); + await expect(syncUserPolarWorkouts("u1")).rejects.toThrow("boom"); + expect(recordSyncSuccess).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/polar/__tests__/sync.test.ts b/src/lib/polar/__tests__/sync.test.ts index 3a1351d1d..b4d36ef63 100644 --- a/src/lib/polar/__tests__/sync.test.ts +++ b/src/lib/polar/__tests__/sync.test.ts @@ -164,7 +164,11 @@ function createdRows() { describe("syncUserPolar", () => { it("no-ops cleanly for an unconnected user", async () => { getConnMock.mockResolvedValue(null); - expect(await syncUserPolar("u1")).toEqual({ imported: 0, failed: false }); + expect(await syncUserPolar("u1")).toEqual({ + imported: 0, + failed: false, + tokenProven: false, + }); expect(createManyAndReturnMock).not.toHaveBeenCalled(); expect(recordSuccessMock).not.toHaveBeenCalled(); }); @@ -370,13 +374,13 @@ describe("syncUserPolar", () => { it("isolates one dead collection — imports the healthy four and records one named failure", async () => { getConnMock.mockResolvedValue(CONN); - // SpO2 403s (no sensor / consent), the other four return one row each. + // SpO2 fails upstream, the other four return one row each. fetchSpo2Mock.mockRejectedValue( new PolarApiError({ verb: "fetchSpo2", - classification: "reauth_required", - httpStatus: 403, - reason: "http_403", + classification: "transient", + httpStatus: 503, + reason: "http_503", }), ); fetchRechargesMock.mockResolvedValue([ @@ -402,6 +406,7 @@ describe("syncUserPolar", () => { // The partial rides back with the count rather than being narrowed away. expect(result.failed).toBe(true); + expect(result.tokenProven).toBe(true); // The healthy four still wrote their rows — the source is not blanked. const written = createdRows().map((row) => row.type); expect(written).toContain("RECOVERY_SCORE"); @@ -410,19 +415,74 @@ describe("syncUserPolar", () => { expect(written).toContain("CARDIO_LOAD"); expect(written).not.toContain("OXYGEN_SATURATION"); expect(imported).toBeGreaterThan(0); - // Exactly one failure, classified off the 403, naming spo2. No success. + // Exactly one failure, naming spo2. No success. expect(recordFailureMock).toHaveBeenCalledTimes(1); expect(recordFailureMock).toHaveBeenCalledWith( expect.objectContaining({ integration: "polar", - kind: "reauth_required", - errorCode: "403", + kind: "transient", + errorCode: "503", message: expect.stringContaining("spo2"), }), ); expect(recordSuccessMock).not.toHaveBeenCalled(); }); + it("reads a 403 beside a healthy sibling as a refused data type, not a dead token", async () => { + getConnMock.mockResolvedValue(CONN); + // An account without the SpO2 sensor or the consent for it: the same token + // reads the other collections fine. + fetchSpo2Mock.mockRejectedValue( + new PolarApiError({ + verb: "fetchSpo2", + classification: "reauth_required", + httpStatus: 403, + reason: "http_403", + }), + ); + fetchCardioLoadsMock.mockResolvedValue([ + { date: "2026-06-10", cardio_load: 55 }, + ]); + + const result = await syncUserPolar("u1"); + + expect(result).toMatchObject({ failed: false, tokenProven: true }); + expect(createdRows().map((row) => row.type)).toContain("CARDIO_LOAD"); + expect(recordFailureMock).not.toHaveBeenCalled(); + expect(recordSuccessMock).toHaveBeenCalledWith("u1", "polar", { + leg: "vitals", + }); + }); + + it("keeps a 403 on every collection a reauth_required failure", async () => { + getConnMock.mockResolvedValue(CONN); + for (const mock of [ + fetchRechargesMock, + fetchSleepsMock, + fetchActivitiesMock, + fetchCardioLoadsMock, + fetchSpo2Mock, + ]) { + mock.mockRejectedValue( + new PolarApiError({ + verb: "fetch", + classification: "reauth_required", + httpStatus: 403, + reason: "http_403", + }), + ); + } + + const result = await syncUserPolar("u1"); + + expect(result).toMatchObject({ failed: true, tokenProven: false }); + expect(recordFailureMock).toHaveBeenCalledTimes(1); + expect(recordFailureMock).toHaveBeenCalledWith( + expect.objectContaining({ kind: "reauth_required", errorCode: "403" }), + ); + expect(recordSuccessMock).not.toHaveBeenCalled(); + }); + it("isolates a throwing mapper — one malformed record does not blank siblings", async () => { getConnMock.mockResolvedValue(CONN); // A malformed recharge record whose mapper throws on first field access @@ -534,9 +594,9 @@ describe("syncUserPolar", () => { mockByName[name]!.mockRejectedValue( new PolarApiError({ verb: name, - classification: "reauth_required", - httpStatus: 403, - reason: "http_403", + classification: "persistent", + httpStatus: 404, + reason: "http_404", }), ); diff --git a/src/lib/polar/response-classifier.ts b/src/lib/polar/response-classifier.ts index 44b2cdc22..45ca52b1f 100644 --- a/src/lib/polar/response-classifier.ts +++ b/src/lib/polar/response-classifier.ts @@ -60,6 +60,17 @@ export function isInvalidGrant(err: unknown): boolean { return isOAuthInvalidGrant(err, "Polar"); } +/** + * True for a collection read Polar refused with 403. AccessLink documents its + * 403 as "User has not accepted all mandatory consents": the token is good, + * the account does not share this one data type (or has nothing behind it). + * Only meaningful next to a read that succeeded with the same token — a 403 on + * every collection is the account-wide consent gap, and stays a failure. + */ +export function isPolarCollectionRefusal(err: unknown): boolean { + return err instanceof PolarApiError && err.httpStatus === 403; +} + /** * Read the classification from any caught error. A non-`PolarApiError` defaults * to `transient` so a plain `Error` retries rather than disabling the diff --git a/src/lib/polar/sync-workouts.ts b/src/lib/polar/sync-workouts.ts index 5b2df23bc..59e15cfec 100644 --- a/src/lib/polar/sync-workouts.ts +++ b/src/lib/polar/sync-workouts.ts @@ -21,12 +21,13 @@ * only write and it is idempotent — no cursor, no backfill queue (the API caps * history at 30 days, so first-connect backfill IS the first regular tick). * - * Ledger: workouts and vitals share the single `polar` integration ledger. This - * leg records only its OWN FAILURE (and rethrows so the cohort handler can - * attribute it); the single success write stays the vitals leg's job so a fully - * successful pass records success exactly once. A workouts-leg failure records a - * transient/reauth failure on the shared ledger but never touches the vitals - * success timestamp (`recordSyncFailure` leaves `lastSuccessAt` intact). + * Ledger: workouts and vitals share the single `polar` integration ledger, each + * under its own leg. This leg records its own failure (and rethrows so the + * cohort handler can attribute it) and its own success. The success used to be + * left to the vitals leg, but a leg-scoped success clears only its own leg, so + * a workouts failure stayed on the row for good once recorded: every later + * vitals success saw the workouts leg still failing and kept the connection in + * error, however healthy the exercises read had become. */ import { prisma } from "@/lib/db"; import { emitInsertedWorkoutArrival } from "@/lib/arrivals/workout-emit"; @@ -35,11 +36,16 @@ import { annotate, getEvent } from "@/lib/logging/context"; import { markSyncFailureRecorded, recordSyncFailure, + recordSyncSuccess, toFailureKind, } from "@/lib/integrations/status"; import { fetchExercises, mapExercise, type PolarWorkoutRow } from "./client"; import { getPolarConnection } from "./credentials"; -import { PolarApiError, classifyPolarError } from "./response-classifier"; +import { + PolarApiError, + classifyPolarError, + isPolarCollectionRefusal, +} from "./response-classifier"; const POLAR_LEG_WORKOUTS = "workouts"; @@ -70,13 +76,16 @@ async function recordPolarWorkoutFailure( * written. A user with no Polar connection is a clean no-op (returns 0, touches * no status row). * - * A 403 on the exercises read (a scope gap under `accesslink.read_all`) is - * classified through the shared response classifier as `reauth_required` — a - * SOFT/recoverable integration error surfaced on the status ledger, NOT a hard - * crash: it is recorded and rethrown so the composite cohort pass isolates this - * user and the vitals leg's own result is untouched. + * A 403 on the exercises read is a refusal when the vitals leg proved the token + * in the same run (`tokenProven`): the account does not share exercises, which + * no reconnect changes, so the leg reads nothing and stays healthy. Without + * that proof the 403 is classified `reauth_required` as before, recorded, and + * rethrown so the composite cohort pass isolates this user. */ -export async function syncUserPolarWorkouts(userId: string): Promise { +export async function syncUserPolarWorkouts( + userId: string, + opts: { tokenProven?: boolean } = {}, +): Promise { const conn = await getPolarConnection(userId); if (!conn) return 0; @@ -91,6 +100,11 @@ export async function syncUserPolarWorkouts(userId: string): Promise { if (row) rows.push(row); } } catch (err) { + if (opts.tokenProven && isPolarCollectionRefusal(err)) { + annotate({ meta: { polar_collections_refused: [POLAR_LEG_WORKOUTS] } }); + await recordSyncSuccess(userId, "polar", { leg: POLAR_LEG_WORKOUTS }); + return 0; + } await recordPolarWorkoutFailure(userId, err); throw markSyncFailureRecorded(err); } @@ -110,6 +124,7 @@ export async function syncUserPolarWorkouts(userId: string): Promise { invalidateUserMeasurements(userId); } + await recordSyncSuccess(userId, "polar", { leg: POLAR_LEG_WORKOUTS }); return imported; } diff --git a/src/lib/polar/sync.ts b/src/lib/polar/sync.ts index fede4d187..080108b79 100644 --- a/src/lib/polar/sync.ts +++ b/src/lib/polar/sync.ts @@ -13,12 +13,19 @@ * exactly like Nightscout's token-rejected path. * * Isolation: the five vitals collections are settled independently - * (`Promise.allSettled`), so a single dead collection — e.g. a 403 on SpO2 or - * cardio-load on an account without the sensor or the higher tier — imports the - * healthy four and records ONE partial failure rather than blanking every - * collection. Mirrors Oura's shipped per-collection isolation, minus the - * token-refresh leg Polar structurally lacks (a 401 is just a per-collection - * failure the classifier maps to `reauth_required`). + * (`Promise.allSettled`), so a single dead collection imports the healthy four + * and records ONE partial failure rather than blanking every collection. + * Mirrors Oura's shipped per-collection isolation, minus the token-refresh leg + * Polar structurally lacks (a 401 is just a per-collection failure the + * classifier maps to `reauth_required`). + * + * A 403 on one collection while a sibling succeeded is a refusal, not a + * failure (`isPolarCollectionRefusal`): the same token just worked, so the + * account simply does not share that data type (SpO2 without the sensor, + * cardio load without the tier). Counting it as `reauth_required` put a + * working connection behind a "reconnect" prompt that no reconnect clears. + * The refused names ride the wide event; the leg records success. Mirrors + * Oura's handling of a scope the grant does not cover. * * Window contract: every hourly tick re-reads the full server-fixed window for * each collection — 28 days for recharge / sleep / activity / cardio-load / @@ -39,7 +46,7 @@ */ import { prisma } from "@/lib/db"; import type { MeasurementType } from "@/generated/prisma/client"; -import { getEvent } from "@/lib/logging/context"; +import { annotate, getEvent } from "@/lib/logging/context"; import { recordSyncFailure, recordSyncSuccess, @@ -80,7 +87,11 @@ import { type SleepSegmentSweep, } from "@/lib/sleep/sweep-stale-segments"; import { getPolarConnection } from "./credentials"; -import { PolarApiError, classifyPolarError } from "./response-classifier"; +import { + PolarApiError, + classifyPolarError, + isPolarCollectionRefusal, +} from "./response-classifier"; import { resolveUserTimezone } from "@/lib/tz/resolver"; /** Map a Polar error onto the shared integration-ledger failure kind. */ @@ -164,6 +175,15 @@ export interface PolarCollectionFailure { err: unknown; } +/** The vitals leg's outcome, plus whether any collection proved the token. */ +export interface PolarVitalsResult extends SyncWriteResult { + /** + * True when at least one collection read succeeded with the token, so a 403 + * elsewhere in this run (the workouts leg included) is a per-type refusal. + */ + tokenProven: boolean; +} + /** * Sync one user's Polar data. Returns the rows written AND whether a * collection did not settle — the bare count used to narrow the partial away, @@ -177,9 +197,11 @@ export interface PolarCollectionFailure { * escapes now is a write-path/unexpected error out of `upsertPolarMeasurements` * — the poll-cohort boundary records that unmarked escape once. */ -export async function syncUserPolar(userId: string): Promise { +export async function syncUserPolar( + userId: string, +): Promise { const conn = await getPolarConnection(userId); - if (!conn) return { imported: 0, failed: false }; + if (!conn) return { imported: 0, failed: false, tokenProven: false }; // Date-keyed rows are anchored at local noon of their date in this zone. const tz = await resolveUserTimezone(userId); @@ -250,15 +272,23 @@ export async function syncUserPolar(userId: string): Promise { collections.map((spec) => spec.collect(conn.accessToken)), ); const readings: PolarMeasurementUpsert[] = []; - const failures: PolarCollectionFailure[] = []; + const settledFailures: PolarCollectionFailure[] = []; settled.forEach((res, i) => { const spec = collections[i]!; if (res.status === "fulfilled") { readings.push(...res.value); } else { - failures.push({ name: spec.name, err: res.reason }); + settledFailures.push({ name: spec.name, err: res.reason }); } }); + const tokenProven = settled.some((res) => res.status === "fulfilled"); + const { failures, refused } = splitPolarRefusals( + settledFailures, + tokenProven, + ); + if (refused.length > 0) { + annotate({ meta: { polar_collections_refused: refused } }); + } // Clear whatever an earlier scoring left under the re-fetched nights before // the fresh set upserts (mirrors Google Health's replace-by-window order). @@ -317,11 +347,29 @@ export async function syncUserPolar(userId: string): Promise { ? String(firstErr.httpStatus) : undefined, }); - return { imported, failed: true }; + return { imported, failed: true, tokenProven }; } await recordSyncSuccess(userId, "polar", { leg: POLAR_LEG_VITALS }); - return { imported, failed: false }; + return { imported, failed: false, tokenProven }; +} + +/** + * Split a run's collection failures into the ones that count and the per-type + * refusals (a 403 while another read proved the token). Without a proof every + * failure counts: a 403 on every collection is the account-wide consent gap. + */ +export function splitPolarRefusals( + failures: PolarCollectionFailure[], + tokenProven: boolean, +): { failures: PolarCollectionFailure[]; refused: string[] } { + if (!tokenProven) return { failures, refused: [] }; + return { + failures: failures.filter((f) => !isPolarCollectionRefusal(f.err)), + refused: failures + .filter((f) => isPolarCollectionRefusal(f.err)) + .map((f) => f.name), + }; } /** From ba1271275fe025526cf4ac0caa202830ed2c24c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 12:46:14 +0200 Subject: [PATCH 5/9] feat(fitbit): stop at the Web API turndown and point to Google Health Google turns the legacy Fitbit Web API off on 30 October 2026 (support ended on 30 September). From that day the hourly poll, the backfill, the sleep repair and the token refresh make no call, the connect, sync and test routes refuse with `provider_retired`, and the connection reads as not connected to the health read and the watchdog, so the shutdown pages nobody and asks nobody to reconnect. The connection row and everything it imported stay. The status envelope publishes `retirement` on the Fitbit entry (the day, whether it has come, the successor), and the Fitbit card shows when the connection stops, how to move to Google Health (same devices, one more consent, tokens cannot be carried over) and that the data stays; sync and connect grey out after the day. Both integrations can run side by side meanwhile: every type Fitbit writes is laddered with Fitbit just above Google Health, so a day both deliver counts once. The test setups pin the turndown off, so the Fitbit suites do not turn red on the day; the turndown's own tests flip it. --- docs/api/openapi.yaml | 47 +++++++- docs/integrations/fitbit.md | 26 ++-- docs/integrations/google-health.md | 9 +- messages/de.json | 12 +- messages/en.json | 12 +- messages/es.json | 12 +- messages/fr.json | 12 +- messages/it.json | 12 +- messages/ko.json | 12 +- messages/nl.json | 12 +- messages/pl.json | 12 +- .../fitbit/connect/__tests__/route.test.ts | 13 ++ src/app/api/fitbit/connect/route.ts | 10 ++ .../api/fitbit/sync/__tests__/route.test.ts | 68 +++++++++++ src/app/api/fitbit/sync/route.ts | 9 ++ .../fitbit/test/__tests__/route.test.ts | 17 +++ src/app/api/integrations/fitbit/test/route.ts | 9 ++ src/app/api/integrations/status/route.ts | 7 ++ .../__tests__/oauth-return-outcome.test.ts | 9 ++ .../fitbit-retirement-notice.test.tsx | 47 ++++++++ .../integrations/connections-panel.tsx | 37 ++++-- .../integrations/fitbit-retirement-notice.tsx | 61 ++++++++++ .../integrations/oauth-integration-card.tsx | 17 ++- .../settings/integrations/shared.tsx | 3 + src/lib/fitbit/__tests__/retirement.test.ts | 37 ++++++ .../fitbit/__tests__/sync-user-fitbit.test.ts | 32 +++++ src/lib/fitbit/retirement.ts | 51 ++++++++ src/lib/fitbit/sync-core.ts | 6 + src/lib/fitbit/sync.ts | 8 ++ .../integrations/connection-health-load.ts | 6 +- .../jobs/__tests__/fitbit-backfill.test.ts | 26 ++++ .../__tests__/fitbit-sleep-repair.test.ts | 27 +++++ .../__tests__/fitbit-sync-retired.test.ts | 47 ++++++++ src/lib/jobs/fitbit-backfill.ts | 14 ++- src/lib/jobs/fitbit-sleep-repair.ts | 13 +- src/lib/jobs/reminder/fitbit-sync.ts | 15 +++ src/lib/openapi/error-codes.ts | 1 + src/lib/openapi/routes/integrations.ts | 31 ++++- tests/integration/environment-setup.ts | 23 +++- tests/integration/fitbit-retirement.test.ts | 113 ++++++++++++++++++ vitest.setup.ts | 24 ++++ 41 files changed, 920 insertions(+), 39 deletions(-) create mode 100644 src/app/api/fitbit/sync/__tests__/route.test.ts create mode 100644 src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx create mode 100644 src/components/settings/integrations/fitbit-retirement-notice.tsx create mode 100644 src/lib/fitbit/__tests__/retirement.test.ts create mode 100644 src/lib/fitbit/retirement.ts create mode 100644 src/lib/jobs/__tests__/fitbit-sync-retired.test.ts create mode 100644 tests/integration/fitbit-retirement.test.ts diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index d38b30a9f..3b3ff54f0 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -20339,6 +20339,13 @@ paths: schema: $ref: "#/components/schemas/ErrorEnvelope" "401": *a1 + "410": + description: Google turned the legacy Fitbit Web API off on 2026-10-30; nothing is fetched. `meta.errorCode` = + `provider_retired`. The data already imported stays; Google Health reads the same devices. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" "413": description: Body exceeds 64 KiB. content: @@ -20953,6 +20960,13 @@ paths: schema: $ref: "#/components/schemas/FitbitIntegrationTestEnvelope" "401": *a1 + "410": + description: Google turned the legacy Fitbit Web API off on 2026-10-30; nothing is probed. `meta.errorCode` = + `provider_retired`. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorEnvelope" "422": *a49 "429": *a50 "502": *a51 @@ -36298,8 +36312,8 @@ components: health_score_config_conflict, insights_layout_conflict, invalid_base_updated_at, json_carries_no_intake_time, json_not_an_array, medication_layout_conflict, missing_required_columns, modules_conflict, mood_tag_layout_conflict, not_configured, notification_prefs_conflict, oidc_only, - private_origin_not_approved, private_origin_not_grantable, rate_limited_self, sync_in_progress, - too_many_rows, unreadable_json, upstream_invalid_json, vapid_not_configured" + private_origin_not_approved, private_origin_not_grantable, provider_retired, rate_limited_self, + sync_in_progress, too_many_rows, unreadable_json, upstream_invalid_json, vapid_not_configured" type: string capability: description: "On an AI refusal: the capability that is unavailable, one of the keys of `AiCapabilities` (`coach`, @@ -79322,13 +79336,36 @@ components: allowPrivateHost: description: Nightscout only. type: boolean + retirement: + description: Fitbit only. The end of the legacy Fitbit Web API. + type: object + properties: + endsOn: + type: string + pattern: ^\d{4}-\d{2}-\d{2}$ + description: The day Google turns the legacy Fitbit Web API off. + ended: + type: boolean + description: "The day has come: the connection no longer syncs, its imported data stays, and `health` is null." + successor: + type: string + enum: + - google-health + description: The integration that reads the same Fitbit and Pixel devices. Moving needs a fresh consent; the Fitbit + grant cannot be carried over. + required: + - endsOn + - ended + - successor + additionalProperties: false health: anyOf: - $ref: "#/components/schemas/ConnectionHealth" - type: "null" - description: The unified connection state, null when the provider is not connected. It supersedes `state`, `lastError`, - `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, `hasActivityScope`, - `planUnavailable` and `scopeReconnect` on this entry; those nine are deprecated and removed in v1.44. + description: The unified connection state, null when the provider is not connected (and for Fitbit once + `retirement.ended`). It supersedes `state`, `lastError`, `consecutiveFailuresByKind`, `syncHealth`, + `tokenExpired`, `needsReauth`, `hasActivityScope`, `planUnavailable` and `scopeReconnect` on this entry; + those nine are deprecated and removed in v1.44. required: - integration - state diff --git a/docs/integrations/fitbit.md b/docs/integrations/fitbit.md index 7e4310c39..4c5684c21 100644 --- a/docs/integrations/fitbit.md +++ b/docs/integrations/fitbit.md @@ -16,14 +16,24 @@ Settings. The internal integration key is `fitbit`. > consent screen with `unauthorized_client — Invalid client_id`, because Fitbit > and Google Cloud are separate client registries. -> **Heads-up — new registrations are closed.** Fitbit has **discontinued -> self-serve app registration**: the form at `dev.fitbit.com/apps/new` is -> disabled for new applications, and the classic Fitbit Web API sunsets in -> **September 2026**. If you already hold a registered Fitbit app, it keeps -> working until the sunset and the setup below still applies. If you are -> connecting Fitbit data for the first time, use the -> [Google Health integration](google-health.md) instead — Fitbit devices sync -> into Google's health stack, which HealthLog reads directly. +> **The classic Fitbit Web API is turned off on 30 October 2026.** Google +> ended support for it on 30 September 2026 and turns it off on +> **30 October 2026**, after which it "will no longer function or sync data" +> ([Google's migration guide](https://developers.google.com/health/migration)). +> New app registrations at `dev.fitbit.com/apps/new` are already closed. +> +> From that day HealthLog stops calling the Web API for every Fitbit +> connection: no hourly sync, no backfill, no token refresh, and no new +> connection. Nothing is deleted. The connection and every reading it imported +> stay, and the Fitbit card explains the move instead of asking for a +> reconnect. To keep Fitbit and Pixel data coming, connect the +> [Google Health integration](google-health.md). It reads the same devices, +> but Google asks for consent once more: Fitbit tokens cannot be carried over. +> +> Both integrations can run side by side while you move. Where both deliver the +> same day, the source-priority ladder keeps one of them per day (Fitbit ranks +> just above Google Health by default), so steps, sleep and the other daily +> values are not counted twice. After the turndown only Google Health delivers. ## 1. Use an existing Fitbit developer app diff --git a/docs/integrations/google-health.md b/docs/integrations/google-health.md index 554f1ed1f..069d09e83 100644 --- a/docs/integrations/google-health.md +++ b/docs/integrations/google-health.md @@ -2,9 +2,12 @@ HealthLog reads Fitbit, Pixel Watch and Fitbit Air data through the **Google Health API** (`health.googleapis.com/v4`), the successor to the classic Fitbit -Web API. Google retires the classic Fitbit Web API in **September 2026**, and -Fitbit access/refresh tokens do not carry over — every account re-consents -through Google OAuth. This integration is that path. +Web API. Google ended support for the classic Fitbit Web API on 30 September +2026 and turns it off on **30 October 2026** +([migration guide](https://developers.google.com/health/migration)); from that +day HealthLog's `fitbit` connection stops syncing and keeps what it imported. +Fitbit access/refresh tokens do not carry over, so every account consents once +more through Google OAuth. This integration is that path. The OAuth app is a **Google Cloud** OAuth client, not a Fitbit developer-console app. Credentials are brought **per user**: each user registers their own Google diff --git a/messages/de.json b/messages/de.json index 90d17c3c7..9fa1c2120 100644 --- a/messages/de.json +++ b/messages/de.json @@ -6619,7 +6619,8 @@ "nocode": "Fitbit hat keinen Autorisierungscode zurückgegeben. Versuche es erneut und erlaube den Zugriff im Fitbit-Dialog.", "nocreds": "Fitbit ist nicht konfiguriert. Hinterlege oben deine Fitbit-OAuth-Zugangsdaten und versuche es erneut.", "token": "Der Token-Austausch mit Fitbit ist fehlgeschlagen. Bitte starte die Verbindung erneut.", - "generic": "Unbekannter Fehler bei der Verbindung. Bitte starte die Verbindung erneut." + "generic": "Unbekannter Fehler bei der Verbindung. Bitte starte die Verbindung erneut.", + "retired": "Google hat die Fitbit-Web-API abgeschaltet, deshalb lässt sich keine neue Fitbit-Verbindung mehr herstellen. Verbinde stattdessen Google Health, das dieselben Geräte liest." }, "integrations": { "withings": { @@ -7439,6 +7440,15 @@ "homePlaceUnknown": "Diesen Ort kennt die Dawarich-Instanz nicht mehr.", "notConnected": "Dawarich ist nicht verbunden.", "generic": "Das hat nicht geklappt. Versuche es erneut." + }, + "fitbitRetirement": { + "ending": "Google schaltet die Fitbit-Web-API am {date} ab. Ab dann synchronisiert diese Verbindung nicht mehr.", + "ended": "Google hat die Fitbit-Web-API am {date} abgeschaltet. Diese Verbindung synchronisiert nicht mehr.", + "move": "Google Health liest dieselben Fitbit- und Pixel-Geräte. Verbinde es unten, damit deine Daten weiter ankommen. Google fragt dich noch einmal nach deiner Zustimmung, weil sich die Fitbit-Verbindung nicht übertragen lässt. Alles bereits Importierte bleibt erhalten.", + "toCard": "Zu Google Health", + "guide": "So verbindest du Google Health", + "badgeEnding": "Wird abgeschaltet", + "badgeEnded": "Abgeschaltet" } }, "admin": { diff --git a/messages/en.json b/messages/en.json index eba85e21c..b682014c7 100644 --- a/messages/en.json +++ b/messages/en.json @@ -6619,7 +6619,8 @@ "nocode": "Fitbit did not return an authorization code. Try again and allow access in the Fitbit dialog.", "nocreds": "Fitbit is not configured. Add your Fitbit OAuth credentials above and try again.", "token": "The token exchange with Fitbit failed. Please start the connection again.", - "generic": "Unknown error while connecting. Please start the connection again." + "generic": "Unknown error while connecting. Please start the connection again.", + "retired": "Google has turned the Fitbit Web API off, so no new Fitbit connection can be made. Connect Google Health instead, which reads the same devices." }, "integrations": { "withings": { @@ -7439,6 +7440,15 @@ "homePlaceUnknown": "The Dawarich instance no longer knows this place.", "notConnected": "Dawarich is not connected.", "generic": "That did not work. Try again." + }, + "fitbitRetirement": { + "ending": "Google turns the Fitbit Web API off on {date}. From then on this connection stops syncing.", + "ended": "Google turned the Fitbit Web API off on {date}. This connection no longer syncs.", + "move": "Google Health reads the same Fitbit and Pixel devices. Connect it below to keep your data coming. Google asks for your consent once more, because the Fitbit connection cannot be carried over. Everything already imported stays.", + "toCard": "Go to Google Health", + "guide": "How to connect Google Health", + "badgeEnding": "Shutting down", + "badgeEnded": "Retired" } }, "admin": { diff --git a/messages/es.json b/messages/es.json index f89ebd220..340e12dfa 100644 --- a/messages/es.json +++ b/messages/es.json @@ -6619,7 +6619,8 @@ "nocode": "Fitbit no devolvió un código de autorización. Inténtalo de nuevo y permite el acceso en el diálogo de Fitbit.", "nocreds": "Fitbit no está configurado. Añade tus credenciales OAuth de Fitbit arriba e inténtalo de nuevo.", "token": "El intercambio de token con Fitbit falló. Inicia la conexión de nuevo.", - "generic": "Error desconocido al conectar. Inicia la conexión de nuevo." + "generic": "Error desconocido al conectar. Inicia la conexión de nuevo.", + "retired": "Google desactivó la API web de Fitbit, así que ya no se puede crear una nueva conexión de Fitbit. Conecta Google Health, que lee los mismos dispositivos." }, "integrations": { "withings": { @@ -7439,6 +7440,15 @@ "homePlaceUnknown": "La instancia de Dawarich ya no conoce este lugar.", "notConnected": "Dawarich no está conectado.", "generic": "No ha funcionado. Inténtalo de nuevo." + }, + "fitbitRetirement": { + "ending": "Google desactivará la API web de Fitbit el {date}. A partir de entonces, esta conexión dejará de sincronizar.", + "ended": "Google desactivó la API web de Fitbit el {date}. Esta conexión ya no sincroniza.", + "move": "Google Health lee los mismos dispositivos Fitbit y Pixel. Conéctalo abajo para que tus datos sigan llegando. Google te pedirá tu consentimiento otra vez, porque la conexión de Fitbit no se puede transferir. Todo lo ya importado se conserva.", + "toCard": "Ir a Google Health", + "guide": "Cómo conectar Google Health", + "badgeEnding": "Se desactiva pronto", + "badgeEnded": "Desactivado" } }, "admin": { diff --git a/messages/fr.json b/messages/fr.json index eada370a5..daec5830d 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -6619,7 +6619,8 @@ "nocode": "Fitbit n'a pas renvoyé de code d'autorisation. Réessayez et autorisez l'accès dans la fenêtre Fitbit.", "nocreds": "Fitbit n'est pas configuré. Ajoutez vos identifiants OAuth Fitbit ci-dessus et réessayez.", "token": "L'échange de jeton avec Fitbit a échoué. Veuillez relancer la connexion.", - "generic": "Erreur inconnue lors de la connexion. Veuillez relancer la connexion." + "generic": "Erreur inconnue lors de la connexion. Veuillez relancer la connexion.", + "retired": "Google a désactivé l'API Web Fitbit : il n'est plus possible de créer une nouvelle connexion Fitbit. Connectez plutôt Google Health, qui lit les mêmes appareils." }, "integrations": { "withings": { @@ -7439,6 +7440,15 @@ "homePlaceUnknown": "L’instance Dawarich ne connaît plus ce lieu.", "notConnected": "Dawarich n’est pas connecté.", "generic": "Ça n’a pas marché. Réessaie." + }, + "fitbitRetirement": { + "ending": "Google désactive l'API Web Fitbit le {date}. Ensuite, cette connexion ne se synchronisera plus.", + "ended": "Google a désactivé l'API Web Fitbit le {date}. Cette connexion ne se synchronise plus.", + "move": "Google Health lit les mêmes appareils Fitbit et Pixel. Connectez-le ci-dessous pour continuer à recevoir vos données. Google vous demandera de nouveau votre consentement, car la connexion Fitbit ne peut pas être transférée. Tout ce qui a déjà été importé est conservé.", + "toCard": "Aller à Google Health", + "guide": "Comment connecter Google Health", + "badgeEnding": "Bientôt désactivé", + "badgeEnded": "Désactivé" } }, "admin": { diff --git a/messages/it.json b/messages/it.json index 42929075e..866de0e05 100644 --- a/messages/it.json +++ b/messages/it.json @@ -6619,7 +6619,8 @@ "nocode": "Fitbit non ha restituito un codice di autorizzazione. Riprova e concedi l'accesso nella finestra di Fitbit.", "nocreds": "Fitbit non è configurato. Aggiungi le tue credenziali OAuth di Fitbit qui sopra e riprova.", "token": "Lo scambio del token con Fitbit è fallito. Riavvia la connessione.", - "generic": "Errore sconosciuto durante la connessione. Riavvia la connessione." + "generic": "Errore sconosciuto durante la connessione. Riavvia la connessione.", + "retired": "Google ha disattivato l'API web di Fitbit, quindi non è più possibile creare una nuova connessione Fitbit. Collega invece Google Health, che legge gli stessi dispositivi." }, "integrations": { "withings": { @@ -7439,6 +7440,15 @@ "homePlaceUnknown": "L’istanza Dawarich non conosce più questo luogo.", "notConnected": "Dawarich non è collegato.", "generic": "Non ha funzionato. Riprova." + }, + "fitbitRetirement": { + "ending": "Google disattiva l'API web di Fitbit il {date}. Da quel momento questa connessione smette di sincronizzarsi.", + "ended": "Google ha disattivato l'API web di Fitbit il {date}. Questa connessione non si sincronizza più.", + "move": "Google Health legge gli stessi dispositivi Fitbit e Pixel. Collegalo qui sotto per continuare a ricevere i tuoi dati. Google ti chiederà di nuovo il consenso, perché la connessione Fitbit non può essere trasferita. Tutto ciò che è già stato importato resta.", + "toCard": "Vai a Google Health", + "guide": "Come collegare Google Health", + "badgeEnding": "In dismissione", + "badgeEnded": "Dismesso" } }, "admin": { diff --git a/messages/ko.json b/messages/ko.json index ffd1b4c6f..7722828c9 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -6619,7 +6619,8 @@ "nocode": "Fitbit이 인증 코드를 보내지 않았어요. 다시 시도하고 Fitbit 창에서 접근을 허용해 주세요.", "nocreds": "Fitbit이 설정되지 않았어요. 위에 Fitbit OAuth 자격 증명을 추가하고 다시 시도해 주세요.", "token": "Fitbit과의 토큰 교환에 실패했어요. 연결을 다시 시작해 주세요.", - "generic": "연결 중 알 수 없는 오류가 생겼어요. 연결을 다시 시작해 주세요." + "generic": "연결 중 알 수 없는 오류가 생겼어요. 연결을 다시 시작해 주세요.", + "retired": "Google이 Fitbit Web API를 종료해서 새 Fitbit 연결을 만들 수 없어요. 같은 기기의 데이터를 읽는 Google Health를 대신 연결해 주세요." }, "integrations": { "withings": { @@ -7439,6 +7440,15 @@ "homePlaceUnknown": "Dawarich 인스턴스에 이 장소가 더 이상 없습니다.", "notConnected": "Dawarich가 연결되어 있지 않습니다.", "generic": "실패했습니다. 다시 시도하세요." + }, + "fitbitRetirement": { + "ending": "Google이 {date}에 Fitbit Web API를 종료해요. 그 이후로 이 연결은 더 이상 동기화되지 않아요.", + "ended": "Google이 {date}에 Fitbit Web API를 종료했어요. 이 연결은 더 이상 동기화되지 않아요.", + "move": "Google Health는 같은 Fitbit 및 Pixel 기기의 데이터를 읽어요. 데이터를 계속 받으려면 아래에서 연결해 주세요. Fitbit 연결은 옮길 수 없어서 Google이 동의를 한 번 더 요청해요. 이미 가져온 데이터는 모두 그대로 남아요.", + "toCard": "Google Health로 이동", + "guide": "Google Health 연결 방법", + "badgeEnding": "종료 예정", + "badgeEnded": "종료됨" } }, "admin": { diff --git a/messages/nl.json b/messages/nl.json index d42032ded..0bb6c3f82 100644 --- a/messages/nl.json +++ b/messages/nl.json @@ -6619,7 +6619,8 @@ "nocode": "Fitbit heeft geen autorisatiecode teruggestuurd. Probeer het opnieuw en sta toegang toe in het Fitbit-venster.", "nocreds": "Fitbit is niet geconfigureerd. Voeg hierboven je Fitbit-OAuth-inloggegevens toe en probeer het opnieuw.", "token": "De tokenuitwisseling met Fitbit is mislukt. Start de koppeling opnieuw.", - "generic": "Onbekende fout bij het koppelen. Start de koppeling opnieuw." + "generic": "Onbekende fout bij het koppelen. Start de koppeling opnieuw.", + "retired": "Google heeft de Fitbit Web API uitgeschakeld, dus er kan geen nieuwe Fitbit-koppeling meer worden gemaakt. Koppel in plaats daarvan Google Health, dat dezelfde apparaten leest." }, "integrations": { "withings": { @@ -7439,6 +7440,15 @@ "homePlaceUnknown": "De Dawarich-instantie kent deze plaats niet meer.", "notConnected": "Dawarich is niet gekoppeld.", "generic": "Dat is niet gelukt. Probeer het opnieuw." + }, + "fitbitRetirement": { + "ending": "Google schakelt de Fitbit Web API uit op {date}. Vanaf dan synchroniseert deze koppeling niet meer.", + "ended": "Google heeft de Fitbit Web API uitgeschakeld op {date}. Deze koppeling synchroniseert niet meer.", + "move": "Google Health leest dezelfde Fitbit- en Pixel-apparaten. Koppel het hieronder om je gegevens te blijven ontvangen. Google vraagt nog een keer om je toestemming, omdat de Fitbit-koppeling niet kan worden overgezet. Alles wat al is geïmporteerd blijft bewaard.", + "toCard": "Naar Google Health", + "guide": "Zo koppel je Google Health", + "badgeEnding": "Wordt uitgeschakeld", + "badgeEnded": "Uitgeschakeld" } }, "admin": { diff --git a/messages/pl.json b/messages/pl.json index 6985f25f3..124c3a6b3 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -6619,7 +6619,8 @@ "nocode": "Fitbit nie zwrócił kodu autoryzacji. Spróbuj ponownie i zezwól na dostęp w oknie Fitbit.", "nocreds": "Fitbit nie jest skonfigurowany. Dodaj powyżej swoje dane OAuth Fitbit i spróbuj ponownie.", "token": "Wymiana tokena z Fitbit nie powiodła się. Rozpocznij połączenie ponownie.", - "generic": "Nieznany błąd podczas łączenia. Rozpocznij połączenie ponownie." + "generic": "Nieznany błąd podczas łączenia. Rozpocznij połączenie ponownie.", + "retired": "Google wyłączył Fitbit Web API, więc nie można już utworzyć nowego połączenia Fitbit. Połącz zamiast tego Google Health, które odczytuje dane z tych samych urządzeń." }, "integrations": { "withings": { @@ -7439,6 +7440,15 @@ "homePlaceUnknown": "Instancja Dawarich nie zna już tego miejsca.", "notConnected": "Dawarich nie jest połączony.", "generic": "Nie udało się. Spróbuj ponownie." + }, + "fitbitRetirement": { + "ending": "Google wyłączy Fitbit Web API dnia {date}. Od tego dnia to połączenie przestanie się synchronizować.", + "ended": "Google wyłączył Fitbit Web API dnia {date}. To połączenie już się nie synchronizuje.", + "move": "Google Health odczytuje dane z tych samych urządzeń Fitbit i Pixel. Połącz je poniżej, aby dane nadal napływały. Google poprosi o zgodę jeszcze raz, ponieważ połączenia Fitbit nie da się przenieść. Wszystko, co już zaimportowano, zostaje.", + "toCard": "Przejdź do Google Health", + "guide": "Jak połączyć Google Health", + "badgeEnding": "Wkrótce wyłączone", + "badgeEnded": "Wyłączone" } }, "admin": { diff --git a/src/app/api/fitbit/connect/__tests__/route.test.ts b/src/app/api/fitbit/connect/__tests__/route.test.ts index aa71ee8a4..003b50857 100644 --- a/src/app/api/fitbit/connect/__tests__/route.test.ts +++ b/src/app/api/fitbit/connect/__tests__/route.test.ts @@ -56,6 +56,7 @@ import { GET } from "../route"; import { prisma } from "@/lib/db"; import { checkRateLimit } from "@/lib/rate-limit"; import { getUserFitbitCredentials } from "@/lib/fitbit/credentials"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; function connectRequest(): NextRequest { return new NextRequest("http://localhost/api/fitbit/connect", { @@ -112,4 +113,16 @@ describe("GET /api/fitbit/connect", () => { expect(location).toContain("reason=nocreds"); expect(prisma.fitbitOAuthState.create).not.toHaveBeenCalled(); }); + it("redirects to reason=retired once Google has turned the Web API off", async () => { + vi.mocked(isFitbitRetired).mockReturnValue(true); + try { + const response = await GET(connectRequest()); + const location = response.headers.get("location") ?? ""; + expect(location).toContain("fitbit=error"); + expect(location).toContain("reason=retired"); + expect(prisma.fitbitOAuthState.create).not.toHaveBeenCalled(); + } finally { + vi.mocked(isFitbitRetired).mockReturnValue(false); + } + }); }); diff --git a/src/app/api/fitbit/connect/route.ts b/src/app/api/fitbit/connect/route.ts index e9380b63f..11823963b 100644 --- a/src/app/api/fitbit/connect/route.ts +++ b/src/app/api/fitbit/connect/route.ts @@ -12,6 +12,7 @@ import { import { NextRequest, NextResponse } from "next/server"; import { relativeRedirect } from "@/lib/http/relative-redirect"; import { shouldEmitSecureCookie } from "@/lib/auth/secure-cookie"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; /** * Redirect the user to the Fitbit OAuth consent page (classic @@ -57,6 +58,15 @@ export const GET = apiHandler(async (req: NextRequest) => { ); } + // A grant issued now would never sync anything: Google has turned the Web + // API off. The card explains the move to Google Health. + if (isFitbitRetired()) { + annotate({ action: { name: "fitbit.connect.retired" } }); + return relativeRedirect( + "/settings/integrations?fitbit=error&reason=retired", + ); + } + const creds = await getUserFitbitCredentials(user.id); if (!creds) { annotate({ action: { name: "fitbit.connect.no_credentials" } }); diff --git a/src/app/api/fitbit/sync/__tests__/route.test.ts b/src/app/api/fitbit/sync/__tests__/route.test.ts new file mode 100644 index 000000000..b96e44345 --- /dev/null +++ b/src/app/api/fitbit/sync/__tests__/route.test.ts @@ -0,0 +1,68 @@ +/** + * POST /api/fitbit/sync once Google has turned the Fitbit Web API off: a + * clear 410 naming the cause, and no sync attempt behind it. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; +import type * as ApiHandlerModule from "@/lib/api-handler"; + +const { syncUserFitbitMock } = vi.hoisted(() => ({ + syncUserFitbitMock: vi.fn(), +})); + +vi.mock("@/lib/api-handler", async () => { + const actual = + await vi.importActual("@/lib/api-handler"); + return { + ...actual, + requireAuth: vi.fn(async () => ({ user: { id: "u1" } })), + }; +}); +vi.mock("@/lib/logging/transports", () => ({ emitIfSampled: vi.fn() })); +vi.mock("@/lib/rate-limit", () => ({ + checkRateLimit: vi.fn(async () => ({ allowed: true })), +})); +vi.mock("@/lib/fitbit/sync", () => ({ syncUserFitbit: syncUserFitbitMock })); + +import { POST } from "../route"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; + +const post = POST as unknown as (r: NextRequest) => Promise; + +function request(): NextRequest { + return new NextRequest("http://localhost/api/fitbit/sync", { + method: "POST", + }); +} + +beforeEach(() => { + vi.clearAllMocks(); + syncUserFitbitMock.mockResolvedValue({ imported: 2, failed: false }); +}); + +describe("POST /api/fitbit/sync", () => { + it("syncs while the Web API is still up", async () => { + const response = await post(request()); + expect(response.status).toBe(200); + expect(syncUserFitbitMock).toHaveBeenCalledWith("u1", { + fullSync: false, + }); + }); + + it("answers 410 provider_retired after the turndown and syncs nothing", async () => { + vi.mocked(isFitbitRetired).mockReturnValue(true); + try { + const response = await post(request()); + expect(response.status).toBe(410); + const body = (await response.json()) as { + error: string; + meta?: { errorCode?: string }; + }; + expect(body.meta?.errorCode).toBe("provider_retired"); + expect(body.error).toMatch(/Google Health/); + expect(syncUserFitbitMock).not.toHaveBeenCalled(); + } finally { + vi.mocked(isFitbitRetired).mockReturnValue(false); + } + }); +}); diff --git a/src/app/api/fitbit/sync/route.ts b/src/app/api/fitbit/sync/route.ts index be981267e..0c70598d6 100644 --- a/src/app/api/fitbit/sync/route.ts +++ b/src/app/api/fitbit/sync/route.ts @@ -4,6 +4,7 @@ import { checkRateLimit } from "@/lib/rate-limit"; import { annotate } from "@/lib/logging/context"; import { readSyncTriggerBody } from "@/lib/integrations/sync-request"; import { syncUserFitbit } from "@/lib/fitbit/sync"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; import { resolveSyncOutcome } from "@/lib/outcome/written-outcome"; import { NextRequest } from "next/server"; @@ -31,6 +32,14 @@ export const POST = apiHandler(async (request: NextRequest) => { }); } + if (isFitbitRetired()) { + return apiError( + "Google has turned the Fitbit Web API off. Connect Google Health instead.", + 410, + { errorCode: "provider_retired" }, + ); + } + const body = await readSyncTriggerBody(request); if (body.error) return body.error; const { fullSync } = body; diff --git a/src/app/api/integrations/fitbit/test/__tests__/route.test.ts b/src/app/api/integrations/fitbit/test/__tests__/route.test.ts index 31481035d..58a55d5b1 100644 --- a/src/app/api/integrations/fitbit/test/__tests__/route.test.ts +++ b/src/app/api/integrations/fitbit/test/__tests__/route.test.ts @@ -36,6 +36,7 @@ import { POST } from "../route"; import { prisma } from "@/lib/db"; import { checkRateLimit } from "@/lib/rate-limit"; import { getValidToken } from "@/lib/fitbit/sync-core"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; interface ApiErrorEnvelope { data: null; @@ -109,6 +110,22 @@ describe("POST /api/integrations/fitbit/test", () => { expect(global.fetch).not.toHaveBeenCalled(); }); + it("answers 410 provider_retired once Google has turned the Web API off", async () => { + vi.mocked(isFitbitRetired).mockReturnValue(true); + try { + const response = await POST(emptyRequest() as never); + expect(response.status).toBe(410); + const body = (await response.json()) as ApiErrorEnvelope & { + meta?: { errorCode?: string }; + }; + expect(body.meta?.errorCode).toBe("provider_retired"); + expect(getValidToken).not.toHaveBeenCalled(); + expect(global.fetch).not.toHaveBeenCalled(); + } finally { + vi.mocked(isFitbitRetired).mockReturnValue(false); + } + }); + it("returns 422 when no Fitbit connection exists", async () => { vi.mocked(getValidToken).mockResolvedValueOnce(null); const response = await POST(emptyRequest() as never); diff --git a/src/app/api/integrations/fitbit/test/route.ts b/src/app/api/integrations/fitbit/test/route.ts index 509e617bc..c9be8f9b1 100644 --- a/src/app/api/integrations/fitbit/test/route.ts +++ b/src/app/api/integrations/fitbit/test/route.ts @@ -7,6 +7,7 @@ import { prisma } from "@/lib/db"; import { getValidToken } from "@/lib/fitbit/sync-core"; import { FITBIT_API_BASE } from "@/lib/fitbit/client"; import { safeFetch, SafeFetchError } from "@/lib/safe-fetch"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; export const dynamic = "force-dynamic"; @@ -55,6 +56,14 @@ export const POST = apiHandler(async (request: NextRequest) => { }); } + if (isFitbitRetired()) { + return apiError( + "Google has turned the Fitbit Web API off. Connect Google Health instead.", + 410, + { errorCode: "provider_retired" }, + ); + } + const tokenInfo = await getValidToken(user.id); if (!tokenInfo) { return apiError("Fitbit not connected", 422, { diff --git a/src/app/api/integrations/status/route.ts b/src/app/api/integrations/status/route.ts index 73396103d..04917d270 100644 --- a/src/app/api/integrations/status/route.ts +++ b/src/app/api/integrations/status/route.ts @@ -57,6 +57,10 @@ import { type GoogleHealthBackfillEndedEntry, } from "@/lib/google-health/backfill-progress"; import type { ConnectionHealth } from "@/lib/integrations/connection-health"; +import { + fitbitRetirement, + type FitbitRetirement, +} from "@/lib/fitbit/retirement"; export const dynamic = "force-dynamic"; @@ -378,6 +382,7 @@ export const GET = apiHandler(async () => { ? fitbitConn.tokenExpiresAt.getTime() <= now : null, backfillCompleted: fitbitConn ? !!fitbitConn.backfillCompletedAt : null, + retirement: fitbitRetirement(), health: health["fitbit"], ...resolve("fitbit", fitbitStatus, { connected: fitbitConnected, @@ -558,6 +563,8 @@ interface FitbitExtras { tokenExpiresAt: string | null; tokenExpired: boolean | null; backfillCompleted: boolean | null; + /** The end of the legacy Fitbit Web API (`@/lib/fitbit/retirement`). */ + retirement: FitbitRetirement; } // v1.27.0 — Google Health mirrors the Fitbit shape plus `needsReauth`: Google diff --git a/src/components/settings/__tests__/oauth-return-outcome.test.ts b/src/components/settings/__tests__/oauth-return-outcome.test.ts index e768a03dd..caa988bdb 100644 --- a/src/components/settings/__tests__/oauth-return-outcome.test.ts +++ b/src/components/settings/__tests__/oauth-return-outcome.test.ts @@ -71,6 +71,15 @@ describe("oauthReasonKey", () => { ); }); + it("names the Fitbit turndown refusal with its own copy, and only for Fitbit", () => { + expect(oauthReasonKey("fitbit", "retired")).toBe( + "settings.fitbitOauthError.retired", + ); + expect(oauthReasonKey("whoop", "retired")).toBe( + "settings.whoopOauthError.generic", + ); + }); + it("falls back to generic for an unknown tag", () => { expect(oauthReasonKey("whoop", "totally_unknown")).toBe( "settings.whoopOauthError.generic", diff --git a/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx b/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx new file mode 100644 index 000000000..21fd77d23 --- /dev/null +++ b/src/components/settings/integrations/__tests__/fitbit-retirement-notice.test.tsx @@ -0,0 +1,47 @@ +/** + * The Fitbit card's notice about the end of the legacy Web API: before the + * day it says when the connection stops, from the day on that it has, and + * both times it points at Google Health and says the data stays. + */ +import { describe, expect, it } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; + +import { I18nProvider } from "@/lib/i18n/context"; +import { FitbitRetirementNotice } from "../fitbit-retirement-notice"; + +function render(ended: boolean | null) { + return renderToStaticMarkup( + + + , + ); +} + +describe("", () => { + it("names the day the connection stops, and the way on", () => { + const html = render(false); + expect(html).toContain('data-ended="false"'); + expect(html).toContain("Google turns the Fitbit Web API off on"); + expect(html).toContain("Google Health reads the same Fitbit and Pixel"); + expect(html).toContain("Everything already imported stays."); + expect(html).toContain('href="#google-health"'); + expect(html).toContain("/google-health"); + }); + + it("says the connection has stopped once the day has come", () => { + const html = render(true); + expect(html).toContain('data-ended="true"'); + expect(html).toContain("Google turned the Fitbit Web API off on"); + expect(html).not.toContain("Google turns the Fitbit Web API off on"); + }); + + it("renders nothing without a retirement on the status entry", () => { + expect(render(null)).toBe(""); + }); +}); diff --git a/src/components/settings/integrations/connections-panel.tsx b/src/components/settings/integrations/connections-panel.tsx index bb4fa54cd..6382cdd54 100644 --- a/src/components/settings/integrations/connections-panel.tsx +++ b/src/components/settings/integrations/connections-panel.tsx @@ -19,6 +19,7 @@ import { showErrorToast } from "@/lib/errors/error-toast"; import { AppleHealthCard } from "@/components/settings/integrations/apple-health-card"; import { DawarichCard } from "@/components/settings/integrations/dawarich-card"; +import { FitbitRetirementNotice } from "@/components/settings/integrations/fitbit-retirement-notice"; import { GarminInfoNote } from "@/components/settings/integrations/garmin-info-note"; import { GoogleHealthCard } from "@/components/settings/integrations/google-health-card"; import { IntegrationFallbackRow } from "@/components/settings/integrations/integration-fallback-row"; @@ -136,6 +137,9 @@ const PROVIDER_ONLY_REASONS: Partial< Record> > = { oura: new Set(["issuer"]), + // Fitbit's `retired`: Google has turned the Web API off, so the connect + // route refuses a new grant. + fitbit: new Set(["retired"]), }; type OAuthOutcome = @@ -216,6 +220,8 @@ export function ConnectionsPanel({ const queryClient = useQueryClient(); const { data: integrationStatus } = useIntegrationStatuses(isAuthenticated); + const fitbitStatus = pickStatus(integrationStatus, "fitbit"); + const fitbitRetired = fitbitStatus?.retirement?.ended === true; // OAuth callback handler — reads `?withings=connected|error&reason=…` // from the URL (lazy initialiser, same shape as the Codex handler in @@ -332,20 +338,35 @@ export function ConnectionsPanel({ />
+ {t("settings.fitbitTag")} - - {t("settings.fitbitExperimentalBadge")} - + {fitbitStatus?.retirement ? ( + + {fitbitRetired + ? t("settings.fitbitRetirement.badgeEnded") + : t("settings.fitbitRetirement.badgeEnding")} + + ) : ( + + {t("settings.fitbitExperimentalBadge")} + + )} } /> diff --git a/src/components/settings/integrations/fitbit-retirement-notice.tsx b/src/components/settings/integrations/fitbit-retirement-notice.tsx new file mode 100644 index 000000000..de56f1ca1 --- /dev/null +++ b/src/components/settings/integrations/fitbit-retirement-notice.tsx @@ -0,0 +1,61 @@ +"use client"; + +import { ArrowDown, ExternalLink } from "lucide-react"; + +import { InlineAction } from "@/components/ui/inline-action"; +import { useFormatters, useTranslations } from "@/lib/i18n/context"; +import type { FitbitRetirement } from "@/lib/fitbit/retirement"; + +import { integrationDocsHref } from "./setup-guide-link"; + +/** + * The end of the legacy Fitbit Web API, on the Fitbit card. Before the day it + * says when the connection stops; from the day on, that it has stopped. Both + * name the way on: Google Health reads the same devices, needs one more + * consent, and leaves everything already imported in place. The server owns + * the date and whether it has passed (`retirement` on the status envelope). + */ +export function FitbitRetirementNotice({ + retirement, +}: { + retirement: FitbitRetirement | undefined; +}) { + const { t } = useTranslations(); + const fmt = useFormatters(); + if (!retirement) return null; + const date = fmt.dateShortSmartCalendar(retirement.endsOn); + return ( +
+

+ {retirement.ended + ? t("settings.fitbitRetirement.ended", { date }) + : t("settings.fitbitRetirement.ending", { date })} +

+

{t("settings.fitbitRetirement.move")}

+ +
+ ); +} diff --git a/src/components/settings/integrations/oauth-integration-card.tsx b/src/components/settings/integrations/oauth-integration-card.tsx index 9af68d69e..d3f6eb1af 100644 --- a/src/components/settings/integrations/oauth-integration-card.tsx +++ b/src/components/settings/integrations/oauth-integration-card.tsx @@ -220,6 +220,10 @@ export function OAuthIntegrationCard(props: { callbackUrl: IntegrationCallbackUrls[OAuthIntegrationKey]; icon?: LucideIcon; titleAccessory?: ReactNode; + /** The provider cannot sync any more (Fitbit after its API turndown). */ + syncDisabled?: boolean; + /** …and cannot be connected either. */ + connectDisabled?: boolean; }) { const { config } = props; const sync = useStandardSync({ @@ -229,7 +233,13 @@ export function OAuthIntegrationCard(props: { fullSyncResult: config.fullSyncResult, statusQueryKey: config.statusQueryKey(), }); - return ; + return ( + + ); } /** @@ -248,6 +258,7 @@ export function OAuthIntegrationCardView({ fullSyncDisabled, syncResults, details, + connectDisabled, }: { config: OAuthIntegrationConfig; viewModel: IntegrationStatusViewModel | undefined; @@ -263,6 +274,8 @@ export function OAuthIntegrationCardView({ syncResults?: ReactNode; /** Provider extras for the details section. */ details?: ReactNode; + /** Greys out "Connect" whatever the credentials say. */ + connectDisabled?: boolean; }) { const { t } = useTranslations(); const queryClient = useQueryClient(); @@ -374,7 +387,7 @@ export function OAuthIntegrationCardView({ type="button" size="sm" className="min-h-11 w-full sm:min-h-9 sm:w-auto" - disabled={!configured} + disabled={!configured || connectDisabled} onClick={connect} data-slot="integration-primary-action" data-action="connect" diff --git a/src/components/settings/integrations/shared.tsx b/src/components/settings/integrations/shared.tsx index c5c558ef3..f9afa9193 100644 --- a/src/components/settings/integrations/shared.tsx +++ b/src/components/settings/integrations/shared.tsx @@ -14,6 +14,7 @@ import type { GoogleHealthBackfillEndedEntry } from "@/lib/google-health/backfil import type { ConnectionHealth } from "@/lib/integrations/connection-health"; import type { MetricFreshnessEntry } from "@/lib/integrations/sync-verdict"; import { queryKeys } from "@/lib/query-keys"; +import type { FitbitRetirement } from "@/lib/fitbit/retirement"; export type IntegrationKey = | "withings" @@ -71,6 +72,8 @@ export interface IntegrationStatusViewModel { hasToken?: boolean; /** Per-metric last-seen timestamps with the server-computed stale flag. */ metricFreshness?: MetricFreshnessEntry[]; + /** Fitbit: the end of the legacy Web API, resolved on the server. */ + retirement?: FitbitRetirement; } export interface IntegrationStatusEnvelope { diff --git a/src/lib/fitbit/__tests__/retirement.test.ts b/src/lib/fitbit/__tests__/retirement.test.ts new file mode 100644 index 000000000..7a95cb230 --- /dev/null +++ b/src/lib/fitbit/__tests__/retirement.test.ts @@ -0,0 +1,37 @@ +/** + * The turndown date of the legacy Fitbit Web API, read off the real module + * (the unit setup pins it off for every other suite). + */ +import { describe, expect, it, vi } from "vitest"; + +const actual = await vi.importActual( + "@/lib/fitbit/retirement", +); + +describe("Fitbit Web API retirement", () => { + it("names the day Google published", () => { + // developers.google.com/health/migration: "On October 30, 2026, the + // Fitbit Web API will be turned off". + expect(actual.FITBIT_WEB_API_TURNDOWN_DATE).toBe("2026-10-30"); + }); + + it("turns at midnight UTC at the start of that day", () => { + expect(actual.isFitbitRetired(new Date("2026-10-29T23:59:59.999Z"))).toBe( + false, + ); + expect(actual.isFitbitRetired(new Date("2026-10-30T00:00:00.000Z"))).toBe( + true, + ); + }); + + it("publishes the day, whether it has come, and the successor", () => { + expect(actual.fitbitRetirement(new Date("2026-10-21T12:00:00Z"))).toEqual({ + endsOn: "2026-10-30", + ended: false, + successor: "google-health", + }); + expect( + actual.fitbitRetirement(new Date("2026-11-02T12:00:00Z")).ended, + ).toBe(true); + }); +}); diff --git a/src/lib/fitbit/__tests__/sync-user-fitbit.test.ts b/src/lib/fitbit/__tests__/sync-user-fitbit.test.ts index 6fd5448f2..2be8cada5 100644 --- a/src/lib/fitbit/__tests__/sync-user-fitbit.test.ts +++ b/src/lib/fitbit/__tests__/sync-user-fitbit.test.ts @@ -91,10 +91,12 @@ vi.mock("../sync-workout", () => ({ })); import { + getValidToken, handleCollectionFetchError, noteHardFailure, upsertFitbitMeasurements, } from "../sync-core"; +import { isFitbitRetired } from "../retirement"; import { syncUserFitbit } from "../sync"; import { FitbitApiError } from "../response-classifier"; import { prisma } from "@/lib/db"; @@ -453,3 +455,33 @@ describe("syncUserFitbit — backfill collapses rollup recompute to one pass", ( expect(update).not.toHaveBeenCalled(); }); }); + +describe("after Google turned the Fitbit Web API off", () => { + it("syncs nothing and writes neither a success nor a failure", async () => { + vi.mocked(isFitbitRetired).mockReturnValue(true); + try { + await expect(syncUserFitbit("u1")).resolves.toEqual({ + imported: 0, + failed: false, + }); + expect(syncUserMetrics).not.toHaveBeenCalled(); + expect(syncUserSleep).not.toHaveBeenCalled(); + expect(recordSyncSuccess).not.toHaveBeenCalled(); + expect(recordSyncFailure).not.toHaveBeenCalled(); + expect(update).not.toHaveBeenCalled(); + } finally { + vi.mocked(isFitbitRetired).mockReturnValue(false); + } + }); + + it("hands out no token and attempts no refresh", async () => { + vi.mocked(isFitbitRetired).mockReturnValue(true); + try { + await expect(getValidToken("u1")).resolves.toBeNull(); + expect(findUnique).not.toHaveBeenCalled(); + expect(recordSyncFailure).not.toHaveBeenCalled(); + } finally { + vi.mocked(isFitbitRetired).mockReturnValue(false); + } + }); +}); diff --git a/src/lib/fitbit/retirement.ts b/src/lib/fitbit/retirement.ts new file mode 100644 index 000000000..3f42a7f52 --- /dev/null +++ b/src/lib/fitbit/retirement.ts @@ -0,0 +1,51 @@ +/** + * The end of the legacy Fitbit Web API. + * + * Google turns it off on 30 October 2026: "On October 30, 2026, the Fitbit + * Web API will be turned off and will no longer function or sync data to or + * from Fitbit users" (https://developers.google.com/health/migration and the + * August 2026 Google Health developer newsletter; support already ended on + * 30 September 2026). Its successor is the Google Health API, which reads the + * same Fitbit and Pixel devices. Access and refresh tokens cannot be carried + * over, so a person moves by connecting Google Health and consenting once more. + * + * From the turndown on, HealthLog makes no Web API call for any Fitbit + * connection: no poll, no backfill, no sleep repair, no token refresh, and no + * new connection. The connection row and everything it imported stay. The + * connection reads as not connected to the health read and the watchdog, so a + * silence that is the provider's own shutdown pages nobody and asks nobody to + * reconnect; the card keeps explaining the move instead. + * + * The cut is midnight UTC at the start of the published day, still the + * evening before in the Americas. Stopping those few hours early costs + * nothing Google Health does not bring back with its own history. + */ + +/** The day Google turns the Web API off, as a calendar date. */ +export const FITBIT_WEB_API_TURNDOWN_DATE = "2026-10-30"; + +// eslint-disable-next-line healthlog/no-utc-day-key -- UTC by design: one instant for every instance, the start of the published day (see the header) +const TURNDOWN_AT_MS = Date.parse(`${FITBIT_WEB_API_TURNDOWN_DATE}T00:00:00Z`); + +/** True from the turndown on. */ +export function isFitbitRetired(now: Date = new Date()): boolean { + return now.getTime() >= TURNDOWN_AT_MS; +} + +/** What the status envelope publishes on the Fitbit entry. */ +export interface FitbitRetirement { + /** The turndown day, `YYYY-MM-DD`. */ + endsOn: string; + /** The day has come: the connection no longer syncs. */ + ended: boolean; + /** The integration that reads the same devices. */ + successor: "google-health"; +} + +export function fitbitRetirement(now: Date = new Date()): FitbitRetirement { + return { + endsOn: FITBIT_WEB_API_TURNDOWN_DATE, + ended: isFitbitRetired(now), + successor: "google-health", + }; +} diff --git a/src/lib/fitbit/sync-core.ts b/src/lib/fitbit/sync-core.ts index e52c656c9..d78d1e28d 100644 --- a/src/lib/fitbit/sync-core.ts +++ b/src/lib/fitbit/sync-core.ts @@ -21,6 +21,7 @@ import { recomputeBucketsForKeys, } from "@/lib/rollups/measurement-rollups"; import { refreshAccessToken } from "./client"; +import { isFitbitRetired } from "./retirement"; import { getUserFitbitCredentials } from "./credentials"; import { FitbitApiError, classifyFitbitError } from "./response-classifier"; import type { FitbitClassification } from "./response-classifier"; @@ -71,10 +72,15 @@ export interface FitbitTokenInfo { * response ever omits the new refresh token (a malformed reply), keep the stored * one rather than wiping it — a blank refresh token would brick every future * refresh. + * + * Once Google has turned the Web API off (`./retirement`) there is no token + * worth holding: null, with no refresh attempt and nothing on the ledger, so + * every caller reads the connection as having nothing to fetch. */ export async function getValidToken( userId: string, ): Promise { + if (isFitbitRetired()) return null; const connection = await prisma.fitbitConnection.findUnique({ where: { userId }, }); diff --git a/src/lib/fitbit/sync.ts b/src/lib/fitbit/sync.ts index 84f1f84e5..3e45ac27d 100644 --- a/src/lib/fitbit/sync.ts +++ b/src/lib/fitbit/sync.ts @@ -23,6 +23,7 @@ import type { import { syncUserMetrics } from "./sync-metrics"; import { syncUserSleep } from "./sync-sleep"; import { syncUserWorkout } from "./sync-workout"; +import { isFitbitRetired } from "./retirement"; /** * Full per-user sync across every Fitbit resource. Drives the hourly poll @@ -49,6 +50,13 @@ export async function syncUserFitbit( userId: string, opts: { fullSync?: boolean } = {}, ): Promise<{ imported: number; failed: boolean }> { + // The Web API is gone (`./retirement`): nothing to fetch, nothing to stamp. + // Neither a success (it would paint the connection green) nor a failure (it + // would ask for a reconnect that cannot help). + if (isFitbitRetired()) { + annotate({ action: { name: "fitbit.sync.retired" } }); + return { imported: 0, failed: false }; + } if (await isReauthRequired(userId, "fitbit")) { getEvent()?.addWarning( `fitbit sync skipped for ${userId}: parked at error_reauth`, diff --git a/src/lib/integrations/connection-health-load.ts b/src/lib/integrations/connection-health-load.ts index 4ada42924..e0134cb09 100644 --- a/src/lib/integrations/connection-health-load.ts +++ b/src/lib/integrations/connection-health-load.ts @@ -46,6 +46,7 @@ import { readOuraScopeDenied } from "@/lib/oura/credentials"; import { ouraScopeReconnectList } from "@/lib/oura/scope"; import { localDayWindow } from "@/lib/tz/local-day"; import { resolveUserTimezone } from "@/lib/tz/resolver"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; export type ConnectionHealthMap = Record< ConnectionKey, @@ -234,7 +235,10 @@ export async function loadConnectionFacts( const connected: Record = { withings: !!withingsConn, whoop: !!whoopConn?.whoopUserId, - fitbit: !!fitbitConn, + // Once Google has turned the Fitbit Web API off the row stays (it carries + // the imported history) but nothing will sync through it: no health to + // judge and nothing for the watchdog to call silent. + fitbit: !!fitbitConn && !isFitbitRetired(now), "google-health": !!googleHealthConn, polar: !!dbUser?.polarAccessTokenEncrypted, oura: !!dbUser?.ouraAccessTokenEncrypted, diff --git a/src/lib/jobs/__tests__/fitbit-backfill.test.ts b/src/lib/jobs/__tests__/fitbit-backfill.test.ts index b34e9972a..fb1720bfa 100644 --- a/src/lib/jobs/__tests__/fitbit-backfill.test.ts +++ b/src/lib/jobs/__tests__/fitbit-backfill.test.ts @@ -43,6 +43,7 @@ import { enqueueBootTimeFitbitBackfill, runFitbitBackfillForUser, } from "../fitbit-backfill"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; beforeEach(() => { vi.clearAllMocks(); @@ -146,3 +147,28 @@ describe("runFitbitBackfillForUser — a walk that reached its ceiling", () => { expect(prismaMock.fitbitConnection.update).not.toHaveBeenCalled(); }); }); + +describe("after Google turned the Fitbit Web API off", () => { + it("enqueues no backfill and runs a queued one without walking or stamping", async () => { + vi.mocked(isFitbitRetired).mockReturnValue(true); + try { + prismaMock.fitbitConnection.findMany.mockResolvedValue([ + { userId: "u1" }, + ]); + await expect(enqueueBootTimeFitbitBackfill()).resolves.toEqual({ + enqueued: 0, + skipped: 0, + error: null, + }); + expect(bossSend).not.toHaveBeenCalled(); + + await expect(runFitbitBackfillForUser("u1")).resolves.toEqual({ + imported: 0, + }); + expect(syncUserFitbit).not.toHaveBeenCalled(); + expect(prismaMock.fitbitConnection.update).not.toHaveBeenCalled(); + } finally { + vi.mocked(isFitbitRetired).mockReturnValue(false); + } + }); +}); diff --git a/src/lib/jobs/__tests__/fitbit-sleep-repair.test.ts b/src/lib/jobs/__tests__/fitbit-sleep-repair.test.ts index f35929a20..f768c9799 100644 --- a/src/lib/jobs/__tests__/fitbit-sleep-repair.test.ts +++ b/src/lib/jobs/__tests__/fitbit-sleep-repair.test.ts @@ -71,6 +71,7 @@ import { enqueueBootTimeFitbitSleepRepair, runFitbitSleepRepairForUser, } from "../fitbit-sleep-repair"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; beforeEach(() => { vi.clearAllMocks(); @@ -231,3 +232,29 @@ describe("runFitbitSleepRepairForUser", () => { expect(prismaMock.fitbitConnection.update).not.toHaveBeenCalled(); }); }); + +describe("after Google turned the Fitbit Web API off", () => { + it("enqueues no repair and runs a queued one without reading or stamping", async () => { + vi.mocked(isFitbitRetired).mockReturnValue(true); + try { + prismaMock.fitbitConnection.findMany.mockResolvedValue([ + { userId: "u1" }, + ]); + await expect(enqueueBootTimeFitbitSleepRepair()).resolves.toEqual({ + enqueued: 0, + skipped: 0, + error: null, + }); + expect(bossSend).not.toHaveBeenCalled(); + + await expect(runFitbitSleepRepairForUser("u1")).resolves.toEqual({ + imported: 0, + removed: 0, + }); + expect(syncUserSleep).not.toHaveBeenCalled(); + expect(prismaMock.fitbitConnection.update).not.toHaveBeenCalled(); + } finally { + vi.mocked(isFitbitRetired).mockReturnValue(false); + } + }); +}); diff --git a/src/lib/jobs/__tests__/fitbit-sync-retired.test.ts b/src/lib/jobs/__tests__/fitbit-sync-retired.test.ts new file mode 100644 index 000000000..445930b16 --- /dev/null +++ b/src/lib/jobs/__tests__/fitbit-sync-retired.test.ts @@ -0,0 +1,47 @@ +/** + * The hourly Fitbit cron after Google turned the Web API off: it reads no + * connection, runs no cohort, and finishes as a clean no-op, so the shutdown + * cannot turn into an hourly storm of failed polls. + */ +import { describe, expect, it, vi } from "vitest"; + +const { findMany, runFitbitPollCohort } = vi.hoisted(() => ({ + findMany: vi.fn(async () => [{ userId: "u1" }]), + runFitbitPollCohort: vi.fn(async () => ({ + usersSynced: 1, + measurementsImported: 3, + })), +})); + +vi.mock("@/lib/jobs/reminder/shared", () => ({ + getWorkerPrisma: () => ({ fitbitConnection: { findMany } }), +})); +vi.mock("@/lib/fitbit/sync", () => ({ runFitbitPollCohort })); +vi.mock("@/lib/jobs/reminder-satisfy", () => ({ + enqueueReminderSatisfy: vi.fn(async () => {}), +})); + +import { handleFitbitSync } from "../reminder/fitbit-sync"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; + +describe("handleFitbitSync", () => { + it("polls the cohort while the Web API is up", async () => { + await handleFitbitSync([]); + expect(findMany).toHaveBeenCalledTimes(1); + expect(runFitbitPollCohort).toHaveBeenCalledTimes(1); + }); + + it("does nothing at all after the turndown", async () => { + findMany.mockClear(); + runFitbitPollCohort.mockClear(); + vi.mocked(isFitbitRetired).mockReturnValue(true); + try { + const outcome = await handleFitbitSync([]); + expect(findMany).not.toHaveBeenCalled(); + expect(runFitbitPollCohort).not.toHaveBeenCalled(); + expect(outcome).toMatchObject({ ok: true, did: { total: 0 } }); + } finally { + vi.mocked(isFitbitRetired).mockReturnValue(false); + } + }); +}); diff --git a/src/lib/jobs/fitbit-backfill.ts b/src/lib/jobs/fitbit-backfill.ts index 7fbb0e8f4..ad457bfae 100644 --- a/src/lib/jobs/fitbit-backfill.ts +++ b/src/lib/jobs/fitbit-backfill.ts @@ -20,6 +20,7 @@ import { withPageWalkLedger, } from "@/lib/integrations/page-walk"; import { syncUserFitbit } from "@/lib/fitbit/sync"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; export const FITBIT_BACKFILL_QUEUE = "fitbit-backfill"; @@ -58,6 +59,17 @@ export interface FitbitBackfillPayload { export async function runFitbitBackfillForUser( userId: string, ): Promise<{ imported: number }> { + // A job queued before the turndown: the history it would walk is gone with + // the API. Returns without stamping, so the marker never claims a walk. + if (isFitbitRetired()) { + annotate({ + action: { + name: "fitbit.backfill.skipped_retired", + details: { imported: 0 }, + }, + }); + return { imported: 0 }; + } if (await isReauthRequired(userId, "fitbit")) { annotate({ action: { @@ -133,7 +145,7 @@ export async function enqueueBootTimeFitbitBackfill( error: string | null; }> { const boss = getGlobalBoss(); - if (!boss) { + if (!boss || isFitbitRetired()) { return { enqueued: 0, skipped: 0, error: null }; } diff --git a/src/lib/jobs/fitbit-sleep-repair.ts b/src/lib/jobs/fitbit-sleep-repair.ts index 13cb7c3f4..0b681ff1b 100644 --- a/src/lib/jobs/fitbit-sleep-repair.ts +++ b/src/lib/jobs/fitbit-sleep-repair.ts @@ -45,6 +45,7 @@ import { runWithFitbitHardFailLedger, } from "@/lib/fitbit/sync-core"; import { syncUserSleep } from "@/lib/fitbit/sync-sleep"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; export const FITBIT_SLEEP_REPAIR_QUEUE = "fitbit-sleep-repair"; @@ -82,6 +83,16 @@ export interface FitbitSleepRepairPayload { export async function runFitbitSleepRepairForUser( userId: string, ): Promise<{ imported: number; removed: number }> { + // Nothing left to re-read once the Web API is gone; no stamp either. + if (isFitbitRetired()) { + annotate({ + action: { + name: "fitbit.sleepRepair.skipped_retired", + details: { imported: 0, removed: 0 }, + }, + }); + return { imported: 0, removed: 0 }; + } if (await isReauthRequired(userId, "fitbit")) { annotate({ action: { @@ -155,7 +166,7 @@ export async function enqueueBootTimeFitbitSleepRepair( error: string | null; }> { const boss = getGlobalBoss(); - if (!boss) { + if (!boss || isFitbitRetired()) { return { enqueued: 0, skipped: 0, error: null }; } diff --git a/src/lib/jobs/reminder/fitbit-sync.ts b/src/lib/jobs/reminder/fitbit-sync.ts index e55886ff5..9a560eb38 100644 --- a/src/lib/jobs/reminder/fitbit-sync.ts +++ b/src/lib/jobs/reminder/fitbit-sync.ts @@ -13,6 +13,7 @@ import { runFitbitPollCohort } from "@/lib/fitbit/sync"; import { enqueueReminderSatisfy } from "@/lib/jobs/reminder-satisfy"; import { cleanupExpiredFitbitOAuthStates } from "@/lib/jobs/fitbit-oauth-state-cleanup"; import { getWorkerPrisma } from "./shared"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; /** * v1.12.0 — Fitbit poll-sync payload. Poll-only (no webhook at launch): the @@ -28,6 +29,20 @@ export async function handleFitbitSync( jobs: Job[], ): Promise { return withBackgroundEvent("job.fitbit_sync", async (evt) => { + // Google turned the Web API off (`@/lib/fitbit/retirement`): the cron + // tick reads no connection and calls nothing. + if (isFitbitRetired()) { + evt.setBackground({ + task_name: "job.fitbit_sync", + result: { retired: true }, + }); + return jobDone({ + total: 0, + users_synced: 0, + users_failed: 0, + measurements_imported: 0, + }); + } const prisma = getWorkerPrisma(); try { const targets: Array<{ userId: string }> = []; diff --git a/src/lib/openapi/error-codes.ts b/src/lib/openapi/error-codes.ts index 7849dc564..e509e18b3 100644 --- a/src/lib/openapi/error-codes.ts +++ b/src/lib/openapi/error-codes.ts @@ -484,6 +484,7 @@ export const ERROR_CODE_CATALOGUE: Readonly> = "oidc_only", "private_origin_not_approved", "private_origin_not_grantable", + "provider_retired", "rate_limited_self", "sync_in_progress", "too_many_rows", diff --git a/src/lib/openapi/routes/integrations.ts b/src/lib/openapi/routes/integrations.ts index d0e0500b2..212eb487b 100644 --- a/src/lib/openapi/routes/integrations.ts +++ b/src/lib/openapi/routes/integrations.ts @@ -1151,10 +1151,29 @@ const integrationEnvelopeEntry = z ), hasToken: z.boolean().optional().describe("Nightscout only."), allowPrivateHost: z.boolean().optional().describe("Nightscout only."), + retirement: z + .object({ + endsOn: z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/) + .describe("The day Google turns the legacy Fitbit Web API off."), + ended: z + .boolean() + .describe( + "The day has come: the connection no longer syncs, its imported data stays, and `health` is null.", + ), + successor: z + .enum(["google-health"]) + .describe( + "The integration that reads the same Fitbit and Pixel devices. Moving needs a fresh consent; the Fitbit grant cannot be carried over.", + ), + }) + .optional() + .describe("Fitbit only. The end of the legacy Fitbit Web API."), health: connectionHealth .nullable() .describe( - "The unified connection state, null when the provider is not connected. It supersedes `state`, `lastError`, `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, `hasActivityScope`, `planUnavailable` and `scopeReconnect` on this entry; those nine are deprecated and removed in v1.44.", + "The unified connection state, null when the provider is not connected (and for Fitbit once `retirement.ended`). It supersedes `state`, `lastError`, `consecutiveFailuresByKind`, `syncHealth`, `tokenExpired`, `needsReauth`, `hasActivityScope`, `planUnavailable` and `scopeReconnect` on this entry; those nine are deprecated and removed in v1.44.", ), }) .meta({ @@ -2020,6 +2039,11 @@ export const integrationPaths: NonNullable = { "Either limiter tripped: more than 5 syncs in 60 s, or a second full sync within the hour. `meta.errorCode` = `rate_limited_self` for both; the message distinguishes them.", content: { "application/json": { schema: errorEnvelope } }, }, + "410": { + description: + "Google turned the legacy Fitbit Web API off on 2026-10-30; nothing is fetched. `meta.errorCode` = `provider_retired`. The data already imported stays; Google Health reads the same devices.", + content: { "application/json": { schema: errorEnvelope } }, + }, }, }, }, @@ -2663,6 +2687,11 @@ export const integrationPaths: NonNullable = { ...stdResponses, ...probeNotConfigured422, ...selfRateLimited429, + "410": { + description: + "Google turned the legacy Fitbit Web API off on 2026-10-30; nothing is probed. `meta.errorCode` = `provider_retired`.", + content: { "application/json": { schema: errorEnvelope } }, + }, }, }, }, diff --git a/tests/integration/environment-setup.ts b/tests/integration/environment-setup.ts index a3c7c4da6..2613d151e 100644 --- a/tests/integration/environment-setup.ts +++ b/tests/integration/environment-setup.ts @@ -1,4 +1,4 @@ -import { afterEach, inject } from "vitest"; +import { afterEach, inject, vi } from "vitest"; declare module "vitest" { export interface ProvidedContext { @@ -32,3 +32,24 @@ afterEach(async () => { await import("@/lib/logging/background-tasks"); await settleBackgroundTasks(); }); + +/** + * The same pin the unit setup applies (see `vitest.setup.ts`): the Fitbit + * suites run before the Web API turndown, whatever the real date, and a test + * about the turndown flips `isFitbitRetired` itself. The factory imports the + * module lazily, so nothing of the application loads before DATABASE_URL is + * bridged above. + */ +vi.mock("@/lib/fitbit/retirement", async (importOriginal) => { + const actual = + await importOriginal(); + const isFitbitRetired = vi.fn((_now?: Date) => false); + return { + ...actual, + isFitbitRetired, + fitbitRetirement: vi.fn((now?: Date) => ({ + ...actual.fitbitRetirement(now), + ended: isFitbitRetired(now), + })), + }; +}); diff --git a/tests/integration/fitbit-retirement.test.ts b/tests/integration/fitbit-retirement.test.ts new file mode 100644 index 000000000..b75f0d2b3 --- /dev/null +++ b/tests/integration/fitbit-retirement.test.ts @@ -0,0 +1,113 @@ +/** + * After Google turned the Fitbit Web API off, a Fitbit connection that has + * gone quiet is the provider's shutdown, not a broken pipe. Against real + * Postgres: the health read publishes no state for it (so no card or banner + * asks for a reconnect), the watchdog pages nobody, and the connection row + * and every imported reading stay. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); +vi.mock("@/lib/notifications/dispatcher", () => ({ + dispatchNotification: vi.fn().mockResolvedValue({ + dispatched: true, + channelsAttempted: 1, + channelsSucceeded: 1, + }), +})); + +import { dispatchNotification } from "@/lib/notifications/dispatcher"; +import { runIntegrationWatchdog } from "@/lib/integrations/watchdog"; +import { loadConnectionHealth } from "@/lib/integrations/connection-health-load"; +import { recordSyncSuccess } from "@/lib/integrations/status"; +import { isFitbitRetired } from "@/lib/fitbit/retirement"; + +const HOUR = 60 * 60 * 1000; +const DAY = 24 * HOUR; +const NOW = new Date("2026-11-03T12:00:00.000Z"); +const USER = "user-fitbit-retired"; +const ADMIN = "user-fitbit-retired-admin"; +const prisma = () => getPrismaClient(); + +beforeEach(async () => { + await truncateAllTables(prisma()); + await prisma().user.create({ + data: { + id: ADMIN, + username: "fr-admin", + email: "admin@example.test", + role: "ADMIN", + }, + }); + await prisma().user.create({ + data: { id: USER, username: "fr-subject", email: "subject@example.test" }, + }); + await prisma().fitbitConnection.create({ + data: { + userId: USER, + fitbitUserId: "fb-1", + accessToken: "not-a-real-token", + refreshToken: "not-a-real-token", + tokenExpiresAt: new Date(NOW.getTime() - DAY), + lastSyncedAt: new Date(NOW.getTime() - 5 * DAY), + }, + }); + // A fortnight of daily steps that stopped with the turndown. + await prisma().measurement.createMany({ + data: Array.from({ length: 14 }, (_, i) => ({ + userId: USER, + type: "ACTIVITY_STEPS" as const, + value: 8000, + unit: "count", + source: "FITBIT" as const, + measuredAt: new Date(NOW.getTime() - (i + 5) * DAY), + externalId: `stats:day-${i}`, + })), + }); + vi.mocked(dispatchNotification).mockClear(); + // The watchdog has run on this instance before, so it pages on a fresh + // incident instead of recording what stands. + await prisma().integrationWatch.create({ + data: { userId: ADMIN, integration: "withings" }, + }); +}); + +describe("Fitbit after the Web API turndown", () => { + it("still reads as a connection with a health state before the day", async () => { + await recordSyncSuccess(USER, "fitbit"); + const health = await loadConnectionHealth(USER, ["fitbit"]); + expect(health.fitbit).not.toBeNull(); + }); + + it("publishes no health, pages nobody, and keeps the row and the data", async () => { + await recordSyncSuccess(USER, "fitbit"); + vi.mocked(isFitbitRetired).mockReturnValue(true); + try { + const health = await loadConnectionHealth(USER, ["fitbit"]); + expect(health.fitbit).toBeNull(); + + await runIntegrationWatchdog({ mode: "hourly", now: NOW }); + await runIntegrationWatchdog({ mode: "daily", now: NOW }); + expect(dispatchNotification).not.toHaveBeenCalled(); + const watch = await prisma().integrationWatch.findUnique({ + where: { userId_integration: { userId: USER, integration: "fitbit" } }, + }); + expect(watch?.alertedAt ?? null).toBeNull(); + } finally { + vi.mocked(isFitbitRetired).mockReturnValue(false); + } + + expect( + await prisma().fitbitConnection.count({ where: { userId: USER } }), + ).toBe(1); + expect( + await prisma().measurement.count({ + where: { userId: USER, source: "FITBIT" }, + }), + ).toBe(14); + }); +}); diff --git a/vitest.setup.ts b/vitest.setup.ts index 8e1a180de..5412a6d6d 100644 --- a/vitest.setup.ts +++ b/vitest.setup.ts @@ -59,3 +59,27 @@ vi.mock("@/lib/auth/write-meter", async (importOriginal) => { await importOriginal(); return { ...actual, meterAccountWrite: vi.fn(async () => undefined) }; }); + +/** + * The end of the legacy Fitbit Web API is a calendar date + * (`src/lib/fitbit/retirement.ts`). Read off the real clock, every suite that + * exercises the Fitbit sync, its jobs or its routes would turn red on that day + * and stay red until the provider code is removed. The suites therefore run + * before the turndown by default; a test about the turndown itself flips + * `isFitbitRetired` (`vi.mocked(isFitbitRetired).mockReturnValue(true)`) or + * loads the real module with `vi.importActual`, as + * `src/lib/fitbit/__tests__/retirement.test.ts` does. + */ +vi.mock("@/lib/fitbit/retirement", async (importOriginal) => { + const actual = + await importOriginal(); + const isFitbitRetired = vi.fn((_now?: Date) => false); + return { + ...actual, + isFitbitRetired, + fitbitRetirement: vi.fn((now?: Date) => ({ + ...actual.fitbitRetirement(now), + ended: isFitbitRetired(now), + })), + }; +}); From 3b1cdd7a1f980540c41c41d4ab048679c0d61f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 12:51:19 +0200 Subject: [PATCH 6/9] fix(strava): keep a rate-limited run's progress and a summary-only row's calories A 429 in the middle of the activity walk threw out of it before the upsert, so everything the run had read was discarded and the next run started over. The walk now ends where the limit hit: what was read lands, the cursor moves only as far as an oldest-first walk proves, the failure is booked transient, and the run reports the wait Strava's X-RateLimit / X-ReadRateLimit headers name (next quarter hour, or midnight UTC once the daily count is spent). The backfill waits that long before its next run. `walkPages` learns an `interrupted` page for this, reported truncated like a walk cut off at its ceiling. An update mapped from the summary alone (detail budget spent, or the detail read failed) wrote a null over the calories an earlier detail read had stored, and over its heart rate and description too. Those detail-only fields are now kept unless this run read the detail. --- src/lib/integrations/page-walk.ts | 12 +- src/lib/jobs/strava-backfill.ts | 23 +- .../strava/__tests__/sync-rate-limit.test.ts | 279 ++++++++++++++++++ src/lib/strava/client.ts | 51 ++++ src/lib/strava/response-classifier.ts | 16 +- src/lib/strava/sync.ts | 140 +++++++-- 6 files changed, 491 insertions(+), 30 deletions(-) create mode 100644 src/lib/strava/__tests__/sync-rate-limit.test.ts diff --git a/src/lib/integrations/page-walk.ts b/src/lib/integrations/page-walk.ts index b8b92f451..e05e8cf89 100644 --- a/src/lib/integrations/page-walk.ts +++ b/src/lib/integrations/page-walk.ts @@ -49,6 +49,13 @@ import { annotate } from "@/lib/logging/context"; export interface WalkPage { items: readonly T[]; next: C | null | undefined; + /** + * The provider refused to go on for now (a rate limit) with history left. + * The walk ends here with what it read, reported `truncated` like a walk + * cut off at its ceiling, so a backfill resumes later instead of stamping + * a history it did not finish. + */ + interrupted?: boolean; } /** What the walk came to. */ @@ -78,6 +85,9 @@ export async function walkPages( for (let page = 0; page < maxPages; page += 1) { const result = await fetchPage(cursor, page); items.push(...result.items); + if (result.interrupted) { + return truncated(label, items, page + 1, "interrupted"); + } const next = result.next; if (next === null || next === undefined || next === "") { return { items, pages: page + 1, truncated: false }; @@ -93,7 +103,7 @@ function truncated( label: string, items: T[], pages: number, - reason: "ceiling" | "repeat", + reason: "ceiling" | "repeat" | "interrupted", ): PageWalk { annotate({ action: { name: "integration.page_walk.truncated" }, diff --git a/src/lib/jobs/strava-backfill.ts b/src/lib/jobs/strava-backfill.ts index 5c36fc905..10bb6dd21 100644 --- a/src/lib/jobs/strava-backfill.ts +++ b/src/lib/jobs/strava-backfill.ts @@ -93,7 +93,7 @@ export async function runStravaBackfillForUser( const { result, truncated } = await withPageWalkLedger(() => syncStravaHistory(userId, before), ); - const { imported, oldestStart } = result; + const { imported, oldestStart, retryAfterSeconds } = result; // A failure has already thrown; what is left to decide is whether the walk // reached the beginning of the history. if (backfillVerdict({ failed: false, truncated }) === "incomplete") { @@ -111,7 +111,18 @@ export async function runStravaBackfillForUser( data: { stravaBackfillResumeBefore: resumeBefore }, }); } - const continued = progressed && (await sendNextRun(userId, slice)); + // A run Strava's rate limit ended goes on once the window it named has + // reset, progress or not: the limit is the app's, and passes. + const continued = + (progressed || retryAfterSeconds !== undefined) && + (await sendNextRun( + userId, + slice, + Math.max( + STRAVA_BACKFILL_CONTINUATION_DELAY_SECONDS, + retryAfterSeconds ?? 0, + ), + )); annotate({ action: { name: "strava.backfill.truncated", @@ -143,7 +154,11 @@ export async function runStravaBackfillForUser( return { imported, continued: false }; } -async function sendNextRun(userId: string, slice: number): Promise { +async function sendNextRun( + userId: string, + slice: number, + delaySeconds: number, +): Promise { const boss = getGlobalBoss(); if (!boss) return false; const sent = await enqueueIntegrationBackfillAdmission( @@ -152,7 +167,7 @@ async function sendNextRun(userId: string, slice: number): Promise { kind: "strava-backfill", data: { userId, enqueuedAt: new Date().toISOString(), slice: slice + 1 }, }, - STRAVA_BACKFILL_CONTINUATION_DELAY_SECONDS, + delaySeconds, ); return Boolean(sent); } diff --git a/src/lib/strava/__tests__/sync-rate-limit.test.ts b/src/lib/strava/__tests__/sync-rate-limit.test.ts new file mode 100644 index 000000000..08e5aede8 --- /dev/null +++ b/src/lib/strava/__tests__/sync-rate-limit.test.ts @@ -0,0 +1,279 @@ +/** + * Strava's rate limit and its detail budget. + * + * - A 429 mid-walk used to discard everything the run had read: the error + * threw out of the walk before the upsert. The activities read before it + * land now, the cursor moves only as far as an oldest-first walk proves + * safe, the failure is booked as transient, and the backfill waits for + * the window Strava's headers name before it goes on. + * - An update mapped from the summary alone (detail budget spent, or a + * detail read that failed) wrote `totalEnergyKcal: null` over the + * calories an earlier detail read had stored. The fields only the detail + * carries are kept when this run did not read it. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + fetchActivities: vi.fn(), + fetchActivityById: vi.fn(), + annotate: vi.fn(), + findUser: vi.fn(), + updateUser: vi.fn(), + updateManyUser: vi.fn(async (_arg: unknown) => ({ count: 1 })), + createManyAndReturn: vi.fn(), + updateWorkout: vi.fn(async (_arg: { data: Record }) => ({})), + findWorkout: vi.fn(), + recordSyncFailure: vi.fn(async () => {}), + recordSyncSuccess: vi.fn(async () => {}), + isReauthRequired: vi.fn(async () => false), + send: vi.fn(async (..._args: unknown[]) => "job-next"), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { + user: { + findUnique: mocks.findUser, + updateMany: mocks.updateManyUser, + update: mocks.updateUser, + }, + workout: { + createManyAndReturn: mocks.createManyAndReturn, + update: mocks.updateWorkout, + findUnique: mocks.findWorkout, + }, + }, +})); +vi.mock("@/lib/arrivals/workout-emit", () => ({ + emitInsertedWorkoutArrival: vi.fn(async () => {}), +})); +vi.mock("@/lib/cache/invalidate", () => ({ + invalidateUserMeasurements: vi.fn(), +})); +vi.mock("@/lib/logging/context", () => ({ + annotate: mocks.annotate, + getEvent: () => ({ addWarning: vi.fn(), addExternalCall: vi.fn() }), +})); +vi.mock("@/lib/integrations/status", async (importOriginal) => ({ + ...(await importOriginal()), + recordSyncFailure: mocks.recordSyncFailure, + recordSyncSuccess: mocks.recordSyncSuccess, + isReauthRequired: mocks.isReauthRequired, +})); +vi.mock("../credentials", () => ({ + getStravaConnection: vi.fn(async () => ({ + accessToken: "access-token", + refreshToken: "refresh-token", + refreshTokenCiphertext: Buffer.from("ciphertext"), + })), + getStravaClientCredentials: vi.fn(), + storeStravaTokens: vi.fn(), +})); +vi.mock("../client", async (importOriginal) => ({ + ...(await importOriginal()), + fetchActivities: mocks.fetchActivities, + fetchActivityById: mocks.fetchActivityById, +})); +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: vi.fn(() => ({ send: mocks.send })), +})); + +import { runStravaBackfillForUser } from "@/lib/jobs/strava-backfill"; +import { StravaApiError } from "../response-classifier"; +import { stravaRateLimitRetryAfterSeconds } from "../client"; +import { syncUserStrava } from "../sync"; + +const HOUR = 3_600_000; +const T0 = Date.UTC(2026, 9, 1); + +function activity(id: number) { + return { + id, + sport_type: "Run", + start_date: new Date(T0 + id * HOUR).toISOString(), + moving_time: 1800, + elapsed_time: 1900, + average_heartrate: 140, + }; +} + +function rateLimited(retryAfterSeconds: number): StravaApiError { + return new StravaApiError({ + verb: "fetchActivities", + classification: "transient", + httpStatus: 429, + reason: "http_429", + retryAfterSeconds, + }); +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.findUser.mockResolvedValue({ + stravaLastActivityAt: new Date(T0), + stravaBackfillResumeBefore: null, + }); + mocks.createManyAndReturn.mockImplementation( + async ({ data }: { data: { externalId: string; startedAt: Date } }) => [ + { id: `w-${data.externalId}`, startedAt: data.startedAt }, + ], + ); + mocks.fetchActivityById.mockImplementation( + async (_t: string, id: number) => ({ + ...activity(id), + calories: 300, + }), + ); +}); + +describe("a 429 in the middle of a walk", () => { + it("keeps what was read, moves the cursor an oldest-first walk proves, and books it transient", async () => { + // An incremental walk (`after` set): Strava answers oldest first. Three + // activities read, then the detail read of the fourth is refused. + mocks.fetchActivities.mockResolvedValue([1, 2, 3, 4, 5].map(activity)); + mocks.fetchActivityById.mockImplementation( + async (_t: string, id: number) => { + if (id === 4) throw rateLimited(600); + return { ...activity(id), calories: 300 }; + }, + ); + + await expect(syncUserStrava("user-1")).resolves.toBe(3); + + expect(mocks.createManyAndReturn).toHaveBeenCalledTimes(3); + expect(mocks.updateManyUser).toHaveBeenCalledWith( + expect.objectContaining({ + data: { stravaLastActivityAt: new Date(T0 + 3 * HOUR) }, + }), + ); + expect(mocks.recordSyncFailure).toHaveBeenCalledWith( + expect.objectContaining({ kind: "transient", errorCode: "429" }), + ); + expect(mocks.recordSyncSuccess).not.toHaveBeenCalled(); + }); + + it("does not move the cursor past a newest-first walk it did not finish", async () => { + mocks.fetchActivities.mockResolvedValue([9, 8, 7].map(activity)); + mocks.fetchActivityById.mockImplementation( + async (_t: string, id: number) => { + if (id === 7) throw rateLimited(600); + return { ...activity(id), calories: 300 }; + }, + ); + + await expect(syncUserStrava("user-1")).resolves.toBe(2); + expect(mocks.updateManyUser).not.toHaveBeenCalled(); + }); + + it("sends the next backfill run after the window Strava named", async () => { + // A history walk (no `after`): newest first, refused on the second page. + mocks.fetchActivities.mockImplementation( + async (_t: string, q: { page: number }) => { + if (q.page === 2) throw rateLimited(5400); + return Array.from({ length: 100 }, (_, i) => activity(500 - i)); + }, + ); + + const run = await runStravaBackfillForUser("user-1"); + + expect(run).toEqual({ imported: 100, continued: true }); + expect(mocks.updateUser).toHaveBeenCalledWith({ + where: { id: "user-1" }, + data: { + stravaBackfillResumeBefore: new Date(T0 + 401 * HOUR + 1000), + }, + }); + const [, , options] = mocks.send.mock.calls[0] as unknown as [ + string, + unknown, + { startAfter?: number }, + ]; + expect(options.startAfter).toBe(5400); + expect(mocks.updateUser).not.toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + stravaBackfillCompletedAt: expect.any(Date), + }), + }), + ); + }); +}); + +describe("an update without the detail read", () => { + it("keeps the stored calories, heart rate and description", async () => { + // The row exists already; this run reads only the summary (no HR on it) + // because the detail read failed. + mocks.createManyAndReturn.mockResolvedValue([]); + mocks.fetchActivities.mockResolvedValue([ + { ...activity(1), average_heartrate: undefined }, + ]); + mocks.fetchActivityById.mockRejectedValue(new Error("detail down")); + mocks.findWorkout.mockResolvedValue({ + metadata: { stravaName: "old", stravaDescription: "Hill reps" }, + }); + + await syncUserStrava("user-1"); + + const data = mocks.updateWorkout.mock.calls[0]![0].data as Record< + string, + unknown + >; + expect(data).not.toHaveProperty("totalEnergyKcal"); + expect(data).not.toHaveProperty("avgHeartRate"); + expect(data).not.toHaveProperty("maxHeartRate"); + expect(data.metadata).toMatchObject({ stravaDescription: "Hill reps" }); + expect(data.durationSec).toBe(1800); + }); + + it("writes what a detail read returned, nulls included", async () => { + mocks.createManyAndReturn.mockResolvedValue([]); + mocks.fetchActivities.mockResolvedValue([activity(1)]); + mocks.fetchActivityById.mockResolvedValue({ + ...activity(1), + calories: null, + }); + + await syncUserStrava("user-1"); + + const data = mocks.updateWorkout.mock.calls[0]![0].data as Record< + string, + unknown + >; + expect(data.totalEnergyKcal).toBeNull(); + expect(mocks.findWorkout).not.toHaveBeenCalled(); + }); +}); + +describe("stravaRateLimitRetryAfterSeconds", () => { + const at = new Date("2026-10-21T10:07:30Z"); + + it("waits for the next quarter hour when the short window is spent", () => { + const headers = new Headers({ + "X-RateLimit-Limit": "200,2000", + "X-RateLimit-Usage": "200,950", + }); + expect(stravaRateLimitRetryAfterSeconds(headers, at)).toBe(450); + }); + + it("waits for midnight UTC when the daily limit is spent", () => { + const headers = new Headers({ + "X-ReadRateLimit-Limit": "100,1000", + "X-ReadRateLimit-Usage": "40,1000", + }); + expect(stravaRateLimitRetryAfterSeconds(headers, at)).toBe( + 13 * 3600 + 52 * 60 + 30, + ); + }); + + it("honours a Retry-After header when one is sent", () => { + expect( + stravaRateLimitRetryAfterSeconds( + new Headers({ "Retry-After": "120" }), + at, + ), + ).toBe(120); + }); + + it("falls back to the next quarter hour without any header", () => { + expect(stravaRateLimitRetryAfterSeconds(new Headers(), at)).toBe(450); + }); +}); diff --git a/src/lib/strava/client.ts b/src/lib/strava/client.ts index a5407f915..3bc4d1f5d 100644 --- a/src/lib/strava/client.ts +++ b/src/lib/strava/client.ts @@ -258,11 +258,54 @@ async function stravaGet( classification: verdict.classification, httpStatus: verdict.httpStatus, reason: verdict.reason, + ...(res.status === 429 + ? { retryAfterSeconds: stravaRateLimitRetryAfterSeconds(res.headers) } + : {}), }); } return (await res.json().catch(() => null)) as T; } +const QUARTER_HOUR_MS = 15 * 60 * 1000; +const DAY_MS = 24 * 60 * 60 * 1000; + +/** + * How long to wait after a 429, from the response's own headers. Strava sends + * no `Retry-After` (one is honoured should it appear); it sends + * `X-RateLimit-Limit` / `X-RateLimit-Usage` and, for reads, + * `X-ReadRateLimit-Limit` / `X-ReadRateLimit-Usage`, each `<15-min>,`. + * The 15-minute window resets at :00, :15, :30 and :45, the daily one at + * midnight UTC (developers.strava.com/docs/rate-limits). A spent daily count + * waits for midnight; anything else, the next quarter hour. + */ +export function stravaRateLimitRetryAfterSeconds( + headers: Headers, + now: Date = new Date(), +): number { + const retryAfter = Number(headers.get("retry-after")); + if (Number.isFinite(retryAfter) && retryAfter > 0) { + return Math.ceil(retryAfter); + } + const pair = (name: string): [number, number] | null => { + const parts = (headers.get(name) ?? "").split(",").map(Number); + return parts.length === 2 && parts.every(Number.isFinite) + ? [parts[0]!, parts[1]!] + : null; + }; + const dailySpent = (limitName: string, usageName: string): boolean => { + const limit = pair(limitName); + const usage = pair(usageName); + return limit !== null && usage !== null && usage[1] >= limit[1]; + }; + const nowMs = now.getTime(); + const resetAt = + dailySpent("x-ratelimit-limit", "x-ratelimit-usage") || + dailySpent("x-readratelimit-limit", "x-readratelimit-usage") + ? Math.floor(nowMs / DAY_MS) * DAY_MS + DAY_MS + : Math.floor(nowMs / QUARTER_HOUR_MS) * QUARTER_HOUR_MS + QUARTER_HOUR_MS; + return Math.max(1, Math.ceil((resetAt - nowMs) / 1000)); +} + /** One page of the athlete's activities (newest first). */ export async function fetchActivities( accessToken: string, @@ -312,6 +355,13 @@ export interface StravaWorkoutRow { maxHeartRate: number | null; elevationM: number | null; metadata: Prisma.InputJsonValue; + /** + * True when the row was mapped from the summary alone. The calories, the + * description and (when the summary lacks it) the heart rate exist only on + * the detail read, so an update from such a row must not write their + * absence over what an earlier detail read stored. + */ + summaryOnly?: boolean; } function nonNegInt(n: number | null | undefined): number | null { @@ -414,6 +464,7 @@ export function mapActivity( summary.total_elevation_gain ?? detail?.total_elevation_gain, ), metadata, + summaryOnly: detail == null, }; } diff --git a/src/lib/strava/response-classifier.ts b/src/lib/strava/response-classifier.ts index 57f5c81fc..bae3bcf70 100644 --- a/src/lib/strava/response-classifier.ts +++ b/src/lib/strava/response-classifier.ts @@ -28,18 +28,32 @@ export function classifyStravaResponse( } export class StravaApiError extends IntegrationApiError { + /** + * On a 429: how long Strava's own headers say to wait before asking again + * (`stravaRateLimitRetryAfterSeconds`). Undefined on every other failure. + */ + readonly retryAfterSeconds: number | undefined; + constructor(opts: { verb: string; classification: IntegrationClassification; httpStatus: number | undefined; reason: string; upstreamError?: string; + retryAfterSeconds?: number; }) { - super({ vendor: "Strava", ...opts }); + const { retryAfterSeconds, ...rest } = opts; + super({ vendor: "Strava", ...rest }); this.name = "StravaApiError"; + this.retryAfterSeconds = retryAfterSeconds; } } +/** True for Strava's rate-limit refusal. */ +export function isStravaRateLimited(err: unknown): err is StravaApiError { + return err instanceof StravaApiError && err.httpStatus === 429; +} + /** * True when a caught error is an OAuth `invalid_grant` on the token endpoint — * the canonical revoked / rotated-away refresh-token signal. Lifts a 400 diff --git a/src/lib/strava/sync.ts b/src/lib/strava/sync.ts index 7405530d0..35752d1b3 100644 --- a/src/lib/strava/sync.ts +++ b/src/lib/strava/sync.ts @@ -21,10 +21,14 @@ * Rate limits: 200 req / 15 min + 2000 / day per app. The walk is page-bounded * (`walkPages`, which reports a walk cut off at its ceiling instead of * passing it off as the whole history) and the per-activity detail fetch (for `calories`, absent from the summary) - * is budget-capped; a 429 mid-walk records a transient failure and the next - * cron tick resumes from the cursor. + * is budget-capped. A 429 mid-walk ends the walk where it stands: what was + * read lands, the cursor moves only as far as an oldest-first walk proves, the + * failure is booked transient, and the run reports how long Strava's headers + * say to wait (`retryAfterSeconds`), which the backfill waits out before its + * next run. It used to throw out of the walk and discard everything read. */ import { prisma } from "@/lib/db"; +import type { Prisma } from "@/generated/prisma/client"; import { emitInsertedWorkoutArrival } from "@/lib/arrivals/workout-emit"; import { invalidateUserMeasurements } from "@/lib/cache/invalidate"; import { annotate, getEvent } from "@/lib/logging/context"; @@ -50,7 +54,11 @@ import { getStravaConnection, storeStravaTokens, } from "./credentials"; -import { StravaApiError, classifyStravaError } from "./response-classifier"; +import { + StravaApiError, + classifyStravaError, + isStravaRateLimited, +} from "./response-classifier"; /** Default first-incremental lookback (days) when no cursor exists yet. */ export const STRAVA_SYNC_LOOKBACK_DAYS = 30; @@ -115,6 +123,8 @@ export interface StravaHistoryRun { * expect. */ oldestStart: Date | null; + /** Set when Strava's rate limit ended the run: seconds until it resets. */ + retryAfterSeconds?: number; } /** @@ -203,6 +213,9 @@ async function runStravaSync( let oldestStart: Date | null = null; let previousStart: number | null = null; let newestFirst = true; + // Assigned inside the page callback; the cast keeps the compiler from + // narrowing it to its initial `null` after the walk. + let rateLimited = null as StravaApiError | null; try { // Strava pages by number; a short page is the last one. A walk that @@ -213,29 +226,37 @@ async function runStravaSync( maxPages, async (cursor) => { const page = cursor ?? 1; - const activities = await authed((t) => - fetchActivities(t, { - after: afterEpoch, - before: beforeEpoch, - page, - perPage: PER_PAGE, - }), - ); + let activities: Awaited>; + try { + activities = await authed((t) => + fetchActivities(t, { + after: afterEpoch, + before: beforeEpoch, + page, + perPage: PER_PAGE, + }), + ); + } catch (err) { + if (!isStravaRateLimited(err)) throw err; + rateLimited = err; + return { items: [], next: null, interrupted: true }; + } const mapped: StravaWorkoutRow[] = []; for (const a of activities) { // Fetch detail only to fill `calories` (absent from the summary), // and only while the budget allows. When the summary already lacks // HR we also let the detail top it up. A detail failure is - // non-fatal — map from the summary — but a 429 aborts the whole - // walk (transient). + // non-fatal — map from the summary — but a 429 ends the walk here, + // keeping the activities before this one. let detail: StravaDetailedActivity | null = null; if (detailBudget > 0) { try { detail = await authed((t) => fetchActivityById(t, a.id)); detailBudget -= 1; } catch (err) { - if (err instanceof StravaApiError && err.httpStatus === 429) { - throw err; + if (isStravaRateLimited(err)) { + rateLimited = err; + return { items: mapped, next: null, interrupted: true }; } getEvent()?.addWarning( `strava: activity detail fetch failed for ${a.id}: ${err}`, @@ -277,6 +298,12 @@ async function runStravaSync( throw markSyncFailureRecorded(err); } + // A walk the rate limit ended is a prefix of the pages in the order Strava + // served them. Oldest first (an incremental walk), every activity up to the + // newest one read is in; newest first, older ones are still missing, and + // moving the cursor would skip them for good. + const cursorSafe = rateLimited === null || !newestFirst; + let imported: number; try { imported = await upsertStravaWorkouts(userId, rows); @@ -284,7 +311,7 @@ async function runStravaSync( // Advance the incremental cursor to the newest activity start we saw, but // never move it backwards (a bounded backfill page-walk may not reach the // very newest activity in one pass). - if (newestStart) { + if (newestStart && cursorSafe) { await prisma.user.updateMany({ where: { id: userId, @@ -301,6 +328,26 @@ async function runStravaSync( throw markSyncFailureRecorded(err); } + if (rateLimited) { + // Not finished, so no success; booked transient, the next run goes on. + await recordStravaFailure(userId, rateLimited); + if (imported > 0) invalidateUserMeasurements(userId); + annotate({ + action: { + name: "strava.sync.rate_limited", + details: { + imported, + retry_after_seconds: rateLimited.retryAfterSeconds ?? null, + }, + }, + }); + return { + imported, + oldestStart: newestFirst ? oldestStart : null, + retryAfterSeconds: rateLimited.retryAfterSeconds, + }; + } + await recordSyncSuccess(userId, "strava"); // Background-sync posture: sweep the workouts cache and mark the analytics @@ -316,6 +363,42 @@ async function runStravaSync( return { imported, oldestStart: newestFirst ? oldestStart : null }; } +/** + * The update for a row mapped from the summary alone. Calories, the + * description and (when the summary has none) the heart rate come only from + * the detail read, so their absence here says nothing: the stored values stay. + * A detail read that did run is written as it came, nulls included. + */ +async function summaryOnlyUpdate< + D extends { + totalEnergyKcal: number | null; + avgHeartRate: number | null; + maxHeartRate: number | null; + metadata: Prisma.InputJsonValue; + }, +>(data: D, stored: { metadata: Prisma.JsonValue } | null) { + const { totalEnergyKcal, avgHeartRate, maxHeartRate, metadata, ...rest } = + data; + const storedMeta = + stored?.metadata && typeof stored.metadata === "object" + ? (stored.metadata as Record) + : {}; + const description = storedMeta.stravaDescription; + return { + ...rest, + ...(totalEnergyKcal != null ? { totalEnergyKcal } : {}), + ...(avgHeartRate != null ? { avgHeartRate } : {}), + ...(maxHeartRate != null ? { maxHeartRate } : {}), + metadata: + typeof description === "string" + ? ({ + ...(metadata as Record), + stravaDescription: description, + } as Prisma.InputJsonValue) + : metadata, + }; +} + /** * Upsert a batch of mapped Strava activities into `Workout`, keyed on * `(userId, source = STRAVA, externalId)` so a re-fetch overwrites in place. @@ -362,15 +445,24 @@ export async function upsertStravaWorkouts( () => {}, ); } else { - await prisma.workout.update({ - where: { - userId_source_externalId: { - userId, - source: "STRAVA", - externalId: r.externalId, - }, + const where = { + userId_source_externalId: { + userId, + source: "STRAVA" as const, + externalId: r.externalId, }, - data, + }; + await prisma.workout.update({ + where, + data: r.summaryOnly + ? await summaryOnlyUpdate( + data, + await prisma.workout.findUnique({ + where, + select: { metadata: true }, + }), + ) + : data, }); } imported += 1; From c1921d1dd1ddc4cf87f5ece3f51cb20db1df9972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 12:59:43 +0200 Subject: [PATCH 7/9] fix(withings): read the sleep stages night by night Sleep v2 Get answers only the first 24 hours after its startdate. The sync asked for the whole 30-day window, so it got the night a month back and nothing since: every night's stages arrived when the sliding window reached it, weeks late. The summary now runs first and lists the nights; each night's stages are read over its own span (at most 24 hours). Every run re-reads the nights that ended, or that Withings changed, in the last 48 hours, and fills older nights that hold no stage row yet, eight per run with a rotation so a night without stages cannot hold up the rest; a full sync reads them all. Without the night list the last 24 hours are read as one slice and nothing is swept, since a slice can cut a night. A summary failure used to be a warning beside a green leg. It is now the sleep leg's failure, recorded after the stages land. --- src/lib/withings/__tests__/sync-sleep.test.ts | 303 +++++++++++++--- src/lib/withings/sync-sleep.ts | 328 +++++++++++++----- tests/integration/withings-sleep-sync.test.ts | 63 +++- 3 files changed, 536 insertions(+), 158 deletions(-) diff --git a/src/lib/withings/__tests__/sync-sleep.test.ts b/src/lib/withings/__tests__/sync-sleep.test.ts index ea945c843..96847511c 100644 --- a/src/lib/withings/__tests__/sync-sleep.test.ts +++ b/src/lib/withings/__tests__/sync-sleep.test.ts @@ -17,6 +17,7 @@ vi.mock("@/lib/db", () => ({ prisma: { measurement: { findFirst: vi.fn(), + findMany: vi.fn(), create: vi.fn(), update: vi.fn(), upsert: vi.fn(), @@ -105,18 +106,71 @@ interface FakeSegment { id?: number; } -function installFetchMock(segments: FakeSegment[]) { - const fetchMock = vi.fn(async () => ({ - status: 200, - json: async () => ({ - status: 0, - body: { series: segments }, - }), - })); +type FakeSummary = { + id?: number; + startdate: number; + enddate: number; + modified?: number; + data?: Record; +}; + +/** The night list Withings would derive from these segments, one per session. */ +function nightsOf(segments: FakeSegment[]): FakeSummary[] { + const bySession = new Map(); + for (const seg of segments) { + const key = String(seg.id ?? "no-id"); + const night = bySession.get(key); + bySession.set( + key, + night + ? { + ...night, + startdate: Math.min(night.startdate, seg.startdate), + enddate: Math.max(night.enddate, seg.enddate), + } + : { id: seg.id, startdate: seg.startdate, enddate: seg.enddate }, + ); + } + return [...bySession.values()]; +} + +/** + * Answer the two Sleep v2 actions the way Withings does: `getsummary` lists + * the nights, `get` returns the segments inside the requested span, and no + * more than its first 24 hours. + */ +function installSleepApi(segments: FakeSegment[], summaries: FakeSummary[]) { + const fetchMock = vi.fn(async (_url: string, init: { body: string }) => { + const params = new URLSearchParams(init.body); + if (params.get("action") === "getsummary") { + return { + status: 200, + json: async () => ({ status: 0, body: { series: summaries } }), + }; + } + // Sleep v2 Get: "only the first 24h after startdate will be returned". + const from = Number(params.get("startdate")); + const to = Math.min(Number(params.get("enddate")), from + 24 * 3600); + return { + status: 200, + json: async () => ({ + status: 0, + body: { + series: segments.filter( + (seg) => seg.startdate >= from && seg.enddate <= to, + ), + }, + }), + }; + }); vi.stubGlobal("fetch", fetchMock); return fetchMock; } +function installFetchMock(segments: FakeSegment[]) { + return installSleepApi(segments, nightsOf(segments)); +} + beforeEach(() => { vi.resetAllMocks(); transactionMock.mockImplementation(async (run: (tx: unknown) => unknown) => @@ -144,6 +198,8 @@ beforeEach(() => { vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({ scope: "user.metrics,user.activity", } as never); + // No stage row stored yet: every listed night is one to read. + vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); }); afterEach(() => { @@ -158,21 +214,9 @@ afterEach(() => { */ function installSegmentThenSummaryFetch( segments: FakeSegment[], - summaries: Array<{ - id?: number; - startdate: number; - enddate: number; - data?: Record; - }>, + summaries: FakeSummary[], ) { - let call = 0; - const fetchMock = vi.fn(async () => { - const body = call === 0 ? { series: segments } : { series: summaries }; - call++; - return { status: 200, json: async () => ({ status: 0, body }) }; - }); - vi.stubGlobal("fetch", fetchMock); - return fetchMock; + return installSleepApi(segments, summaries); } describe("mapWithingsSleepSummary", () => { @@ -559,44 +603,50 @@ describe("syncUserSleep — segment writes + idempotency", () => { expect(byType.get("OXYGEN_SATURATION")?.value).toBe(95); }); - it("keeps stage segments when the summary fetch fails (v1.18.10 P0)", async () => { - let call = 0; - vi.stubGlobal( - "fetch", - vi.fn(async () => { - const isSummary = call > 0; - call++; - return { - status: 200, - json: async () => - isSummary - ? { status: 293 } // summary errors - : { - status: 0, - body: { - series: [ - { - startdate: 1715000000, - enddate: 1715003600, - state: 2, - id: 1, - }, - ], - }, + it("reads the last 24 hours and books the leg failed when the summary fails", async () => { + const fetchMock = vi.fn(async (_url: string, init: { body: string }) => { + const params = new URLSearchParams(init.body); + return { + status: 200, + json: async () => + params.get("action") === "getsummary" + ? { status: 293 } // summary errors + : { + status: 0, + body: { + series: [ + { + startdate: 1715000000, + enddate: 1715003600, + state: 2, + id: 1, + }, + ], }, - }; - }), - ); - vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null); - vi.mocked(prisma.measurement.create).mockResolvedValue({} as never); + }, + }; + }); + vi.stubGlobal("fetch", fetchMock); const imported = await syncUserSleep("user-1"); - // The one stage segment is still written; the summary failure is swallowed. + + // Without the night list one slice of the last 24 hours is read, so + // last night still lands; it is not swept, since a slice can cut a night. expect(imported).toBe(1); - expect(reconcileMock).toHaveBeenCalledTimes(1); - expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings", { - leg: "sleep", - }); + const stageCalls = fetchMock.mock.calls + .map(([, init]) => new URLSearchParams(init.body)) + .filter((p) => p.get("action") === "get"); + expect(stageCalls).toHaveLength(1); + expect( + Number(stageCalls[0]!.get("enddate")) - + Number(stageCalls[0]!.get("startdate")), + ).toBe(24 * 3600); + expect(prisma.measurement.updateMany).not.toHaveBeenCalled(); + // The failure is the leg's, not a warning beside a green leg. + expect(recordSyncSuccess).not.toHaveBeenCalled(); + expect(recordSyncFailure).toHaveBeenCalledWith( + expect.objectContaining({ integration: "withings", leg: "sleep" }), + ); }); it("tolerates a ScanWatch night with no REM segment (no all-zeros row)", async () => { @@ -848,3 +898,144 @@ describe("syncUserSleep — Withings API plan (status 277)", () => { }); }); }); + +describe("syncUserSleep — one stage read per night (Sleep v2 Get answers 24 h)", () => { + const NOW = new Date("2026-10-21T09:00:00.000Z"); + const nowUnix = Math.floor(NOW.getTime() / 1000); + const DAY = 24 * 3600; + + /** Night `n` days back: 22:00 to 06:00, one CORE and one DEEP segment. */ + function night(n: number, id: number) { + const end = nowUnix - 3 * 3600 - n * DAY; // 06:00 that morning + const startdate = end - 8 * 3600; + return { + summary: { id, startdate, enddate: end }, + segments: [ + { startdate, enddate: startdate + 4 * 3600, state: 1, id }, + { startdate: startdate + 4 * 3600, enddate: end, state: 2, id }, + ], + }; + } + + function stageReads(fetchMock: ReturnType) { + return fetchMock.mock.calls + .map(([, init]) => new URLSearchParams(init.body)) + .filter((p) => p.get("action") === "get") + .map((p) => [Number(p.get("startdate")), Number(p.get("enddate"))]); + } + + beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(NOW); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it("reads every listed night with its own span and lands all of them", async () => { + // Before, one read over the 30-day window answered only its first 24 + // hours: last night's stages came weeks late. + const nights = [0, 3, 9].map((n, i) => night(n, 100 + i)); + const fetchMock = installSleepApi( + nights.flatMap((x) => x.segments), + nights.map((x) => x.summary), + ); + + const imported = await syncUserSleep("user-1"); + + expect(imported).toBe(6); + const bySpan = (a: number[], b: number[]) => a[0]! - b[0]!; + expect([...stageReads(fetchMock)].sort(bySpan)).toEqual( + nights.map((x) => [x.summary.startdate, x.summary.enddate]).sort(bySpan), + ); + for (const [from, to] of stageReads(fetchMock)) { + expect(to - from).toBeLessThanOrEqual(DAY); + } + expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings", { + leg: "sleep", + }); + }); + + it("re-reads the recent nights every run and skips older nights already stored", async () => { + const recent = night(0, 1); + const stored = night(5, 2); + const missing = night(6, 3); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([ + { measuredAt: new Date(stored.summary.enddate * 1000) }, + ] as never); + // The recent night is stored too; it is read again for a re-score. + const fetchMock = installSleepApi( + [...recent.segments, ...stored.segments, ...missing.segments], + [recent.summary, stored.summary, missing.summary], + ); + + await syncUserSleep("user-1"); + + expect(stageReads(fetchMock)).toEqual([ + [recent.summary.startdate, recent.summary.enddate], + [missing.summary.startdate, missing.summary.enddate], + ]); + }); + + it("re-reads an older stored night Withings changed in the last 48 hours", async () => { + const rescored = night(6, 4); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([ + { measuredAt: new Date(rescored.summary.enddate * 1000) }, + ] as never); + const fetchMock = installSleepApi(rescored.segments, [ + { ...rescored.summary, modified: nowUnix - 3600 } as FakeSummary, + ]); + + await syncUserSleep("user-1"); + + expect(stageReads(fetchMock)).toEqual([ + [rescored.summary.startdate, rescored.summary.enddate], + ]); + }); + + it("reads at most a run's budget of missing nights, rotating which", async () => { + const nights = Array.from({ length: 12 }, (_, i) => night(i + 3, i + 1)); + const fetchMock = installSleepApi( + nights.flatMap((x) => x.segments), + nights.map((x) => x.summary), + ); + + await syncUserSleep("user-1"); + const first = stageReads(fetchMock); + expect(first).toHaveLength(8); + + fetchMock.mockClear(); + vi.setSystemTime(new Date(NOW.getTime() + 3600_000)); + await syncUserSleep("user-1"); + const second = stageReads(fetchMock); + expect(second).toHaveLength(8); + expect(second).not.toEqual(first); + }); + + it("reads every night of the window on a full sync", async () => { + const nights = Array.from({ length: 12 }, (_, i) => night(i + 3, i + 1)); + const fetchMock = installSleepApi( + nights.flatMap((x) => x.segments), + nights.map((x) => x.summary), + ); + + await syncUserSleep("user-1", { fullSync: true }); + + expect(stageReads(fetchMock)).toHaveLength(12); + }); + + it("caps a night longer than a day at the 24 hours one read can return", async () => { + const startdate = nowUnix - 30 * 3600; + const enddate = nowUnix - 2 * 3600; // 28 hours in bed + const fetchMock = installSleepApi( + [{ startdate, enddate: startdate + 3600, state: 1, id: 5 }], + [{ id: 5, startdate, enddate }], + ); + + await syncUserSleep("user-1"); + + expect(stageReads(fetchMock)).toEqual([[startdate, startdate + DAY]]); + // A capped read did not see the whole night, so the night is not swept. + expect(prisma.measurement.updateMany).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/withings/sync-sleep.ts b/src/lib/withings/sync-sleep.ts index 41dd7f5b1..d2e6a0855 100644 --- a/src/lib/withings/sync-sleep.ts +++ b/src/lib/withings/sync-sleep.ts @@ -122,6 +122,8 @@ export interface WithingsSleepSummary { id?: number; startdate: number; enddate: number; + /** When Withings last changed the night (unix seconds), e.g. a re-score. */ + modified?: number; data?: { hr_average?: number | null; rr_average?: number | null; @@ -385,56 +387,145 @@ export async function syncUserSleep( return 0; } - void opts.fullSync; const now = new Date(); const start = new Date( now.getTime() - SLEEP_BACKFILL_DAYS * 24 * 60 * 60 * 1000, ); - const startUnix = Math.floor(start.getTime() / 1000); const endUnix = Math.floor(now.getTime() / 1000); - let segments: WithingsSleepSegment[]; + // The nights come first: `action=getsummary` lists every night of the + // window (and carries its vitals), and the stage timeline is then read per + // night. `action=get` answers only the first 24 hours after its + // `startdate` ("If your input startdate and enddate are separated by more + // than 24h, only the first 24h after startdate will be returned", Sleep v2 + // Get); asked for the whole window it returned the night a month back and + // nothing since, so every night's stages arrived when the sliding window + // reached it, weeks late. + // + // Withings API plans (2026-10-12): the sleep score, the overnight HRV and + // the respiration rate are Advanced Biomarkers, and one of them in + // `data_fields` has the whole summary refused with 277. Ask only for what + // the plan allows, remember what it refused, and re-probe weekly. + let summaries: WithingsSleepSummary[] = []; + let summaryFailure: unknown = null; + let nightsKnown = false; try { - segments = await fetchWithingsSleep( - tokenInfo.accessToken, - startUnix, - endUnix, - ); - } catch (err) { - // The stage timeline is a free-plan biomarker, so a plan refusal here is - // not expected. If Withings ever moves it, the leg reads nothing rather - // than failing: a 277 must not climb the strike ladder and park the - // measures, activity and ECG legs along with it. - if (isPlanGatedError(err)) { - annotate({ meta: { withings_sleep_segments_skipped: "api_plan" } }); - await recordSyncSuccess(userId, "withings", { leg: WITHINGS_LEG_SLEEP }); - return 0; - } - // v1.4.43 — typed-classification path (see sync-activity.ts - // for the full rationale). The thrown `WithingsApiError` carries - // the classification verdict; `recordWithingsSyncFailure` routes - // it through `classifyError`, which falls back to the legacy regex - // for un-prototyped errors (pg-boss JSON round-trip). - // - // BL-P3-2 defence-in-depth — symmetric to sync-activity. A 403 on - // this scope-gated endpoint always means scope-missing or - // token-revoked; force `reauth_required` so the 3-strike alert - // still pages on the unexpected case. - const withingsStatus = - err instanceof WithingsApiError ? err.withingsStatus : undefined; - if (withingsStatus === 403) { - await recordSyncFailure({ - userId, - integration: "withings", - kind: "reauth_required", - message: err instanceof Error ? err.message : String(err), - errorCode: "403", - leg: WITHINGS_LEG_SLEEP, + const planRow = await prisma.withingsConnection.findUnique({ + where: { userId }, + select: { apiPlanState: true }, + }); + const sleepPlan = parseWithingsApiPlanState( + planRow?.apiPlanState, + ).sleepSummary; + const reprobed = isPlanReprobeDue(sleepPlan, now); + const skip = planSkipFor(sleepPlan, now); + const plan = await requestWithinPlan({ + keys: SLEEP_SUMMARY_DATA_FIELDS.filter((f) => !skip.includes(f)), + knownGated: WITHINGS_ADVANCED_SLEEP_FIELDS, + fetch: (fields) => + fetchWithingsSleepSummary( + tokenInfo.accessToken, + ymdUtc(start), + ymdUtc(now), + fields, + ), + }); + summaries = plan.results; + nightsKnown = plan.acceptedAny; + if ( + shouldPersistPlanLeg({ + gated: plan.gated, + reprobed, + acceptedAny: plan.acceptedAny, + }) + ) { + await persistWithingsPlanLeg(userId, "sleepSummary", { + unavailable: [...skip, ...plan.unavailable], + checkedAt: planCheckedAtFor(sleepPlan, reprobed, now), }); + } + } catch (err) { + // Booked as this leg's failure once the stages are written: the summary + // carries the vitals and names the nights, and a run without it is not + // a healthy one. It used to be a warning, and the leg read green. + summaryFailure = err; + getEvent()?.addWarning( + `withings sleep: summary fetch failed for ${userId}: ${err}`, + ); + } + + // Without the night list (the summary failed, or the plan refused every + // field it was asked for) the last 24 hours are read as one slice, so last + // night still lands. Such a slice can cut a night, so it is never swept. + const windows: StageWindow[] = nightsKnown + ? await stageWindowsToFetch(userId, summaries, now, { + all: opts.fullSync === true, + }) + : [ + { + startdate: endUnix - DAY_SECONDS, + enddate: endUnix, + wholeNight: false, + }, + ]; + + const segments: WithingsSleepSegment[] = []; + const wholeNightSessions = new Set(); + const seenSegments = new Set(); + for (const window of windows) { + let series: WithingsSleepSegment[]; + try { + series = await fetchWithingsSleep( + tokenInfo.accessToken, + window.startdate, + window.enddate, + ); + } catch (err) { + // The stage timeline is a free-plan biomarker, so a plan refusal here + // is not expected. If Withings ever moves it, the leg reads no stages + // rather than failing: a 277 must not climb the strike ladder and park + // the measures, activity and ECG legs along with it. + if (isPlanGatedError(err)) { + annotate({ meta: { withings_sleep_segments_skipped: "api_plan" } }); + break; + } + // v1.4.43 — typed-classification path (see sync-activity.ts + // for the full rationale). The thrown `WithingsApiError` carries + // the classification verdict; `recordWithingsSyncFailure` routes + // it through `classifyError`, which falls back to the legacy regex + // for un-prototyped errors (pg-boss JSON round-trip). + // + // BL-P3-2 defence-in-depth — symmetric to sync-activity. A 403 on + // this scope-gated endpoint always means scope-missing or + // token-revoked; force `reauth_required` so the 3-strike alert + // still pages on the unexpected case. + const withingsStatus = + err instanceof WithingsApiError ? err.withingsStatus : undefined; + if (withingsStatus === 403) { + await recordSyncFailure({ + userId, + integration: "withings", + kind: "reauth_required", + message: err instanceof Error ? err.message : String(err), + errorCode: "403", + leg: WITHINGS_LEG_SLEEP, + }); + throw err; + } + await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_SLEEP); throw err; } - await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_SLEEP); - throw err; + for (const segment of series) { + // Two windows can return the same segment (a night longer than its + // 24-hour slice, a nap inside a night); it is written once. + const key = `${segment.id ?? "no-id"}:${segment.startdate}`; + if (seenSegments.has(key)) continue; + seenSegments.add(key); + segments.push(segment); + if (window.wholeNight && typeof segment.id === "number") { + wholeNightSessions.add(segment.id); + } + } } let imported = 0; @@ -528,63 +619,19 @@ export async function syncUserSleep( // running-index row for re-fetched sessions. Sessions absent from this fetch // are never touched; a sweep failure never fails the sync (the session stays // inside the rolling 30-day window, so the hourly cron retries it). - const sweeps: SleepSegmentSweep[] = [...freshBySession].map( - ([sessionId, keepIds]) => ({ + const sweeps: SleepSegmentSweep[] = [...freshBySession] + // Only a session read across its whole night is swept: a 24-hour slice + // can cut one, and sweeping it would retire the part outside the slice. + .filter(([sessionId]) => wholeNightSessions.has(sessionId)) + .map(([sessionId, keepIds]) => ({ prefix: `withings:sleep:${userId}:${sessionId}:`, keepIds, - }), - ); + })); await sweepStaleSleepSegments(userId, "WITHINGS", sweeps); // v1.18.10 — nightly sleep vitals (avg HR / respiratory rate / SDNN HRV - // / avg SpO2 / sleep score) from the per-night summary. Fetch failure stays - // isolated from the stage writes; database failures are hard sync failures. - // - // Withings API plans (2026-10-12): the sleep score, the overnight HRV and the - // respiration rate are Advanced Biomarkers, and one of them in `data_fields` - // has the whole summary refused with 277. Ask only for what the plan allows, - // remember what it refused, and re-probe weekly. - let summaries: WithingsSleepSummary[] = []; - try { - const planRow = await prisma.withingsConnection.findUnique({ - where: { userId }, - select: { apiPlanState: true }, - }); - const sleepPlan = parseWithingsApiPlanState( - planRow?.apiPlanState, - ).sleepSummary; - const reprobed = isPlanReprobeDue(sleepPlan, now); - const skip = planSkipFor(sleepPlan, now); - const plan = await requestWithinPlan({ - keys: SLEEP_SUMMARY_DATA_FIELDS.filter((f) => !skip.includes(f)), - knownGated: WITHINGS_ADVANCED_SLEEP_FIELDS, - fetch: (fields) => - fetchWithingsSleepSummary( - tokenInfo.accessToken, - ymdUtc(start), - ymdUtc(now), - fields, - ), - }); - summaries = plan.results; - if ( - shouldPersistPlanLeg({ - gated: plan.gated, - reprobed, - acceptedAny: plan.acceptedAny, - }) - ) { - await persistWithingsPlanLeg(userId, "sleepSummary", { - unavailable: [...skip, ...plan.unavailable], - checkedAt: planCheckedAtFor(sleepPlan, reprobed, now), - }); - } - } catch (err) { - getEvent()?.addWarning( - `withings sleep: summary fetch failed for ${userId}: ${err}`, - ); - } - + // / avg SpO2 / sleep score) from the per-night summary read above. Database + // failures are hard sync failures. await prisma .$transaction( async (tx) => { @@ -656,6 +703,103 @@ export async function syncUserSleep( () => {}, ); + if (summaryFailure !== null) { + await recordWithingsSyncFailure(userId, summaryFailure, WITHINGS_LEG_SLEEP); + return imported; + } await recordSyncSuccess(userId, "withings", { leg: WITHINGS_LEG_SLEEP }); return imported; } + +const DAY_SECONDS = 24 * 60 * 60; + +/** Nights that ended this recently are read again on every run (re-scores). */ +const RECENT_NIGHT_SECONDS = 2 * DAY_SECONDS; + +/** + * Stage reads per run beyond the recent nights. A new connection, or a + * window the sleep-timeline backfill just cleared, fills in over a few runs + * instead of firing thirty requests at once; a full sync reads every night. + */ +export const SLEEP_STAGE_NIGHTS_PER_RUN = 8; + +/** One `action=get` read: a night's own span, at most 24 hours. */ +interface StageWindow { + startdate: number; + enddate: number; + /** The read covers the whole night, so its session may be swept. */ + wholeNight: boolean; +} + +/** + * The nights whose stage timeline this run reads, from the summary's night + * list: every night that ended, or that Withings changed (`modified`), in the + * last 48 hours, since Withings re-scores a night after the fact; then nights + * that hold no Withings stage row yet, + * up to `SLEEP_STAGE_NIGHTS_PER_RUN`. The missing nights rotate by the hour, + * so a night that never carries stages (a tracker without them) cannot keep + * the ones behind it waiting. + */ +async function stageWindowsToFetch( + userId: string, + summaries: readonly WithingsSleepSummary[], + now: Date, + opts: { all: boolean }, +): Promise { + const nights = new Map< + string, + { startdate: number; enddate: number; changed: number } + >(); + for (const s of summaries) { + if (!Number.isFinite(s.startdate) || !Number.isFinite(s.enddate)) continue; + if (s.enddate <= s.startdate) continue; + const modified = Number.isFinite(s.modified) ? s.modified! : 0; + const key = `${s.startdate}:${s.enddate}`; + nights.set(key, { + startdate: s.startdate, + enddate: s.enddate, + changed: Math.max(s.enddate, modified, nights.get(key)?.changed ?? 0), + }); + } + const all = [...nights.values()].sort((x, y) => y.enddate - x.enddate); + const toWindow = (n: { + startdate: number; + enddate: number; + }): StageWindow => ({ + startdate: n.startdate, + enddate: Math.min(n.enddate, n.startdate + DAY_SECONDS), + wholeNight: n.enddate - n.startdate <= DAY_SECONDS, + }); + if (opts.all || all.length === 0) return all.map(toWindow); + + const nowUnix = Math.floor(now.getTime() / 1000); + // Recent: ended, or was changed by Withings, within the last 48 hours. + const recent = all.filter((n) => n.changed >= nowUnix - RECENT_NIGHT_SECONDS); + const older = all.filter((n) => n.changed < nowUnix - RECENT_NIGHT_SECONDS); + if (older.length === 0) return recent.map(toWindow); + + // Segments are stamped at their END, so a night holds a stage row when one + // ends inside (startdate, enddate]. One read over the span of the older + // nights, on the `(user_id, type, measured_at)` index. + const first = Math.min(...older.map((n) => n.startdate)); + const last = Math.max(...older.map((n) => n.enddate)); + const stored = await prisma.measurement.findMany({ + where: { + userId, + source: "WITHINGS", + type: SLEEP_TYPE, + deletedAt: null, + measuredAt: { gt: new Date(first * 1000), lte: new Date(last * 1000) }, + }, + select: { measuredAt: true }, + }); + const endsAt = stored.map((row) => row.measuredAt.getTime() / 1000); + const missing = older + .filter((n) => !endsAt.some((t) => t > n.startdate && t <= n.enddate)) + .sort((x, y) => x.enddate - y.enddate); + const budget = Math.max(0, SLEEP_STAGE_NIGHTS_PER_RUN - recent.length); + if (missing.length <= budget) return [...recent, ...missing].map(toWindow); + const offset = Math.floor(nowUnix / 3600) % missing.length; + const rotated = [...missing.slice(offset), ...missing.slice(0, offset)]; + return [...recent, ...rotated.slice(0, budget)].map(toWindow); +} diff --git a/tests/integration/withings-sleep-sync.test.ts b/tests/integration/withings-sleep-sync.test.ts index 29134c408..de98eef94 100644 --- a/tests/integration/withings-sleep-sync.test.ts +++ b/tests/integration/withings-sleep-sync.test.ts @@ -51,26 +51,70 @@ afterEach(() => { vi.restoreAllMocks(); }); -function stubWithingsSleep(series: unknown[]) { +type Segment = { + startdate: number; + enddate: number; + state: number; + id: number; +}; + +/** + * Answer Sleep v2 the way Withings does: `getsummary` lists the nights, + * `get` returns the segments inside the span asked for, at most its first + * 24 hours. + */ +function stubWithingsSleep(series: Segment[]) { + const nights = new Map< + number, + { id: number; startdate: number; enddate: number } + >(); + for (const seg of series) { + const night = nights.get(seg.id); + nights.set(seg.id, { + id: seg.id, + startdate: Math.min(night?.startdate ?? seg.startdate, seg.startdate), + enddate: Math.max(night?.enddate ?? seg.enddate, seg.enddate), + }); + } vi.stubGlobal( "fetch", - vi.fn(async () => ({ - status: 200, - json: async () => ({ - status: 0, - body: { series }, - }), - })), + vi.fn(async (_url: string, init: { body: string }) => { + const params = new URLSearchParams(init.body); + if (params.get("action") === "getsummary") { + return { + status: 200, + json: async () => ({ + status: 0, + body: { series: [...nights.values()] }, + }), + }; + } + const from = Number(params.get("startdate")); + const to = Math.min(Number(params.get("enddate")), from + 24 * 3600); + return { + status: 200, + json: async () => ({ + status: 0, + body: { + series: series.filter( + (seg) => seg.startdate >= from && seg.enddate <= to, + ), + }, + }), + }; + }), ); } +/** Last night: it started ten hours ago, so a run reads it again for a re-score. */ +const base = Math.floor(Date.now() / 1000) - 10 * 3600; + describe("syncUserSleep — integration", () => { it("persists one Measurement row per stage segment for a single night", async () => { // Sleep session #99: 4 segments — CORE 60m, DEEP 30m, REM 30m, AWAKE 10m. // All segments share the same parent night but each carries a // distinct sleepStage, so the W17b/c composite (Migration 0055) // keeps every row. - const base = 1715000000; // 2024-05-06 13:33:20 UTC (arbitrary) stubWithingsSleep([ { startdate: base, enddate: base + 3600, state: 1, id: 99 }, { startdate: base + 3600, enddate: base + 5400, state: 2, id: 99 }, @@ -106,7 +150,6 @@ describe("syncUserSleep — integration", () => { }); it("re-syncing the same night updates rows in place (no duplicates)", async () => { - const base = 1715000000; // First sync: DEEP 30 minutes. stubWithingsSleep([ { startdate: base, enddate: base + 1800, state: 2, id: 100 }, From d02fba65096bd9dbfaaa00c199ffb1ba4478639d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 12:59:44 +0200 Subject: [PATCH 8/9] fix(withings): skip activity days Health Mate holds from another app Getactivity marks a day that came from outside Withings with brand 18 ("external"): an Apple Health or Google Fit day written into Health Mate. Imported under WITHINGS it was a second copy of data HealthLog reads from that app's own integration, and on a day it was the only Withings row it could outrank a native source lower on the ladder. Such days are now skipped at ingest and counted on the wide event. --- .../withings/__tests__/sync-activity.test.ts | 23 ++++++++++++++++ src/lib/withings/sync-activity.ts | 26 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/lib/withings/__tests__/sync-activity.test.ts b/src/lib/withings/__tests__/sync-activity.test.ts index 7155da2f8..f4df49f8a 100644 --- a/src/lib/withings/__tests__/sync-activity.test.ts +++ b/src/lib/withings/__tests__/sync-activity.test.ts @@ -101,6 +101,7 @@ interface FakeActivity { steps?: number; distance?: number; calories?: number; + brand?: number; } function installFetchMock(entries: FakeActivity[]) { @@ -414,6 +415,28 @@ describe("syncUserActivity — field mapping + idempotency", () => { expect(types).not.toContain("WALKING_RUNNING_DISTANCE"); }); + it("skips a day Health Mate holds from another app (brand 18)", async () => { + // An Apple Health or Google Fit day written into Health Mate comes back + // from Getactivity as brand 18 ("external"). It reaches HealthLog from + // its own integration; under WITHINGS it was a second copy. + installFetchMock([ + { date: "2026-05-12", steps: 8000, calories: 380, brand: 1 }, + { date: "2026-05-13", steps: 9100, calories: 410, brand: 18 }, + ]); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.measurement.createMany).mockResolvedValue({ + count: 2, + } as never); + + await syncUserActivity("user-1"); + + const rows = createManyRows(); + expect(rows).toHaveLength(2); + expect(rows.map((r) => r.value)).toEqual( + expect.not.arrayContaining([9100, 410]), + ); + }); + it("ingests a 0-step rest day rather than dropping it", async () => { // Withings reports 0 steps for a day a user spent in bed — that's // a valid datapoint, not a missing one. The mapper must not skip diff --git a/src/lib/withings/sync-activity.ts b/src/lib/withings/sync-activity.ts index 1a075fec6..bd6425162 100644 --- a/src/lib/withings/sync-activity.ts +++ b/src/lib/withings/sync-activity.ts @@ -82,6 +82,9 @@ const WITHINGS_MEASURE_URL = "https://wbsapi.withings.net/v2/measure"; */ const ACTIVITY_BACKFILL_DAYS = 30; +/** `brand` of a Getactivity day that came from outside Withings. */ +const WITHINGS_EXTERNAL_BRAND = 18; + /** * Withings `getactivity` response shape — one entry per calendar day. * Only the fields we consume are typed; the response carries 15+ more @@ -95,6 +98,13 @@ export interface WithingsActivityEntry { steps?: number; distance?: number; // metres calories?: number; // active kcal (NOT totalcalories) + /** + * Where the day came from: 1 for a Withings device or the Health Mate + * app's own tracker, 18 for "an external way" (Getactivity: "Value is 1 + * for Withings and 18 for external"), i.e. data another app wrote into + * Health Mate, such as an Apple Health or Google Fit day. + */ + brand?: number; // The fields below are not ingested today (see file header): elevation?: number; totalcalories?: number; @@ -357,8 +367,18 @@ export async function syncUserActivity( externalId: string; } const collected: PlannedRow[] = []; + let external = 0; for (const entry of entries) { if (!entry.date) continue; + // A day Health Mate holds from another app is that app's data, which + // reaches HealthLog from its own integration (Apple Health, Health + // Connect, Google Health). Imported under WITHINGS it was a second copy + // under the wrong name, and on a day that copy was the only Withings row + // it could outrank a native source further down the ladder. + if (entry.brand === WITHINGS_EXTERNAL_BRAND) { + external++; + continue; + } // Local noon of the date in the user's zone: the instant reads the same // calendar day there. Noon UTC read as the next day from UTC+13 on // (Auckland in summer, Tonga, Samoa), so every daily total landed a day @@ -380,6 +400,12 @@ export async function syncUserActivity( } } + if (external > 0) { + getEvent()?.addWarning( + `withings activity: skipped ${external} day(s) Health Mate holds from another app`, + ); + } + // A daily-activity total the app's own band calls impossible (a step count // past 200 000, a negative distance) is a provider artefact, not a day. Drop // it before the plan is read against the table, so it can neither insert a From c456a236a7c9e83adbb1e1236820d4c4a1df1765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Wed, 21 Oct 2026 13:07:34 +0200 Subject: [PATCH 9/9] fix(analytics): rank Health Connect and the hand-entered sources on the ladders A source a ladder does not name ranks 90, below every source it does. Five had no place anywhere: a weight typed as a reply to a Telegram reminder lost its day to a WHOOP strap's estimate, one an assistant logged over MCP lost to a Fitbit estimate, and a Health Connect day lost to a hand-typed count. Health Connect now sits right after Apple Health on the ladders for what it imports; EXTERNAL, MCP and TELEGRAM sit right after MANUAL, in the alphabetical order they already broke ties in, so MANUAL still outranks the person's own bridge. Relative to each other the sources the ladders already named keep their order, and saved ladders gain the new entries at their end as before. --- .../workouts/__tests__/batch-scope.test.ts | 12 +- .../__tests__/source-priority.test.ts | 45 +++--- src/lib/i18n/source-labels.ts | 5 +- .../__tests__/source-priority.test.ts | 80 ++++++++++- src/lib/validations/source-priority.ts | 69 +++++++++- .../source-ladder-new-sources.test.ts | 128 ++++++++++++++++++ 6 files changed, 307 insertions(+), 32 deletions(-) create mode 100644 tests/integration/source-ladder-new-sources.test.ts diff --git a/src/app/api/workouts/__tests__/batch-scope.test.ts b/src/app/api/workouts/__tests__/batch-scope.test.ts index 61a063093..091b0ad8f 100644 --- a/src/app/api/workouts/__tests__/batch-scope.test.ts +++ b/src/app/api/workouts/__tests__/batch-scope.test.ts @@ -246,8 +246,18 @@ describe("POST /api/workouts/batch — rate limit buckets", () => { describe("POST /api/workouts/batch — what a scoped caller writes", () => { it("attributes its rows to EXTERNAL", async () => { armToken([WORKOUTS_WRITE_SCOPE]); + // Two sessions of the day: EXTERNAL ranks on the workout ladder, so two + // rows of one session would land once. const res = await POST( - makeRequest({ workouts: [workout("s-1"), workout("s-2")] }), + makeRequest({ + workouts: [ + workout("s-1"), + workout("s-2", { + startedAt: "2026-09-20T18:00:00.000Z", + endedAt: "2026-09-20T18:45:00.000Z", + }), + ], + }), ); expect(res.status).toBe(200); const rows = insertedRows(); diff --git a/src/lib/analytics/__tests__/source-priority.test.ts b/src/lib/analytics/__tests__/source-priority.test.ts index 14abaf374..e7d5bb110 100644 --- a/src/lib/analytics/__tests__/source-priority.test.ts +++ b/src/lib/analytics/__tests__/source-priority.test.ts @@ -519,25 +519,23 @@ describe("pickCanonicalSourceRows — v1.4.25 W8c two-axis device-type picker", /** * v1.38.x — what the EXTERNAL source does to a contested day. * - * `EXTERNAL` is on no priority ladder, deliberately: the `steps` and `spo2` - * defaults already sit at `metricSourceLadder`'s `.max(8)`, so a ninth entry - * would come back from `GET /api/auth/me/source-priority`, be PUT back whole - * by the settings page, and 422 — breaking ladder saves for everyone. + * `EXTERNAL` was on no priority ladder at first, because the ladders were + * capped at eight entries and a ninth would have 422'd every settings save. + * The cap is the number of sources now, and since v1.43.11 `EXTERNAL` sits + * right after `MANUAL` on the ladders that rank `MANUAL`. * - * That omission is not inert. It decides a real behaviour, and one that - * CHANGED when ingested rows stopped being labelled `MANUAL`: before, a typed - * reading and a bridged one were the same source and both survived a day; now - * `MANUAL` ranks, `EXTERNAL` does not, and the bridged row leaves the - * canonical set for that day. Seven call sites consume this picker (daily - * totals, the health score, the doctor report, correlations, mood crosstabs, - * PR detection), so the effect is user-visible and is written up in the - * changelog. + * The behaviour this pins CHANGED when ingested rows stopped being labelled + * `MANUAL`: before, a typed reading and a bridged one were the same source + * and both survived a day; now `MANUAL` outranks `EXTERNAL`, and the bridged + * row leaves the canonical set for that day. Seven call sites consume this + * picker (daily totals, the health score, the doctor report, correlations, + * mood crosstabs, PR detection), so the effect is user-visible. * * Pinned here because it is emergent from a rank ladder rather than stated - * anywhere in code: put `EXTERNAL` on a ladder and the first case below goes - * red, which is the review anybody adding it deserves. + * anywhere in code: rank `EXTERNAL` above `MANUAL` and the first case below + * goes red, which is the review anybody doing that deserves. */ -describe("pickCanonicalSourceRows — the unranked EXTERNAL source", () => { +describe("pickCanonicalSourceRows — the EXTERNAL source", () => { const day = "2026-09-02"; function weightRow( @@ -573,10 +571,9 @@ describe("pickCanonicalSourceRows — the unranked EXTERNAL source", () => { expect(out.pickedByDay.get(day)).toBe("MANUAL"); }); - it("keeps a bridged-only day, since nothing on the ladder is present", () => { - // The `!picked` pass-through. Without this arm an account that ONLY ever - // pushes readings would go dark on every surface listed above, which - // would be a far worse regression than the one above is a change. + it("keeps a bridged-only day, picked from the ladder", () => { + // An account that ONLY ever pushes readings keeps every one of them: + // the bridge is the day's one source, and the ladder now names it. const out = pickCanonicalSourceRows( [ weightRow("EXTERNAL", 82.4, "06:30"), @@ -588,8 +585,7 @@ describe("pickCanonicalSourceRows — the unranked EXTERNAL source", () => { ); expect(out.canonicalRows).toHaveLength(2); expect(out.canonicalRows.every((r) => r.source === "EXTERNAL")).toBe(true); - // No ladder source won, so the day names no pick. - expect(out.pickedByDay.has(day)).toBe(false); + expect(out.pickedByDay.get(day)).toBe("EXTERNAL"); }); it("kept both when the bridged row was still labelled MANUAL", () => { @@ -607,10 +603,9 @@ describe("pickCanonicalSourceRows — the unranked EXTERNAL source", () => { expect(out.canonicalRows).toHaveLength(2); }); - it("lets a user who ranks EXTERNAL themselves have it win", () => { - // Not reachable from the settings UI today (the ladder cap keeps it off - // the page), but the picker resolves whatever ladder it is handed, and a - // stored ladder naming it must be honoured rather than ignored. + it("lets a user who ranks EXTERNAL above MANUAL have it win", () => { + // The picker resolves whatever ladder it is handed; a stored ladder that + // puts the bridge first is honoured. const out = pickCanonicalSourceRows( [ weightRow("EXTERNAL", 82.4, "06:30"), diff --git a/src/lib/i18n/source-labels.ts b/src/lib/i18n/source-labels.ts index a8ef4fb2e..57e3bc5f6 100644 --- a/src/lib/i18n/source-labels.ts +++ b/src/lib/i18n/source-labels.ts @@ -40,9 +40,8 @@ export const MEASUREMENT_SOURCE_SETTINGS_LABEL_KEYS: Record< MCP: "settings.sections.sources.sourceLabels.MCP", GOOGLE_HEALTH: "settings.sections.sources.sourceLabels.GOOGLE_HEALTH", STRAVA: "settings.sections.sources.sourceLabels.STRAVA", - // v1.38.x — readings pushed in under a `measurements:write` token. - // Never offered in the ladder (it is on no default priority list), but - // the map is keyed by the enum, so the entry is not optional. + // v1.38.x — readings pushed in under a `measurements:write` token. On the + // default ladders right after MANUAL since v1.43.11. EXTERNAL: "settings.sections.sources.sourceLabels.EXTERNAL", HEALTH_CONNECT: "settings.sections.sources.sourceLabels.HEALTH_CONNECT", }; diff --git a/src/lib/validations/__tests__/source-priority.test.ts b/src/lib/validations/__tests__/source-priority.test.ts index 9db9585f8..aa3b205e4 100644 --- a/src/lib/validations/__tests__/source-priority.test.ts +++ b/src/lib/validations/__tests__/source-priority.test.ts @@ -533,6 +533,84 @@ describe("parseSourcePriority — duplicate tolerance (v1.16.11)", () => { const ladder = resolved.metricPriority.weight; expect(new Set(ladder).size).toBe(ladder.length); expect(ladder.slice(0, 2)).toEqual(["MANUAL", "WITHINGS"]); - expect(ladder.length).toBeLessThanOrEqual(8); + // The schema caps a ladder at one entry per source. + expect(ladder.length).toBeLessThanOrEqual( + measurementSourceEnum.options.length, + ); + }); +}); + +describe("DEFAULT_SOURCE_PRIORITY — the sources that had no place (v1.43.11)", () => { + const HAND_ENTERED = ["EXTERNAL", "MCP", "TELEGRAM"] as const; + // What Health Connect imports, by ladder (`import/health-connect/mapper.ts`). + const HEALTH_CONNECT_LADDERS = [ + "steps", + "activeEnergy", + "walkingRunningDistance", + "bloodPressure", + "bodyFat", + "bodyTemperature", + "hrvRmssd", + "spo2", + "respiratoryRate", + "restingHeartRate", + "vo2Max", + "weight", + ] as const; + + it("ranks the hand-entered sources right after MANUAL wherever MANUAL ranks", () => { + for (const [key, ladder] of Object.entries(DEFAULT_SOURCE_PRIORITY)) { + const i = (ladder as string[]).indexOf("MANUAL"); + if (i === -1) continue; + expect(ladder.slice(i + 1, i + 1 + HAND_ENTERED.length), key).toEqual([ + ...HAND_ENTERED, + ]); + } + }); + + it("ranks Health Connect right after Apple Health on every ladder it imports for", () => { + for (const key of HEALTH_CONNECT_LADDERS) { + const ladder = DEFAULT_SOURCE_PRIORITY[key] as string[]; + expect(ladder[ladder.indexOf("APPLE_HEALTH") + 1], key).toBe( + "HEALTH_CONNECT", + ); + } + }); + + it("leaves the order of the sources the ladders already named alone", () => { + const added = new Set([...HAND_ENTERED, "HEALTH_CONNECT"]); + const without = (key: keyof typeof DEFAULT_SOURCE_PRIORITY) => + DEFAULT_SOURCE_PRIORITY[key].filter((s) => !added.has(s)); + expect(without("weight")).toEqual([ + "WITHINGS", + "APPLE_HEALTH", + "MANUAL", + "WHOOP", + "FITBIT", + "GOOGLE_HEALTH", + ]); + expect(without("bloodPressure")).toEqual([ + "WITHINGS", + "APPLE_HEALTH", + "MANUAL", + ]); + expect(without("spo2")).toEqual([ + "WITHINGS", + "WHOOP", + "FITBIT", + "GOOGLE_HEALTH", + "OURA", + "POLAR", + "APPLE_HEALTH", + "MANUAL", + ]); + }); + + it("fits every default ladder within the schema cap", () => { + for (const [key, ladder] of Object.entries(DEFAULT_SOURCE_PRIORITY)) { + expect(ladder.length, key).toBeLessThanOrEqual( + measurementSourceEnum.options.length, + ); + } }); }); diff --git a/src/lib/validations/source-priority.ts b/src/lib/validations/source-priority.ts index 6ad587dbd..da0bb5a2f 100644 --- a/src/lib/validations/source-priority.ts +++ b/src/lib/validations/source-priority.ts @@ -234,6 +234,16 @@ export type DeviceTypePriority = z.infer; * pulse-ox, Thermo). Apple Health is second-hand (HealthKit * receives the same reading via Withings' Health Mate iOS app). */ +// v1.43.11 — the sources that had no place on any ladder. A source a ladder +// does not name ranks 90, below everything it does, so a typed weight from a +// Telegram reminder lost its day to a WHOOP strap's estimate, and a Health +// Connect day lost to every other source. Health Connect is the Android +// counterpart of the HealthKit relay and sits right after APPLE_HEALTH on +// the ladders for what it imports; the readings a person enters or bridges +// themselves (EXTERNAL: their own hardware's uploader, MCP: an assistant +// writing on their word, TELEGRAM: a reply to a reminder) sit right after +// MANUAL, in the alphabetical order they already broke ties in. Relative to +// each other the existing sources keep their order. export const DEFAULT_SOURCE_PRIORITY: Required = { // v1.12.0 — Fitbit/Pixel rides below Apple Health for cumulative metrics: // HealthKit still aggregates the broadest device set, and a Fitbit-only @@ -253,6 +263,7 @@ export const DEFAULT_SOURCE_PRIORITY: Required = { // still survives. steps: [ "APPLE_HEALTH", + "HEALTH_CONNECT", "WITHINGS", "FITBIT", "GOOGLE_HEALTH", @@ -260,24 +271,35 @@ export const DEFAULT_SOURCE_PRIORITY: Required = { "POLAR", "STRAVA", "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", ], activeEnergy: [ "APPLE_HEALTH", + "HEALTH_CONNECT", "WITHINGS", "FITBIT", "GOOGLE_HEALTH", "OURA", "POLAR", "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", ], walkingRunningDistance: [ "APPLE_HEALTH", + "HEALTH_CONNECT", "WITHINGS", "FITBIT", "GOOGLE_HEALTH", "OURA", "POLAR", "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", ], flightsClimbed: [ "APPLE_HEALTH", @@ -285,6 +307,9 @@ export const DEFAULT_SOURCE_PRIORITY: Required = { "FITBIT", "GOOGLE_HEALTH", "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", ], // v1.11.0 — WHOOP leads the recovery-input ladders (sleep / HRV / RHR): // a worn-all-night strap has higher-resolution overnight sampling than @@ -319,18 +344,31 @@ export const DEFAULT_SOURCE_PRIORITY: Required = { "OURA", "POLAR", "APPLE_HEALTH", + "HEALTH_CONNECT", "WITHINGS", ], // A real scale beats a strap's body-measurement estimate for weight. weight: [ "WITHINGS", "APPLE_HEALTH", + "HEALTH_CONNECT", "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", "WHOOP", "FITBIT", "GOOGLE_HEALTH", ], - bloodPressure: ["WITHINGS", "APPLE_HEALTH", "MANUAL"], + bloodPressure: [ + "WITHINGS", + "APPLE_HEALTH", + "HEALTH_CONNECT", + "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", + ], // v1.17.0 — continuous optical-HR wearables (Fitbit / Polar / Oura) rank // above a hand-typed MANUAL pulse; the device-source order matches the // rhr / hrv ladders (FITBIT > OURA > POLAR). Withings BPM cuff + the @@ -343,12 +381,29 @@ export const DEFAULT_SOURCE_PRIORITY: Required = { "OURA", "POLAR", "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", + ], + bodyFat: [ + "WITHINGS", + "APPLE_HEALTH", + "HEALTH_CONNECT", + "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", + "FITBIT", + "GOOGLE_HEALTH", ], - bodyFat: ["WITHINGS", "APPLE_HEALTH", "MANUAL", "FITBIT", "GOOGLE_HEALTH"], bodyTemperature: [ "WITHINGS", "APPLE_HEALTH", + "HEALTH_CONNECT", "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", "FITBIT", "GOOGLE_HEALTH", ], @@ -363,15 +418,23 @@ export const DEFAULT_SOURCE_PRIORITY: Required = { "OURA", "POLAR", "APPLE_HEALTH", + "HEALTH_CONNECT", "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", ], vo2Max: [ "WITHINGS", "APPLE_HEALTH", + "HEALTH_CONNECT", "FITBIT", "GOOGLE_HEALTH", "OURA", "MANUAL", + "EXTERNAL", + "MCP", + "TELEGRAM", ], // v1.11.0 — new WHOOP-overlapping keys. ScanWatch dermal reading is the // primary skin-temperature sensor; WHOOP's strap is second, Fitbit third, @@ -386,6 +449,7 @@ export const DEFAULT_SOURCE_PRIORITY: Required = { "OURA", "POLAR", "APPLE_HEALTH", + "HEALTH_CONNECT", "WITHINGS", ], // v1.11.0 — native-vs-derived recovery. WHOOP's device-native Recovery @@ -414,6 +478,7 @@ export const DEFAULT_SOURCE_PRIORITY: Required = { "OURA", "POLAR", "APPLE_HEALTH", + "HEALTH_CONNECT", ], }; diff --git a/tests/integration/source-ladder-new-sources.test.ts b/tests/integration/source-ladder-new-sources.test.ts new file mode 100644 index 000000000..0819ebff8 --- /dev/null +++ b/tests/integration/source-ladder-new-sources.test.ts @@ -0,0 +1,128 @@ +/** + * The sources that had no place on a ladder, against real Postgres through + * the SQL collapse every 90-day aggregate reads (`canonicalMeasurementsFrom`). + * + * A source a ladder does not name ranks 90, below everything it does name: + * a weight typed as a reply to a Telegram reminder lost its day to a WHOOP + * strap's estimate, and a Health Connect day lost to a hand-typed one. The + * hand-entered sources now rank with MANUAL, Health Connect with Apple + * Health, and a day between two sources that were on the ladder before + * resolves exactly as it did. + */ +import { beforeAll, describe, expect, it, vi } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import type { + MeasurementSource, + MeasurementType, +} from "@/generated/prisma/client"; +import { + buildSourceRankCase, + canonicalMeasurementsFrom, +} from "@/lib/analytics/source-rank-sql"; + +vi.mock("@/lib/db-compat", () => ({ + ensureDbCompatibility: vi.fn().mockResolvedValue(undefined), +})); + +const prisma = getPrismaClient(); +const USER = "ladder-new-sources"; +const DAY = 86_400_000; +const today = Math.floor(Date.now() / DAY) * DAY; + +/** One reading per (type, source) on day `n` days back, at 08:00 / 09:00 UTC. */ +function reading( + type: MeasurementType, + source: MeasurementSource, + n: number, + value: number, + hour = 8, +) { + return { + userId: USER, + type, + source, + value, + unit: type === "WEIGHT" ? "kg" : "count", + measuredAt: new Date(today - n * DAY + hour * 3_600_000), + }; +} + +async function canonicalSources(): Promise> { + const rank = buildSourceRankCase(null, '"type"', '"source"'); + const rows = await prisma.$queryRawUnsafe< + Array<{ type: string; day: Date; source: string }> + >( + `SELECT DISTINCT m."type"::text AS type, + date_trunc('day', m."measured_at") AS day, + m."source"::text AS source + FROM ${canonicalMeasurementsFrom(rank, "90 days")} + ORDER BY 1, 2, 3`, + USER, + ); + const out = new Map(); + for (const r of rows) { + const key = `${r.type}@${r.day.toISOString().slice(0, 10)}`; + out.set(key, [...(out.get(key) ?? []), r.source]); + } + return out; +} + +const dayKey = (n: number) => + new Date(today - n * DAY).toISOString().slice(0, 10); + +beforeAll(async () => { + await truncateAllTables(prisma); + await prisma.user.create({ + data: { id: USER, username: USER, timezone: "UTC" }, + }); + await prisma.measurement.createMany({ + data: [ + // Day 1: a strap's weight estimate and a typed reply to a reminder. + reading("WEIGHT", "WHOOP", 1, 84), + reading("WEIGHT", "TELEGRAM", 1, 82.5, 9), + // Day 2: an assistant logging on the person's word beside a Fitbit + // scale-less estimate. + reading("WEIGHT", "FITBIT", 2, 85), + reading("WEIGHT", "MCP", 2, 82.4, 9), + // Day 3: a hand-typed weight and the person's own bridged scale. + reading("WEIGHT", "MANUAL", 3, 82.3), + reading("WEIGHT", "EXTERNAL", 3, 82.6, 9), + // Day 4: two sources the ladder always named: unchanged. + reading("WEIGHT", "APPLE_HEALTH", 4, 82.2), + reading("WEIGHT", "WITHINGS", 4, 82.1, 9), + // Day 1: Health Connect's daily steps beside a hand-typed count. + reading("ACTIVITY_STEPS", "HEALTH_CONNECT", 1, 9000), + reading("ACTIVITY_STEPS", "MANUAL", 1, 5000, 9), + // Day 2: Apple Health still outranks Health Connect. + reading("ACTIVITY_STEPS", "APPLE_HEALTH", 2, 9100), + reading("ACTIVITY_STEPS", "HEALTH_CONNECT", 2, 8900, 9), + ], + }); +}, 120_000); + +describe("source ladders for the sources that had no place", () => { + it("ranks a reading the person entered through Telegram or MCP above a device estimate", async () => { + const picked = await canonicalSources(); + expect(picked.get(`WEIGHT@${dayKey(1)}`)).toEqual(["TELEGRAM"]); + expect(picked.get(`WEIGHT@${dayKey(2)}`)).toEqual(["MCP"]); + }); + + it("keeps MANUAL above the person's own bridge", async () => { + const picked = await canonicalSources(); + expect(picked.get(`WEIGHT@${dayKey(3)}`)).toEqual(["MANUAL"]); + }); + + it("ranks Health Connect right after Apple Health", async () => { + const picked = await canonicalSources(); + expect(picked.get(`ACTIVITY_STEPS@${dayKey(1)}`)).toEqual([ + "HEALTH_CONNECT", + ]); + expect(picked.get(`ACTIVITY_STEPS@${dayKey(2)}`)).toEqual(["APPLE_HEALTH"]); + }); + + it("resolves a day between two long-ranked sources exactly as before", async () => { + const picked = await canonicalSources(); + expect(picked.get(`WEIGHT@${dayKey(4)}`)).toEqual(["WITHINGS"]); + }); +});