From bc12633569f322f7bfd9830955f7bc42750bef90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 14 Jun 2026 23:16:09 +0200 Subject: [PATCH 1/4] fix(sleep): reconstruct an ordered WHOOP hypnogram timeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WHOOP's v2 API returns only per-stage sleep duration totals — no onset timestamps, no hypnogram endpoint. The mapper stamped all five stage totals on the one sleep-END instant, so the hypnogram reconstructed every stage as a span touching the night's right edge: bars stacked right-aligned with no clock times whenever WHOOP was the sole source. Lay the asleep stages back-to-back from sleep onset in a fixed physiological order (AWAKE lead, then CORE → DEEP → REM), emitting one timed row per segment with measuredAt at that segment's end and a distinct indexed externalId. IN_BED stays a single envelope row over the whole window. The existing reader and renderer are untouched — they now see distinct per-segment instants and place each block at its own time. The order is synthesised, not measured, so every reconstructed row is flagged and the night DTO carries a per-session `reconstructed` boolean (GET /api/sleep/night, regenerated OpenAPI). The client renders the hypnogram but labels it an approximate layout and never recomputes; real-series sources (Apple Health, Withings, Fitbit) stay false. --- docs/api/openapi.yaml | 12 +++ src/app/api/sleep/night/route.ts | 9 ++ .../analytics/__tests__/sleep-night.test.ts | 42 +++++++++ src/lib/analytics/sleep-night.ts | 29 ++++++ src/lib/openapi/routes/measurements.ts | 5 + src/lib/whoop/__tests__/client.test.ts | 67 +++++++++++-- src/lib/whoop/client.ts | 93 +++++++++++++++---- src/lib/whoop/sync-sleep.ts | 19 +++- 8 files changed, 246 insertions(+), 30 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index bf628854e..e1fd8c8cc 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -8982,6 +8982,11 @@ components: type: integer minimum: 0 maximum: 9007199254740991 + reconstructed: + type: boolean + description: True when the source has no per-stage onset timestamps and the server synthesised a contiguous timeline in + a fixed physiological order (WHOOP). The client renders the hypnogram but labels it an approximate + layout and never recomputes. False for real-series sources (Apple Health / Withings / Fitbit). stages: type: object propertyNames: @@ -9047,6 +9052,7 @@ components: - inBedMinutes - awakeMinutes - awakenings + - reconstructed - stages - segments additionalProperties: false @@ -9102,6 +9108,11 @@ components: type: integer minimum: 0 maximum: 9007199254740991 + reconstructed: + type: boolean + description: True when the source has no per-stage onset timestamps and the server synthesised a contiguous timeline in + a fixed physiological order (WHOOP). The client renders the hypnogram but labels it an approximate + layout and never recomputes. False for real-series sources (Apple Health / Withings / Fitbit). stages: type: object propertyNames: @@ -9167,6 +9178,7 @@ components: - inBedMinutes - awakeMinutes - awakenings + - reconstructed - stages - segments additionalProperties: false diff --git a/src/app/api/sleep/night/route.ts b/src/app/api/sleep/night/route.ts index a930d3a3d..29d8874f0 100644 --- a/src/app/api/sleep/night/route.ts +++ b/src/app/api/sleep/night/route.ts @@ -12,6 +12,11 @@ * priority ladder — so two sources' timelines never overlay. A view over the * existing per-stage `SLEEP_DURATION` rows; no schema, no new type. * + * `reconstructed` (per session) is true when the winning source has no + * per-stage onset timestamps and the server synthesised a contiguous timeline + * (WHOOP) — the client renders the hypnogram but labels it an approximate + * layout. A real-series source (Apple Health / Withings / Fitbit) is false. + * * `date` omitted → the most recent night in the trailing-year window. * * Auth: cookie session OR Bearer token (`requireAuth`). Soft-delete-filtered. @@ -64,6 +69,10 @@ function serializeSession(s: SleepSession) { inBedMinutes: s.inBedMinutes === null ? null : Math.round(s.inBedMinutes), awakeMinutes: s.awakeMinutes === null ? null : Math.round(s.awakeMinutes), awakenings: s.awakenings, + // iOS #18 — `reconstructed` is true when the source has no per-stage onset + // timestamps and the server synthesised a contiguous timeline (WHOOP). iOS + // labels the hypnogram as an approximate layout; it never recomputes. + reconstructed: s.reconstructed, // iOS #18 — round the per-stage map too; the underlying totals sum // second-precision segments and otherwise serialise as e.g. 88.4999. stages: Object.fromEntries( diff --git a/src/lib/analytics/__tests__/sleep-night.test.ts b/src/lib/analytics/__tests__/sleep-night.test.ts index 17b32973c..00221cea0 100644 --- a/src/lib/analytics/__tests__/sleep-night.test.ts +++ b/src/lib/analytics/__tests__/sleep-night.test.ts @@ -380,6 +380,48 @@ describe("reconstructSleepSessions", () => { expect(sessions[0].asleepMinutes).toBe(420); }); + it("flags a WHOOP-won night as reconstructed (synthetic stage order) with clock-time-distinct segments", () => { + // The ingest mapper reconstructs an ordered, contiguous WHOOP timeline. + // The reconstructed segments carry distinct measuredAt instants, so the + // hypnogram lays them at distinct clock times (no shared right edge). + const rows: SleepStageRow[] = [ + srcRow("2026-06-04T01:00:00.000Z", "CORE", 240, "WHOOP"), // 21:00→01:00 + srcRow("2026-06-04T03:00:00.000Z", "DEEP", 120, "WHOOP"), // 01:00→03:00 + srcRow("2026-06-04T05:00:00.000Z", "REM", 120, "WHOOP"), // 03:00→05:00 + ]; + const sessions = reconstructSleepSessions(rows, "UTC"); + expect(sessions).toHaveLength(1); + const s = sessions[0]; + expect(s.source).toBe("WHOOP"); + expect(s.reconstructed).toBe(true); + // No two segments end on the same instant — not stacked on the right edge. + const ends = s.segments.map((seg) => seg.end.getTime()); + expect(new Set(ends).size).toBe(ends.length); + }); + + it("does NOT flag an Apple-Health night as reconstructed (measured timeline)", () => { + const rows: SleepStageRow[] = [ + srcRow("2026-06-04T01:00:00.000Z", "CORE", 240, "APPLE_HEALTH"), + srcRow("2026-06-04T03:00:00.000Z", "DEEP", 90, "APPLE_HEALTH"), + srcRow("2026-06-04T04:30:00.000Z", "REM", 90, "APPLE_HEALTH"), + ]; + const sessions = reconstructSleepSessions(rows, "UTC"); + expect(sessions).toHaveLength(1); + expect(sessions[0].source).toBe("APPLE_HEALTH"); + expect(sessions[0].reconstructed).toBe(false); + }); + + it("does NOT flag a Withings night as reconstructed (real per-segment series)", () => { + const rows: SleepStageRow[] = [ + srcRow("2026-06-04T01:00:00.000Z", "CORE", 240, "WITHINGS"), + srcRow("2026-06-04T03:00:00.000Z", "DEEP", 90, "WITHINGS"), + ]; + const sessions = reconstructSleepSessions(rows, "UTC"); + expect(sessions).toHaveLength(1); + expect(sessions[0].source).toBe("WITHINGS"); + expect(sessions[0].reconstructed).toBe(false); + }); + it("drops the bare ASLEEP aggregate from segments + stages when granular exists (HIGH H1, v1.11.5)", () => { // Apple-Health shape: granular CORE/DEEP/REM PLUS the unspecified ASLEEP // aggregate covering the same period, plus IN_BED. The hypnogram must not diff --git a/src/lib/analytics/sleep-night.ts b/src/lib/analytics/sleep-night.ts index 590361df7..88024e6c1 100644 --- a/src/lib/analytics/sleep-night.ts +++ b/src/lib/analytics/sleep-night.ts @@ -281,6 +281,19 @@ export interface SleepNight { /** Sentinel for rows that carry no source — collapses to one bucket. */ const NO_SOURCE = "__none__"; +/** + * Sources whose per-segment stage ORDER is synthesised, not measured. WHOOP's + * v2 API exposes only per-stage duration totals (no onset timestamps), so the + * ingest mapper reconstructs a contiguous timeline in a fixed physiological + * order (`whoop/client.ts` `mapSleep`). A session won by such a source carries + * `reconstructed: true` so the UI labels it as an approximate layout. Apple + * Health / Withings / Fitbit all store a real per-segment series and stay off + * this set. + */ +const RECONSTRUCTED_TIMELINE_SOURCES: ReadonlySet = new Set([ + "WHOOP", +]); + /** * Separator between the source and device-type parts of a writer key. * `` cannot appear in a `MeasurementSource` enum literal or a @@ -615,6 +628,17 @@ export interface SleepSession { awakenings: number; /** The canonical source's segments, sorted by start, for the hypnogram. */ segments: SleepSegment[]; + /** + * `true` when the winning writer's stage ORDER is synthesised rather than + * measured. WHOOP's v2 API returns only per-stage duration totals — no onset + * timestamps — so the server reconstructs a contiguous timeline in a fixed + * physiological order (`whoop/client.ts` `mapSleep`). The segment positions + * are then approximate, not measured, and the UI / iOS must label the night + * as an approximate layout (never present it as measured stage timing). + * Sources that store a real per-segment series (Apple Health, Withings, + * Fitbit) stay `false`. + */ + reconstructed: boolean; } /** Resolve a stage row to its absolute span (start = end − duration). */ @@ -760,6 +784,11 @@ export function reconstructSleepSessions( stages, awakenings: countAwakenings(segments), segments, + // The winning writer's stage order is synthetic when its source + // reconstructs the timeline (WHOOP). The UI labels such a night as an + // approximate layout; a real-series source (Apple/Withings/Fitbit) is + // measured. + reconstructed: RECONSTRUCTED_TIMELINE_SOURCES.has(canonicalSource), }); } return sessions.sort((a, b) => a.start.getTime() - b.start.getTime()); diff --git a/src/lib/openapi/routes/measurements.ts b/src/lib/openapi/routes/measurements.ts index 15de359a7..9dbb92663 100644 --- a/src/lib/openapi/routes/measurements.ts +++ b/src/lib/openapi/routes/measurements.ts @@ -200,6 +200,11 @@ const sleepSessionResource = z.object({ inBedMinutes: z.number().int().nonnegative().nullable(), awakeMinutes: z.number().int().nonnegative().nullable(), awakenings: z.number().int().nonnegative(), + reconstructed: z + .boolean() + .describe( + "True when the source has no per-stage onset timestamps and the server synthesised a contiguous timeline in a fixed physiological order (WHOOP). The client renders the hypnogram but labels it an approximate layout and never recomputes. False for real-series sources (Apple Health / Withings / Fitbit).", + ), stages: z.record(sleepStageEnum, z.number().int().nonnegative()), segments: z.array(sleepSegmentResource), }); diff --git a/src/lib/whoop/__tests__/client.test.ts b/src/lib/whoop/__tests__/client.test.ts index 74d3f12bb..385ce9008 100644 --- a/src/lib/whoop/__tests__/client.test.ts +++ b/src/lib/whoop/__tests__/client.test.ts @@ -320,15 +320,64 @@ describe("mapSleep", () => { it("maps per-stage SLEEP_DURATION rows (ms→min) with sleepStage", () => { const rows = mapSleep(base); const dur = rows.filter((r) => r.type === "SLEEP_DURATION"); - const byStage = Object.fromEntries(dur.map((r) => [r.sleepStage, r.value])); - expect(byStage.CORE).toBe(240); // light → CORE - expect(byStage.DEEP).toBe(90); // slow-wave → DEEP - expect(byStage.REM).toBe(120); - expect(byStage.AWAKE).toBe(30); - expect(byStage.IN_BED).toBe(480); + // One asleep/awake row per reconstructed segment + one IN_BED envelope. The + // per-stage TOTALS are unchanged (the reconstruction only reorders timing). + const minutesByStage: Record = {}; + for (const r of dur) { + minutesByStage[r.sleepStage!] = + (minutesByStage[r.sleepStage!] ?? 0) + r.value; + } + expect(minutesByStage.CORE).toBe(240); // light → CORE + expect(minutesByStage.DEEP).toBe(90); // slow-wave → DEEP + expect(minutesByStage.REM).toBe(120); + expect(minutesByStage.AWAKE).toBe(30); + expect(minutesByStage.IN_BED).toBe(480); expect(dur.every((r) => r.unit === "minutes")).toBe(true); }); + it("reconstructs an ordered, contiguous, non-overlapping per-segment timeline from sleep ONSET", () => { + const segs = mapSleep(base) + .filter((r) => r.type === "SLEEP_DURATION" && r.sleepStage !== "IN_BED") + .map((r) => ({ + stage: r.sleepStage, + end: r.measuredAt.getTime(), + start: r.measuredAt.getTime() - r.value * 60_000, + reconstructed: r.reconstructed, + })) + .sort((a, b) => a.start - b.start); + + // AWAKE (lead) → CORE → DEEP → REM, laid back-to-back. + expect(segs.map((s) => s.stage)).toEqual(["AWAKE", "CORE", "DEEP", "REM"]); + // First segment starts at sleep ONSET (s.start), not the END edge. + expect(segs[0].start).toBe(new Date(base.start).getTime()); + // Contiguous + non-overlapping: each segment's start == previous end. + for (let i = 1; i < segs.length; i++) { + expect(segs[i].start).toBe(segs[i - 1].end); + } + // Distinct END instants — no stacking on the night's right edge. + expect(new Set(segs.map((s) => s.end)).size).toBe(segs.length); + // Every reconstructed row is flagged so the UI labels the night approximate. + expect(segs.every((s) => s.reconstructed === true)).toBe(true); + }); + + it("emits IN_BED as a single un-reconstructed envelope row over [start, end]", () => { + const inBed = mapSleep(base).filter((r) => r.sleepStage === "IN_BED"); + expect(inBed).toHaveLength(1); + const r = inBed[0]!; + expect(r.value).toBe(480); + // measuredAt = sleep END → segmentOf resolves the span back to the window. + expect(r.measuredAt.toISOString()).toBe(base.end); + expect(r.reconstructed).toBeUndefined(); + }); + + it("gives each reconstructed segment a distinct indexed externalId", () => { + const ids = mapSleep(base) + .filter((r) => r.reconstructed) + .map((r) => r.externalId); + expect(ids.every((id) => id?.startsWith("sleep-uuid:seg:"))).toBe(true); + expect(new Set(ids).size).toBe(ids.length); + }); + it("sums SLEEP_NEED components ms→min", () => { const need = mapSleep(base).find((r) => r.type === "SLEEP_NEED"); expect(need?.value).toBe(490); // 450 + 30 + 10 + 0 @@ -344,9 +393,9 @@ describe("mapSleep", () => { expect(byType.RESPIRATORY_RATE?.unit).toBe("breaths/min"); }); - it("uses sleep.end as measuredAt", () => { - const row = mapSleep(base)[0]!; - expect(row.measuredAt.toISOString()).toBe("2026-06-01T07:00:00.000Z"); + it("stamps the non-segment scores on sleep.end", () => { + const need = mapSleep(base).find((r) => r.type === "SLEEP_NEED")!; + expect(need.measuredAt.toISOString()).toBe("2026-06-01T07:00:00.000Z"); }); it("emits nothing for an unscored sleep", () => { diff --git a/src/lib/whoop/client.ts b/src/lib/whoop/client.ts index a75ad1e06..19c30ddc7 100644 --- a/src/lib/whoop/client.ts +++ b/src/lib/whoop/client.ts @@ -505,20 +505,24 @@ export interface MappedMeasurement { fieldTag: string; /** Per-stage sleep rows carry the SleepStage; everything else omits it. */ sleepStage?: "CORE" | "DEEP" | "REM" | "AWAKE" | "IN_BED"; + /** + * Full externalId for rows the mapper must key itself rather than letting + * the sync layer build `:`. The reconstructed sleep + * segments (one row per synthetic span) need an index in the key so the + * several rows of one stage stay distinct under + * `userId_type_source_externalId`. When set the sync layer uses it verbatim. + */ + externalId?: string; + /** + * `true` on the synthesised per-segment sleep rows whose ORDER is invented: + * WHOOP's v2 API returns only per-stage duration totals (no onset + * timestamps), so the timeline is reconstructed in a fixed physiological + * order. The night is flagged so the UI labels it as an approximate layout + * and never presents it as measured stage timing. + */ + reconstructed?: boolean; } -/** WHOOP sleep stage → HealthLog SleepStage (light→CORE, slow-wave→DEEP). */ -const SLEEP_STAGE_MAP: Record< - string, - { stage: MappedMeasurement["sleepStage"]; fieldTag: string } -> = { - total_light_sleep_time_milli: { stage: "CORE", fieldTag: "sleep_core" }, - total_slow_wave_sleep_time_milli: { stage: "DEEP", fieldTag: "sleep_deep" }, - total_rem_sleep_time_milli: { stage: "REM", fieldTag: "sleep_rem" }, - total_awake_time_milli: { stage: "AWAKE", fieldTag: "sleep_awake" }, - total_in_bed_time_milli: { stage: "IN_BED", fieldTag: "sleep_in_bed" }, -}; - function round2(n: number): number { return parseFloat(n.toFixed(2)); } @@ -589,16 +593,71 @@ export function mapSleep(s: WhoopSleep): MappedMeasurement[] { const out: MappedMeasurement[] = []; const stages = s.score.stage_summary; - for (const [key, mapping] of Object.entries(SLEEP_STAGE_MAP)) { - const ms = stages[key as keyof typeof stages]; - if (typeof ms !== "number") continue; + + // WHOOP v2 returns only per-stage DURATION totals (no per-stage onset + // timestamps, no hypnogram endpoint). Stamping every stage total on the one + // sleep-END instant makes the hypnogram reconstruct five spans that all touch + // the night's right edge (the stacked-bar artefact). Since the API never + // carries an order, we RECONSTRUCT an ordered, contiguous timeline: lay the + // asleep stages back-to-back from sleep ONSET in a fixed physiological order, + // emitting one timed row per segment with `measuredAt = that segment's END`. + // The ORDER is synthetic, so every reconstructed row is flagged + // `reconstructed: true` and the night DTO advertises it — the UI labels it as + // an approximate layout and never presents it as measured stage timing. + // + // IN_BED stays a single envelope row spanning the whole [start, end] window: + // the in-bed reader wants the union envelope, not a placed segment. AWAKE is + // laid as a leading segment from onset so the asleep stages still partition + // the remaining window. + const onset = new Date(s.start).getTime(); + // Order the laid-out stages: a brief AWAKE settling-in block, then the asleep + // stages CORE → DEEP → REM. Each carries its own duration; the cursor walks + // forward so the spans are contiguous and non-overlapping. + const TIMELINE_ORDER: ReadonlyArray<{ + key: keyof typeof stages; + stage: NonNullable; + tag: string; + }> = [ + { key: "total_awake_time_milli", stage: "AWAKE", tag: "sleep_awake" }, + { key: "total_light_sleep_time_milli", stage: "CORE", tag: "sleep_core" }, + { key: "total_slow_wave_sleep_time_milli", stage: "DEEP", tag: "sleep_deep" }, + { key: "total_rem_sleep_time_milli", stage: "REM", tag: "sleep_rem" }, + ]; + let cursor = onset; + let segIndex = 0; + for (const { key, stage, tag } of TIMELINE_ORDER) { + const ms = stages[key]; + if (typeof ms !== "number" || ms <= 0) continue; + const segEnd = cursor + ms; out.push({ type: "SLEEP_DURATION", value: round2(ms * MS_TO_MIN), unit: "minutes", + measuredAt: new Date(segEnd), + // The sync layer keys reconstructed rows by this externalId verbatim; the + // index keeps the several rows of one night distinct under + // userId_type_source_externalId. + fieldTag: tag, + externalId: `${s.id}:seg:${tag}:${segIndex}`, + sleepStage: stage, + reconstructed: true, + }); + cursor = segEnd; + segIndex += 1; + } + + // IN_BED — single envelope row over the whole sleep window. `measuredAt` is + // the sleep END so `segmentOf` resolves the span back to [start, end]; the + // in-bed union-envelope reader consumes the full window. + const inBedMs = stages.total_in_bed_time_milli; + if (typeof inBedMs === "number" && inBedMs > 0) { + out.push({ + type: "SLEEP_DURATION", + value: round2(inBedMs * MS_TO_MIN), + unit: "minutes", measuredAt, - fieldTag: mapping.fieldTag, - sleepStage: mapping.stage, + fieldTag: "sleep_in_bed", + sleepStage: "IN_BED", }); } diff --git a/src/lib/whoop/sync-sleep.ts b/src/lib/whoop/sync-sleep.ts index 234052ad0..a97173178 100644 --- a/src/lib/whoop/sync-sleep.ts +++ b/src/lib/whoop/sync-sleep.ts @@ -4,8 +4,14 @@ * `mapSleep` (per-stage SLEEP_DURATION rows, SLEEP_NEED, the SLEEP_* * percentages, RESPIRATORY_RATE), and upserts as `source = WHOOP`. * - * Per-stage rows carry the `sleepStage` axis so the five stage rows for one - * night stay distinct under the dedup key. externalId = `:`. + * Sleep rows carry the `sleepStage` axis so the night's rows stay distinct + * under the dedup key. WHOOP exposes only per-stage DURATION totals (no onset + * timestamps), so `mapSleep` RECONSTRUCTS an ordered, contiguous per-segment + * timeline (CORE/DEEP/REM/AWAKE laid back-to-back from sleep onset) and flags + * those rows `reconstructed`; each reconstructed segment carries its own + * indexed externalId `:seg::`. IN_BED stays a single + * envelope row keyed `:sleep_in_bed`. The non-segment scores keep + * the `:` shape. */ import { fetchSleeps, fetchSleepById, mapSleep } from "./client"; import { @@ -53,7 +59,10 @@ export async function syncUserSleep( value: m.value, unit: m.unit, measuredAt: m.measuredAt, - externalId: `${s.id}:${m.fieldTag}`, + // Reconstructed sleep segments carry their own indexed externalId so + // the several rows of one stage stay distinct; everything else keeps + // the `:` shape. + externalId: m.externalId ?? `${s.id}:${m.fieldTag}`, sleepStage: m.sleepStage ?? null, }); } @@ -92,7 +101,9 @@ export async function syncWhoopSleepById( value: m.value, unit: m.unit, measuredAt: m.measuredAt, - externalId: `${record.id}:${m.fieldTag}`, + // Reconstructed sleep segments carry their own indexed externalId + // (see syncUserSleep). + externalId: m.externalId ?? `${record.id}:${m.fieldTag}`, sleepStage: m.sleepStage ?? null, }); } From bd467d0c4daf4daf724207d308d08986ce25548b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 14 Jun 2026 23:16:16 +0200 Subject: [PATCH 2/4] fix(sleep): emit Fitbit sleep as per-segment rows, not a stage collapse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Google sleep payload carries a real per-stage segment series, but the mapper summed each stage's segments into one row stamped on the stage's last end. The hypnogram then reconstructed all of a stage's time as a single block ending at the night's right edge — the same stacked-bar artefact as WHOOP, despite the source carrying true timing. Emit one row per segment with measuredAt at that segment's end and an indexed externalId so the several segments of one stage stay distinct under the dedup key. The timeline is measured (real onsets), so these rows are not flagged reconstructed. --- src/lib/fitbit/__tests__/client.test.ts | 40 ++++++++++++--------- src/lib/fitbit/client.ts | 46 +++++++++++-------------- src/lib/fitbit/sync-sleep.ts | 19 +++++----- 3 files changed, 55 insertions(+), 50 deletions(-) diff --git a/src/lib/fitbit/__tests__/client.test.ts b/src/lib/fitbit/__tests__/client.test.ts index 38b54aeeb..9edb6b4e8 100644 --- a/src/lib/fitbit/__tests__/client.test.ts +++ b/src/lib/fitbit/__tests__/client.test.ts @@ -526,7 +526,7 @@ describe("sleep-stage mapping", () => { expect(mapFitbitSleepStage(42)).toBeNull(); }); - it("maps a session into per-stage SLEEP_DURATION rows with measuredAt = stage END", () => { + it("maps a session into per-SEGMENT SLEEP_DURATION rows with measuredAt = segment END (real timeline)", () => { const session = { sleep: { startTime: "2026-05-10T22:00:00.000Z", @@ -556,23 +556,31 @@ describe("sleep-stage mapping", () => { }, }; const out = mapSleepSession(session); - const byStage = Object.fromEntries(out.map((m) => [m.sleepStage, m])); - // CORE = the two light segments summed (30 + 45 = 75 min). - expect(byStage.CORE!.value).toBe(75); - expect(byStage.CORE!.type).toBe("SLEEP_DURATION"); - expect(byStage.CORE!.unit).toBe("minutes"); - // measuredAt is the LATEST end for that stage (the second light segment). - expect(byStage.CORE!.measuredAt.toISOString()).toBe( + // One row PER segment (no collapse) — 4 segments → 4 rows. + expect(out).toHaveLength(4); + expect(out.every((m) => m.type === "SLEEP_DURATION")).toBe(true); + expect(out.every((m) => m.unit === "minutes")).toBe(true); + + // Each row carries its OWN end instant + duration — a real clock-time + // timeline, not a stage collapse. + const core = out.filter((m) => m.sleepStage === "CORE"); + expect(core.map((m) => m.value)).toEqual([30, 45]); + expect(core.map((m) => m.measuredAt.toISOString())).toEqual([ + "2026-05-10T22:30:00.000Z", "2026-05-11T00:15:00.000Z", - ); - expect(byStage.DEEP!.value).toBe(60); - expect(byStage.REM!.value).toBe(45); + ]); + const deep = out.find((m) => m.sleepStage === "DEEP")!; + expect(deep.value).toBe(60); + expect(deep.measuredAt.toISOString()).toBe("2026-05-10T23:30:00.000Z"); + const rem = out.find((m) => m.sleepStage === "REM")!; + expect(rem.value).toBe(45); - // externalId field-tag is session-anchored so a re-score overwrites in place. - expect(byStage.DEEP!.fieldTag).toBe( - "2026-05-11T06:00:00.000Z:sleep_deep", - ); + // Every fieldTag is distinct (session anchor + stage + segment index) so + // the several segments of one stage never collide under the dedup key. + const tags = out.map((m) => m.fieldTag); + expect(new Set(tags).size).toBe(tags.length); + expect(tags).toContain("2026-05-11T06:00:00.000Z:sleep_deep:1"); }); it("anchors the session externalId on sleep.interval.end_time for an INTERVAL-shaped point", () => { @@ -592,7 +600,7 @@ describe("sleep-stage mapping", () => { }, }); const deep = out.find((m) => m.sleepStage === "DEEP"); - expect(deep!.fieldTag).toBe("2026-05-11T06:00:00.000Z:sleep_deep"); + expect(deep!.fieldTag).toBe("2026-05-11T06:00:00.000Z:sleep_deep:0"); }); it("yields nothing for a session with no parseable segments", () => { diff --git a/src/lib/fitbit/client.ts b/src/lib/fitbit/client.ts index 25c217140..e31ee2450 100644 --- a/src/lib/fitbit/client.ts +++ b/src/lib/fitbit/client.ts @@ -1244,10 +1244,18 @@ function sleepSessionAnchor(point: FitbitDataPoint): string { } /** - * Map one Google sleep session into per-stage `SLEEP_DURATION` rows. Each - * stage's segments are summed into one row carrying the stage's `SleepStage`, - * `measuredAt = the latest segment END for that stage`, and a stage-scoped - * fieldTag so the (up to six) stage rows for one night stay distinct under the + * Map one Google sleep session into per-SEGMENT `SLEEP_DURATION` rows. The + * Google sleep payload carries a real per-stage segment series (each with its + * own start/end), so one row is emitted PER SEGMENT — `measuredAt = that + * segment's END` — rather than collapsing a stage's segments onto a single + * lastEnd instant. The collapsed shape stamped every stage total on its last + * end, so the hypnogram reconstructed all of a stage's time as one block ending + * at the night's right edge (the stacked-bar artefact); per-segment rows lay + * each block at its true clock time. The timeline is MEASURED (real onsets), so + * these rows are NOT flagged reconstructed — unlike WHOOP, which has no onsets. + * + * Each segment carries a stage-scoped, INDEXED fieldTag so the several segments + * of one stage stay distinct under the `(userId, type, source, externalId)` * dedup key. Unknown stage labels are skipped; a session with no parseable * segment yields nothing. */ @@ -1257,38 +1265,24 @@ export function mapSleepSession( const segments = readSleepSegments(point); if (segments.length === 0) return []; - // Accumulate per-stage minutes + track the latest end instant for each stage. - const perStage = new Map< - FitbitSleepStage, - { minutes: number; lastEnd: Date } - >(); + const anchor = sleepSessionAnchor(point); + const out: FitbitMappedMeasurement[] = []; + let segIndex = 0; for (const seg of segments) { const stage = mapFitbitSleepStage(seg.stage); if (!stage) continue; const mins = minutesBetween(seg.startTime, seg.endTime); - if (mins === null) continue; + if (mins === null || !(mins > 0)) continue; const end = new Date(seg.endTime as string); - const prev = perStage.get(stage); - if (prev) { - prev.minutes += mins; - if (end > prev.lastEnd) prev.lastEnd = end; - } else { - perStage.set(stage, { minutes: mins, lastEnd: end }); - } - } - - const anchor = sleepSessionAnchor(point); - const out: FitbitMappedMeasurement[] = []; - for (const [stage, agg] of perStage) { - if (!(agg.minutes > 0)) continue; out.push({ type: "SLEEP_DURATION", - value: round2(agg.minutes), + value: round2(mins), unit: "minutes", - measuredAt: agg.lastEnd, - fieldTag: `${anchor}:sleep_${stage.toLowerCase()}`, + measuredAt: end, + fieldTag: `${anchor}:sleep_${stage.toLowerCase()}:${segIndex}`, sleepStage: stage, }); + segIndex += 1; } return out; } diff --git a/src/lib/fitbit/sync-sleep.ts b/src/lib/fitbit/sync-sleep.ts index 953cd40f9..d7ac50f1f 100644 --- a/src/lib/fitbit/sync-sleep.ts +++ b/src/lib/fitbit/sync-sleep.ts @@ -2,16 +2,19 @@ * Fitbit / Google Health sleep-bundle sync (v1.12.0, W5). * * Reads sleep sessions from the `sleep.readonly` Restricted bundle and maps - * each into per-stage `SLEEP_DURATION` rows via `mapSleepSession` (minutes per - * stage; `measuredAt = the stage's END instant; harmonised onto the shared - * `SleepStage` enum IN_BED/AWAKE/ASLEEP/REM/CORE/DEEP that the night-total + - * hypnogram readers already consume for WHOOP / Apple). Upserts as + * each into per-SEGMENT `SLEEP_DURATION` rows via `mapSleepSession` (one row + * per stage segment; `measuredAt = that segment's END instant; harmonised onto + * the shared `SleepStage` enum IN_BED/AWAKE/ASLEEP/REM/CORE/DEEP that the + * night-total + hypnogram readers already consume for WHOOP / Apple). Google + * carries a real per-segment series, so the rows lay each block at its true + * clock time (a MEASURED timeline, not reconstructed). Upserts as * `source = FITBIT`. * - * Per-stage rows carry the `sleepStage` axis so the (up to six) stage rows for - * one night stay distinct under the `(userId, type, source, externalId)` dedup - * key. externalId = `:sleep_` — a re-scored night - * overwrites in place. A 24 h overlap covers Google's after-the-fact re-score. + * Each segment carries the `sleepStage` axis and an indexed fieldTag so the + * several segments of one stage stay distinct under the + * `(userId, type, source, externalId)` dedup key. externalId = + * `:sleep_:` — a re-scored night overwrites in place. + * A 24 h overlap covers Google's after-the-fact re-score. * * A per-data-class 403 soft-skips the resource — the sleep bundle is granted * independently of activity / metrics in the Google consent flow. From 7ad0b9c4f434d296a79f0dd3f50bd73062175e88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 14 Jun 2026 23:16:23 +0200 Subject: [PATCH 3/4] fix(sleep): stamp Withings sleep segments at the END, not the START MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Withings sleep sync stamped each segment's measuredAt with its startdate, but every reader treats measuredAt as the segment END and resolves the span as start = end − duration. Each night was therefore rendered one segment-length earlier than reality. Stamp the enddate so the span lands at its true clock time. --- src/lib/withings/__tests__/sync-sleep.test.ts | 7 +++++-- src/lib/withings/sync-sleep.ts | 17 ++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/lib/withings/__tests__/sync-sleep.test.ts b/src/lib/withings/__tests__/sync-sleep.test.ts index 28757ebd0..6ae04f3a1 100644 --- a/src/lib/withings/__tests__/sync-sleep.test.ts +++ b/src/lib/withings/__tests__/sync-sleep.test.ts @@ -194,7 +194,7 @@ describe("syncUserSleep — segment writes + idempotency", () => { expect(firstDuration.data.unit).toBe("minutes"); }); - it("converts startdate (unix seconds) to a Date in measuredAt", async () => { + it("stamps measuredAt at the segment END (enddate, unix seconds → Date)", async () => { installFetchMock([ { startdate: 1715000000, enddate: 1715003600, state: 2, id: 1 }, ]); @@ -205,7 +205,10 @@ describe("syncUserSleep — segment writes + idempotency", () => { const arg = vi.mocked(prisma.measurement.create).mock.calls[0][0] as { data: { measuredAt: Date }; }; - expect(arg.data.measuredAt.getTime()).toBe(1715000000 * 1000); + // measuredAt is the segment END — every reader treats it as the END and + // resolves the span as start = end − duration. Stamping the START shifted + // the night one segment-length earlier. + expect(arg.data.measuredAt.getTime()).toBe(1715003600 * 1000); }); it("skips state 4 (synthetic marker) without throwing", async () => { diff --git a/src/lib/withings/sync-sleep.ts b/src/lib/withings/sync-sleep.ts index 231d94f6b..4b92080d2 100644 --- a/src/lib/withings/sync-sleep.ts +++ b/src/lib/withings/sync-sleep.ts @@ -26,10 +26,13 @@ * when Withings re-emits the night with adjusted segment boundaries. * * Date semantics: each segment's `measuredAt` is the segment's - * `startdate` (unix seconds → `Date`). The duration in minutes lands - * in `value`. The HealthLog analytics aggregator already groups - * stage rows under their parent night via the per-night `dayKey` - * helper, so the segment-level timestamp is the canonical sort key. + * `enddate` (unix seconds → `Date`). Every reader treats `measuredAt` + * as the segment END and resolves the span as `start = end − duration` + * (`sleep-night.ts` `segmentOf`); stamping the START shifted the whole + * night one segment-length earlier. The duration in minutes lands in + * `value`. The HealthLog analytics aggregator groups stage rows under + * their parent night via the per-night `dayKey` helper, so the + * segment-level END timestamp is the canonical sort key. */ import type { MeasurementType, SleepStage } from "@/generated/prisma/client"; @@ -258,7 +261,11 @@ export async function syncUserSleep( segmentIndex++; continue; } - const measuredAt = new Date(segment.startdate * 1000); + // `measuredAt` is the segment END (`enddate`). Every reader treats + // `measuredAt` as the END instant — `segmentOf` resolves the span as + // `start = measuredAt − value·60_000` (sleep-night.ts). Stamping the START + // here shifted the whole night one segment-length earlier; END is correct. + const measuredAt = new Date(segment.enddate * 1000); const durationSec = Math.max(0, segment.enddate - segment.startdate); // Canonical SLEEP_DURATION unit is minutes (Apple Health alignment, // v1.4.23 schema note). Round so re-sync doesn't flip values on From ddfed17fa31c95010a468eacbe6dc85bc53439c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sun, 14 Jun 2026 23:16:33 +0200 Subject: [PATCH 4/4] feat(sleep): backfill existing WHOOP + Withings sleep rows after the stamp fix The WHOOP timeline reconstruction and the Withings END-stamp change the stored measuredAt (and, for WHOOP, the externalId), so existing rows keep the old broken shape until re-synced. An in-place update collides with the (userId, type, measuredAt, source, sleepStage) unique index, so the backfill deletes the affected SLEEP_DURATION rows for the source and re-syncs from scratch; the re-sync re-folds the rollup tier in its tail. Add a boot-time pg-boss one-shot per connection, modelled on the WHOOP / Fitbit backfills: discovery enqueues one job per (user, provider) where sleep_timeline_backfill_at is null, the per-user pass deletes + re-syncs and stamps the marker, and the pass is idempotent across reboots. The queue is registered in allQueues, wired to a handler, and enqueued at boot. --- .../migration.sql | 20 +++ prisma/schema.prisma | 14 ++ .../__tests__/sleep-timeline-backfill.test.ts | 149 +++++++++++++++ .../__tests__/sleep-timeline-queues.test.ts | 38 ++++ src/lib/jobs/reminder-worker.ts | 63 +++++++ src/lib/jobs/sleep-timeline-backfill.ts | 169 ++++++++++++++++++ 6 files changed, 453 insertions(+) create mode 100644 prisma/migrations/0172_v1171_sleep_timeline_backfill/migration.sql create mode 100644 src/lib/jobs/__tests__/sleep-timeline-backfill.test.ts create mode 100644 src/lib/jobs/__tests__/sleep-timeline-queues.test.ts create mode 100644 src/lib/jobs/sleep-timeline-backfill.ts diff --git a/prisma/migrations/0172_v1171_sleep_timeline_backfill/migration.sql b/prisma/migrations/0172_v1171_sleep_timeline_backfill/migration.sql new file mode 100644 index 000000000..b5ea8e75b --- /dev/null +++ b/prisma/migrations/0172_v1171_sleep_timeline_backfill/migration.sql @@ -0,0 +1,20 @@ +-- v1.17.1 — one-shot sleep-timeline backfill markers. +-- +-- WHOOP's v2 API returns only per-stage sleep DURATION totals (no onset +-- timestamps), and the pre-fix mapper stamped all five stage totals on the one +-- sleep-END instant. The hypnogram then reconstructed every stage as a span +-- touching the night's right edge — the stacked-bar artefact with no clock +-- times. The fix reconstructs an ordered, contiguous per-segment timeline. +-- +-- Withings stamped each sleep segment with its START while every reader treats +-- `measured_at` as the segment END, shifting each night one segment-length +-- earlier. The fix stamps the END. +-- +-- Both fixes change the stored row shape / instant, so existing rows must be +-- re-synced. The boot-time backfill (per connection) deletes the affected +-- SLEEP_DURATION rows, re-syncs with the corrected mapper, re-folds the rollup +-- tier, and stamps `sleep_timeline_backfill_at` so the discovery query drops +-- the connection. Null = backfill not yet run; the marker makes the one-shot +-- idempotent across reboots. +ALTER TABLE "whoop_connections" ADD COLUMN "sleep_timeline_backfill_at" TIMESTAMP(3); +ALTER TABLE "withings_connections" ADD COLUMN "sleep_timeline_backfill_at" TIMESTAMP(3); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index ba56aa388..5558e34b5 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -2150,6 +2150,13 @@ model WithingsConnection { /// has never re-authed since v1.4.25 — the UI surfaces a banner /// inviting them to reconnect. scope String? @map("scope") + /// v1.17.1 — one-shot marker for the sleep-stamp backfill. Pre-fix Withings + /// sleep rows were stamped with the segment START; the reader treats + /// `measuredAt` as the END, shifting every night one segment-length earlier. + /// The boot-time backfill deletes the affected WITHINGS / SLEEP_DURATION rows + /// and re-syncs with the corrected END stamp, then sets this so the discovery + /// query drops the connection. Null = backfill not yet run. + sleepTimelineBackfillAt DateTime? @map("sleep_timeline_backfill_at") createdAt DateTime @default(now()) @map("created_at") updatedAt DateTime @updatedAt @map("updated_at") @@ -2226,6 +2233,13 @@ model WhoopConnection { /// Set once the self-converging history backfill walks every collection to /// completion. Null = backfill still in flight (or never started). backfillCompletedAt DateTime? @map("backfill_completed_at") + /// v1.17.1 — one-shot marker for the sleep-timeline backfill. Pre-fix WHOOP + /// sleep rows stamped all five stage totals on the one sleep-END instant, so + /// the hypnogram stacked them on the night's right edge. The boot-time + /// backfill deletes the legacy summary rows and re-syncs the reconstructed + /// per-segment timeline, then sets this so the discovery query drops the + /// connection. Null = backfill not yet run. + sleepTimelineBackfillAt DateTime? @map("sleep_timeline_backfill_at") /// Max heart rate from the WHOOP body-measurement profile — a profile /// constant, not a time series, so it lives here rather than as a /// Measurement row. diff --git a/src/lib/jobs/__tests__/sleep-timeline-backfill.test.ts b/src/lib/jobs/__tests__/sleep-timeline-backfill.test.ts new file mode 100644 index 000000000..710f923b9 --- /dev/null +++ b/src/lib/jobs/__tests__/sleep-timeline-backfill.test.ts @@ -0,0 +1,149 @@ +/** + * v1.17.1 — sleep-timeline backfill self-convergence tests (mocked). + * - discovery only matches connections whose sleep rows predate the fix + * (`sleepTimelineBackfillAt IS NULL`), for both WHOOP and Withings; + * - a completed pass DELETES the source's SLEEP_DURATION rows, re-syncs, and + * stamps `sleepTimelineBackfillAt` so the next discovery drops the account + * (idempotent across reboots); + * - the discovery enqueue is singleton-keyed per (provider, user). + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { prismaMock, bossSend, syncUserWhoop, syncWithingsSleep } = vi.hoisted( + () => ({ + prismaMock: { + whoopConnection: { findMany: vi.fn(), update: vi.fn() }, + withingsConnection: { findMany: vi.fn(), update: vi.fn() }, + measurement: { deleteMany: vi.fn() }, + }, + bossSend: vi.fn(), + syncUserWhoop: vi.fn(), + syncWithingsSleep: vi.fn(), + }), +); + +vi.mock("@/lib/db", () => ({ prisma: prismaMock })); + +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: () => ({ send: bossSend }), +})); + +vi.mock("@/lib/whoop/sync", () => ({ + syncUserWhoop: (...a: unknown[]) => syncUserWhoop(...a), +})); + +vi.mock("@/lib/withings/sync-sleep", () => ({ + syncUserSleep: (...a: unknown[]) => syncWithingsSleep(...a), +})); + +vi.mock("@/lib/logging/context", () => ({ + annotate: () => {}, + getEvent: () => null, +})); + +import { + enqueueBootTimeSleepTimelineBackfill, + runSleepTimelineBackfillForUser, +} from "../sleep-timeline-backfill"; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("enqueueBootTimeSleepTimelineBackfill — discovery", () => { + it("queries un-backfilled WHOOP + Withings connections and enqueues one job per (provider, user)", async () => { + prismaMock.whoopConnection.findMany.mockResolvedValue([{ userId: "w1" }]); + prismaMock.withingsConnection.findMany.mockResolvedValue([ + { userId: "v1" }, + ]); + bossSend.mockResolvedValue("job-id"); + + const result = await enqueueBootTimeSleepTimelineBackfill(); + + expect(prismaMock.whoopConnection.findMany).toHaveBeenCalledWith( + expect.objectContaining({ where: { sleepTimelineBackfillAt: null } }), + ); + expect(prismaMock.withingsConnection.findMany).toHaveBeenCalledWith( + expect.objectContaining({ where: { sleepTimelineBackfillAt: null } }), + ); + expect(result.enqueued).toBe(2); + expect(bossSend).toHaveBeenCalledWith( + "sleep-timeline-backfill", + expect.objectContaining({ userId: "w1", provider: "WHOOP" }), + expect.objectContaining({ + singletonKey: "sleep-timeline-backfill|WHOOP|w1", + }), + ); + expect(bossSend).toHaveBeenCalledWith( + "sleep-timeline-backfill", + expect.objectContaining({ userId: "v1", provider: "WITHINGS" }), + expect.objectContaining({ + singletonKey: "sleep-timeline-backfill|WITHINGS|v1", + }), + ); + }); + + it("self-converges: no un-backfilled connections → nothing enqueued", async () => { + prismaMock.whoopConnection.findMany.mockResolvedValue([]); + prismaMock.withingsConnection.findMany.mockResolvedValue([]); + + const result = await enqueueBootTimeSleepTimelineBackfill(); + + expect(result.enqueued).toBe(0); + expect(bossSend).not.toHaveBeenCalled(); + }); + + it("never throws — surfaces a discovery error through the result value", async () => { + prismaMock.whoopConnection.findMany.mockRejectedValue(new Error("db down")); + prismaMock.withingsConnection.findMany.mockResolvedValue([]); + + const result = await enqueueBootTimeSleepTimelineBackfill(); + + expect(result.error).toBe("db down"); + expect(result.enqueued).toBe(0); + }); +}); + +describe("runSleepTimelineBackfillForUser", () => { + it("WHOOP: deletes the source's sleep rows, full-syncs, and stamps the marker", async () => { + prismaMock.measurement.deleteMany.mockResolvedValue({ count: 5 }); + syncUserWhoop.mockResolvedValue(42); + prismaMock.whoopConnection.update.mockResolvedValue({}); + + const { deleted, imported } = await runSleepTimelineBackfillForUser( + "w1", + "WHOOP", + ); + + expect(deleted).toBe(5); + expect(imported).toBe(42); + expect(prismaMock.measurement.deleteMany).toHaveBeenCalledWith({ + where: { userId: "w1", type: "SLEEP_DURATION", source: "WHOOP" }, + }); + expect(syncUserWhoop).toHaveBeenCalledWith("w1", { fullSync: true }); + const updateArg = prismaMock.whoopConnection.update.mock.calls[0]![0]; + expect(updateArg.where).toEqual({ userId: "w1" }); + expect(updateArg.data.sleepTimelineBackfillAt).toBeInstanceOf(Date); + }); + + it("WITHINGS: deletes the source's sleep rows, re-syncs, and stamps the marker", async () => { + prismaMock.measurement.deleteMany.mockResolvedValue({ count: 3 }); + syncWithingsSleep.mockResolvedValue(7); + prismaMock.withingsConnection.update.mockResolvedValue({}); + + const { deleted, imported } = await runSleepTimelineBackfillForUser( + "v1", + "WITHINGS", + ); + + expect(deleted).toBe(3); + expect(imported).toBe(7); + expect(prismaMock.measurement.deleteMany).toHaveBeenCalledWith({ + where: { userId: "v1", type: "SLEEP_DURATION", source: "WITHINGS" }, + }); + expect(syncWithingsSleep).toHaveBeenCalledWith("v1", { fullSync: true }); + const updateArg = prismaMock.withingsConnection.update.mock.calls[0]![0]; + expect(updateArg.where).toEqual({ userId: "v1" }); + expect(updateArg.data.sleepTimelineBackfillAt).toBeInstanceOf(Date); + }); +}); diff --git a/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts b/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts new file mode 100644 index 000000000..2cbccc291 --- /dev/null +++ b/src/lib/jobs/__tests__/sleep-timeline-queues.test.ts @@ -0,0 +1,38 @@ +/** + * v1.17.1 — pg-boss queue registration for the one-shot sleep-timeline backfill. + * + * The reminder-worker imports heavy infrastructure, so we read the source as + * text and assert the queue constant is declared, registered in `allQueues`, + * wired to a `boss.work` handler, and enqueued at boot — the fast guard that + * catches the v1.4.37 dead-queue class (a queue declared but never registered, + * which silently never drains). + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +const REMINDER_WORKER_PATH = join(__dirname, "..", "reminder-worker.ts"); +const source = readFileSync(REMINDER_WORKER_PATH, "utf8"); + +describe("reminder-worker — sleep-timeline backfill queue", () => { + it("registers the queue in allQueues (v1.4.37 dead-queue guard)", () => { + const block = /const allQueues = \[([\s\S]*?)\];/.exec(source); + expect(block).not.toBeNull(); + expect(block![1]!).toContain("SLEEP_TIMELINE_BACKFILL_QUEUE"); + }); + + it("wires a boss.work handler to the per-user runner", () => { + expect(source).toMatch( + /boss\.work[\s\S]{0,260}SLEEP_TIMELINE_BACKFILL_QUEUE[\s\S]{0,260}runSleepTimelineBackfillForUser/, + ); + }); + + it("wires the boot discovery", () => { + expect(source).toMatch(/enqueueBootTimeSleepTimelineBackfill\(\)/); + }); + + it("imports the backfill exports", () => { + expect(source).toMatch(/from\s*["']@\/lib\/jobs\/sleep-timeline-backfill["']/); + }); +}); diff --git a/src/lib/jobs/reminder-worker.ts b/src/lib/jobs/reminder-worker.ts index 57aa681f0..056fbf574 100644 --- a/src/lib/jobs/reminder-worker.ts +++ b/src/lib/jobs/reminder-worker.ts @@ -21,6 +21,13 @@ import { enqueueBootTimeFitbitBackfill, type FitbitBackfillPayload, } from "@/lib/jobs/fitbit-backfill"; +import { + SLEEP_TIMELINE_BACKFILL_QUEUE, + SLEEP_TIMELINE_BACKFILL_CONCURRENCY, + runSleepTimelineBackfillForUser, + enqueueBootTimeSleepTimelineBackfill, + type SleepTimelineBackfillPayload, +} from "@/lib/jobs/sleep-timeline-backfill"; import { reportWorkerError } from "@/lib/jobs/report-worker-error"; import { markWorkerStarted, recordError } from "@/lib/jobs/worker-status"; import { setGlobalBoss } from "@/lib/jobs/boss-instance"; @@ -629,6 +636,13 @@ export async function startReminderWorker() { FITBIT_SYNC_QUEUE, FITBIT_BACKFILL_QUEUE, FITBIT_OAUTH_STATE_CLEANUP_QUEUE, + // v1.17.1 — one-shot sleep-timeline backfill for WHOOP + Withings. + // Discovery enqueues one job per connection whose sleep rows predate the + // stamp/shape fix; the pass deletes the affected SLEEP_DURATION rows and + // re-syncs. Idempotent across reboots. The queue MUST be registered here or + // pg-boss never provisions it and the boot enqueue silently never drains + // (the v1.4.37 dead-queue class). + SLEEP_TIMELINE_BACKFILL_QUEUE, // v1.17.0 — Nightscout CGM poll sync. Poll-only (no webhook, no OAuth, no // backfill queue — the hourly window walks the recent SGV set). The queue // MUST be registered here or pg-boss never provisions it and the schedule @@ -1056,6 +1070,27 @@ export async function startReminderWorker() { { localConcurrency: 1 }, handleFitbitOAuthStateCleanup, ); + // v1.17.1 — one-shot sleep-timeline backfill. The boot enqueue below sends + // one job per (user, provider) whose sleep rows predate the stamp/shape fix; + // this handler deletes the affected rows, re-syncs, and stamps the marker so + // the discovery query drops the connection. + await boss.work( + SLEEP_TIMELINE_BACKFILL_QUEUE, + { localConcurrency: SLEEP_TIMELINE_BACKFILL_CONCURRENCY }, + async (jobs) => { + for (const job of jobs) { + const { userId, provider } = job.data; + const { deleted, imported } = await runSleepTimelineBackfillForUser( + userId, + provider, + ); + workerLog( + "info", + `[sleep-timeline-backfill] user=${userId} provider=${provider} deleted=${deleted} imported=${imported}`, + ); + } + }, + ); // v1.17.0 — Nightscout CGM poll-cohort sync. The hourly cron tick (no // `userId`) walks every configured instance; one user's unreachable host is // warned, not fatal. @@ -1882,6 +1917,34 @@ export async function startReminderWorker() { ); } + // v1.17.1 — fire-and-forget boot discovery for the one-shot sleep-timeline + // backfill. Finds every WHOOP + Withings connection whose sleep rows predate + // the stamp/shape fix and enqueues one job per (user, provider). Idempotent + // across reboots: a completed pass stamps `sleepTimelineBackfillAt`, dropping + // the connection from the discovery set. Errors come back through the + // helper's result value — the worker boot never fails because of a miss. + try { + const { enqueued, skipped, error } = + await enqueueBootTimeSleepTimelineBackfill(); + if (error) { + workerLog( + "error", + `[sleep-timeline-backfill] boot discovery failed: ${error}`, + ); + } else { + workerLog( + "info", + `[sleep-timeline-backfill] boot discovery: enqueued=${enqueued} skipped=${skipped}`, + ); + } + } catch (err) { + workerLog( + "error", + "[sleep-timeline-backfill] boot discovery threw an unexpected error", + err, + ); + } + // v1.7.0 — fire-and-forget boot discovery for the daily-mean // consolidation pass. Finds every user holding live per-sample // high-frequency mean-type rows and enqueues one job per account. diff --git a/src/lib/jobs/sleep-timeline-backfill.ts b/src/lib/jobs/sleep-timeline-backfill.ts new file mode 100644 index 000000000..d3f49f154 --- /dev/null +++ b/src/lib/jobs/sleep-timeline-backfill.ts @@ -0,0 +1,169 @@ +/** + * v1.17.1 — one-shot sleep-timeline backfill for WHOOP + Withings. + * + * Two ingest-side stamp/shape fixes changed the stored sleep rows: + * + * - WHOOP: the old mapper stamped all five per-stage DURATION totals on the + * one sleep-END instant (WHOOP v2 exposes no onset timestamps), so the + * hypnogram reconstructed every stage as a span touching the night's right + * edge — stacked, no clock times. The fix reconstructs an ordered, + * contiguous per-segment timeline with distinct `measuredAt` per segment and + * NEW indexed externalIds (`:seg::`). + * - Withings: each segment was stamped with its START while every reader + * treats `measuredAt` as the END, shifting each night one segment-length + * earlier. The fix stamps the END. + * + * Both change the row's `measuredAt` (and, for WHOOP, the externalId), and the + * unique index `(userId, type, measuredAt, source, sleepStage)` makes an + * in-place UPDATE collision-prone. So the per-connection pass DELETES the + * affected `SLEEP_DURATION` rows for that source, RE-SYNCS with the corrected + * mapper (which re-folds the rollup tier in its tail), and stamps + * `sleepTimelineBackfillAt` so the discovery query drops the connection. + * + * Idempotent across reboots: the discovery predicate is + * `sleep_timeline_backfill_at IS NULL`, and a completed pass stamps the marker. + * A reboot mid-pass re-runs delete + re-sync from scratch — the re-sync upserts + * are key-stable, so the result converges. + * + * Modelled on `whoop-backfill.ts` / `fitbit-backfill.ts`. The queue name MUST + * be registered in `allQueues` in `src/lib/jobs/reminder-worker.ts` or pg-boss + * never provisions it and the boot enqueue silently never drains (the v1.4.37 + * dead-queue class). + */ +import { prisma } from "@/lib/db"; +import { annotate } from "@/lib/logging/context"; +import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { syncUserWhoop } from "@/lib/whoop/sync"; +import { syncUserSleep as syncWithingsSleep } from "@/lib/withings/sync-sleep"; + +export const SLEEP_TIMELINE_BACKFILL_QUEUE = "sleep-timeline-backfill"; + +/** + * Serial concurrency — a pass deletes a source's sleep rows then walks a + * re-sync that is rate-bounded by the upstream API cap. Concurrency-1 keeps it + * from crowding the request pool, matching the other backfill queues. + */ +export const SLEEP_TIMELINE_BACKFILL_CONCURRENCY = 1; + +export type SleepTimelineProvider = "WHOOP" | "WITHINGS"; + +export interface SleepTimelineBackfillPayload { + userId: string; + provider: SleepTimelineProvider; + enqueuedAt: string; +} + +/** + * Delete the legacy `SLEEP_DURATION` rows for one source, then re-sync with the + * corrected mapper. WHOOP runs a full-history sync (the segment externalIds are + * new, so the deleted summary rows are not re-created); Withings re-syncs its + * default trailing window with the corrected END stamp. The re-sync re-folds + * the rollup tier in its own tail. + */ +export async function runSleepTimelineBackfillForUser( + userId: string, + provider: SleepTimelineProvider, +): Promise<{ deleted: number; imported: number }> { + const { count: deleted } = await prisma.measurement.deleteMany({ + where: { userId, type: "SLEEP_DURATION", source: provider }, + }); + + let imported = 0; + if (provider === "WHOOP") { + imported = await syncUserWhoop(userId, { fullSync: true }); + await prisma.whoopConnection.update({ + where: { userId }, + data: { sleepTimelineBackfillAt: new Date() }, + }); + } else { + imported = await syncWithingsSleep(userId, { fullSync: true }); + await prisma.withingsConnection.update({ + where: { userId }, + data: { sleepTimelineBackfillAt: new Date() }, + }); + } + + annotate({ + action: { + name: "sleep.timeline.backfill.complete", + details: { provider, deleted, imported }, + }, + }); + return { deleted, imported }; +} + +/** + * Boot-time discovery. Finds every WHOOP + Withings connection not yet + * backfilled (`sleep_timeline_backfill_at IS NULL`) and enqueues one job per + * (user, provider). Idempotent across reboots: a completed pass stamps the + * marker, dropping the connection from the discovery set. pg-boss + * `singletonKey` coalesces duplicate sends so a fast restart while a job is + * queued doesn't double up. + * + * Best-effort: errors are returned through the result value so worker boot + * never fails because of a backfill miss. + */ +export async function enqueueBootTimeSleepTimelineBackfill(): Promise<{ + enqueued: number; + skipped: number; + error: string | null; +}> { + const boss = getGlobalBoss(); + if (!boss) { + return { enqueued: 0, skipped: 0, error: null }; + } + + try { + const [whoop, withings] = await Promise.all([ + prisma.whoopConnection.findMany({ + where: { sleepTimelineBackfillAt: null }, + select: { userId: true }, + }), + prisma.withingsConnection.findMany({ + where: { sleepTimelineBackfillAt: null }, + select: { userId: true }, + }), + ]); + + const targets: Array<{ userId: string; provider: SleepTimelineProvider }> = + [ + ...whoop.map((c) => ({ userId: c.userId, provider: "WHOOP" as const })), + ...withings.map((c) => ({ + userId: c.userId, + provider: "WITHINGS" as const, + })), + ]; + + if (targets.length === 0) { + return { enqueued: 0, skipped: 0, error: null }; + } + + let enqueued = 0; + let skipped = 0; + for (const { userId, provider } of targets) { + const payload: SleepTimelineBackfillPayload = { + userId, + provider, + enqueuedAt: new Date().toISOString(), + }; + const jobId = await boss.send(SLEEP_TIMELINE_BACKFILL_QUEUE, payload, { + retryLimit: 3, + retryDelay: 60, + retryBackoff: true, + singletonKey: `sleep-timeline-backfill|${provider}|${userId}`, + }); + if (jobId) { + enqueued += 1; + } else { + skipped += 1; + } + } + return { enqueued, skipped, error: null }; + } catch (err) { + return { + enqueued: 0, + skipped: 0, + error: err instanceof Error ? err.message : String(err), + }; + } +}