From 9c63d3ce204365f694ff6a5e041925dd7a4d6311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 11 Jul 2026 01:54:12 +0200 Subject: [PATCH] =?UTF-8?q?chore(release):=20v1.28.25=20=E2=80=94=20every?= =?UTF-8?q?=20integration=20held=20to=20the=20same=20standard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Platform-wide hardening: the failure classes found live this week hunted down across every integration. Fitbit carried the same tombstone wedge fixed for Google — probe now matches soft-deleted rows and the update resurrects them; the comment describing a partial unique index that never existed is corrected. Sleep segment ids are stable across re-scoring for Withings (session+segment-start), WHOOP/Polar (one segment per stage, no positional index) and Oura (run-start keyed), each with a session-scoped sweep that soft-deletes rows a re-score orphaned — so legacy and affected nights heal as they are re-read. One platform rule for deleted rows: source-owned readings resurrect on re-import (Withings, WHOOP, Oura, Polar, Nightscout, mood webhook, CSV re-imports, Apple stats day-totals); Apple sample-grain deletions stick per the paired-client contract, and the mood bulk route's tombstone match is now a true no-op reported as duplicate. Import counts reconcile against actual createMany counts. Google lifecycle: a dead token registers on the cycle's hard-fail ledger, so an expired connection parks instead of stamping success hourly; backfill and sleep-repair markers stamp only after a clean pass (pg-boss retries are now real); failed measurement writes hold the watermark; no-op updates are skipped; mapping.md tells the current truth. Scale: doctor-report reads only consumed columns and enabled sections; exports and backups keyset-paginate; achievements tally via SQL buckets; series charts day-bucket dense kinds beyond 90 days; BP pairing is a linear two-pointer; distinct-type scans run server-side; the graded-series cold fallback is bounded; the fitbit defer tracker loops instead of spreading. --- CHANGELOG.md | 38 +++ docs/api/openapi.yaml | 2 +- package.json | 2 +- public/sw.js | 2 +- .../__tests__/soft-delete-filter.test.ts | 27 +++ src/app/api/export/measurements/route.ts | 22 +- src/app/api/export/route.ts | 29 ++- .../__tests__/hidden-redaction.test.ts | 4 + .../achievements/__tests__/ios-format.test.ts | 4 + .../__tests__/module-gate.test.ts | 5 + src/app/api/google-health/sync/route.ts | 2 +- .../api/import/csv/__tests__/route.test.ts | 58 ++++- src/app/api/import/csv/route.ts | 30 ++- .../insights/generate/__tests__/route.test.ts | 4 + src/app/api/insights/generate/route.ts | 171 +------------ .../insights/targets/__tests__/route.test.ts | 52 ++-- .../api/integrations/moodlog/webhook/route.ts | 7 + .../batch/__tests__/hourly-hr-wire.test.ts | 66 ++++++ src/app/api/measurements/batch/route.ts | 8 + .../series/__tests__/route.test.ts | 224 ++++++++++++++++++ src/app/api/measurements/series/route.ts | 174 +++++++++++++- .../mood-entries/bulk/__tests__/route.test.ts | 68 ++++++ src/app/api/mood-entries/bulk/route.ts | 21 +- src/lib/doctor-report-data.ts | 53 +++++ src/lib/export/full-backup-payload.ts | 27 ++- src/lib/export/paged-measurements.ts | 53 +++++ src/lib/fitbit/__tests__/client.test.ts | 11 +- .../__tests__/sync-upsert-resurrect.test.ts | 123 ++++++++++ src/lib/fitbit/__tests__/sync.test.ts | 30 ++- src/lib/fitbit/client.ts | 11 +- src/lib/fitbit/sync.ts | 63 +++-- src/lib/gamification/achievements-result.ts | 202 +++++++++------- src/lib/gamification/expansion-metrics.ts | 53 ++++- .../__tests__/sync-failsoft.test.ts | 155 +++++++++++- .../__tests__/sync-upsert-resurrect.test.ts | 82 +++++++ src/lib/google-health/client.ts | 3 +- src/lib/google-health/mapping.md | 32 ++- src/lib/google-health/sync-activity.ts | 28 ++- src/lib/google-health/sync-sleep.ts | 18 +- src/lib/google-health/sync-workout.ts | 5 + src/lib/google-health/sync.ts | 133 ++++++++++- .../__tests__/graded-series-rollups.test.ts | 53 ++++- .../invalidate-status-insights.test.ts | 13 +- .../__tests__/pulse-status-timeout.test.ts | 4 + .../insights/__tests__/pulse-status.test.ts | 12 + src/lib/insights/comprehensive-generate.ts | 9 +- src/lib/insights/graded-series.ts | 96 ++++++-- .../__tests__/google-health-backfill.test.ts | 84 +++++++ .../google-health-sleep-repair.test.ts | 81 ++++++- src/lib/jobs/dense-intraday-retention.ts | 9 +- src/lib/jobs/google-health-backfill.ts | 44 +++- src/lib/jobs/google-health-sleep-repair.ts | 48 +++- src/lib/jobs/insight-pregenerate.ts | 14 +- src/lib/jobs/mean-consolidation.ts | 10 +- src/lib/nightscout/__tests__/sync.test.ts | 4 + src/lib/nightscout/sync.ts | 6 +- src/lib/oura/__tests__/client.test.ts | 40 +++- src/lib/oura/__tests__/sync.test.ts | 68 +++++- src/lib/oura/client.ts | 16 +- src/lib/oura/sync.ts | 61 ++++- src/lib/polar/__tests__/client.test.ts | 49 +++- src/lib/polar/__tests__/sync.test.ts | 76 +++++- src/lib/polar/client.ts | 12 +- src/lib/polar/sync.ts | 30 ++- .../__tests__/reconstruct-timeline.test.ts | 54 ++++- .../__tests__/sweep-stale-segments.test.ts | 109 +++++++++ src/lib/sleep/reconstruct-timeline.ts | 22 +- src/lib/sleep/sweep-stale-segments.ts | 83 +++++++ src/lib/targets/build-response.ts | 59 +++-- src/lib/whoop/__tests__/client.test.ts | 42 +++- .../whoop/__tests__/sync-sleep-sweep.test.ts | 143 +++++++++++ src/lib/whoop/client.ts | 10 +- src/lib/whoop/sync-sleep.ts | 54 ++++- src/lib/whoop/sync.ts | 4 + .../withings/__tests__/sync-activity.test.ts | 46 +++- src/lib/withings/__tests__/sync-sleep.test.ts | 103 +++++++- src/lib/withings/sync-activity.ts | 25 +- src/lib/withings/sync-sleep.ts | 84 +++++-- src/lib/withings/sync.ts | 6 +- 79 files changed, 3221 insertions(+), 564 deletions(-) create mode 100644 src/lib/export/paged-measurements.ts create mode 100644 src/lib/fitbit/__tests__/sync-upsert-resurrect.test.ts create mode 100644 src/lib/jobs/__tests__/google-health-backfill.test.ts create mode 100644 src/lib/sleep/__tests__/sweep-stale-segments.test.ts create mode 100644 src/lib/sleep/sweep-stale-segments.ts create mode 100644 src/lib/whoop/__tests__/sync-sleep-sweep.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 382c81b1e..392b93d13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,44 @@ ## [Unreleased] +## [1.28.25] — 2026-07-11 — Every integration held to the same standard + +A platform-wide hardening pass: the failure classes found live this week were +hunted down across every integration, not just where they first surfaced. + +Fitbit carried the same silent wedge fixed for Google Health — a soft-deleted +reading permanently blocked its own re-import, hidden behind a comment +describing a database index that never existed. Fixed the same way: a re-import +revives the deleted row in place. + +Sleep segments now keep stable identities across re-scoring for Withings, +WHOOP, Polar and Oura — previously a source refining a night could duplicate or +orphan its stage rows, quietly inflating the night's total (the class fixed for +Google earlier this week). Each sync now also sweeps rows a re-score left +behind, so already-affected nights heal themselves as they are re-read. + +One platform rule for deleted rows: a reading owned by a connected source comes +back when the source reports it again — across Withings, WHOOP, Oura, Polar, +Nightscout, the mood webhook, CSV re-imports and Apple Health day totals. +Deleting an Apple Health sample in the Health app still sticks, as the paired +client expects. Import counts and per-entry statuses now tell the truth in +every one of these paths. + +Google Health lifecycle: an expired connection no longer lets the hourly sync +stamp success while importing nothing (it parks cleanly for reconnect instead +of retrying against a dead token forever); history backfills and the sleep +repair only mark themselves done after a clean pass, so a transient error now +retries instead of silently leaving a gap; and a failed database write holds +the sync watermark so the affected window is re-fetched. + +Scale: the doctor report reads only the columns and sections it renders; +exports and backups read the measurement table in pages instead of one giant +query; long-window charts for sensor-dense metrics (glucose, pulse) aggregate +per day in the database beyond 90 days instead of shipping every raw sample; +blood-pressure pairing, achievement tallies and several "which types exist" +scans moved from in-memory loops to the database. A year of continuous sensor +data can no longer stall or crash a request. + ## [1.28.24] — 2026-07-11 — A deleted Google reading no longer blocks its re-import A Google Health reading that had been soft-deleted could never be imported diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 81d53e3de..3247dd603 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: HealthLog API - version: 1.28.24 + version: 1.28.25 description: >- Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest. diff --git a/package.json b/package.json index 883c22641..22ad0c9e6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.28.24", + "version": "1.28.25", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "PolyForm-Noncommercial-1.0.0", "homepage": "https://healthlog.dev", diff --git a/public/sw.js b/public/sw.js index f0010277f..7b3ad88a5 100644 --- a/public/sw.js +++ b/public/sw.js @@ -36,7 +36,7 @@ try { // v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild. const CACHE_VERSION = (typeof self !== "undefined" && self.__APP_VERSION__) || - /* @sw-version-fallback */ "v1.28.24"; + /* @sw-version-fallback */ "v1.28.25"; const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`; const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`; // v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*` diff --git a/src/app/api/export/__tests__/soft-delete-filter.test.ts b/src/app/api/export/__tests__/soft-delete-filter.test.ts index a34d1c03b..ae3d95f6d 100644 --- a/src/app/api/export/__tests__/soft-delete-filter.test.ts +++ b/src/app/api/export/__tests__/soft-delete-filter.test.ts @@ -17,6 +17,9 @@ import { NextRequest } from "next/server"; vi.mock("@/lib/db", () => ({ prisma: { + // v1.28.25 — the achievements vitals read is a raw (day, hour, type) + // bucket aggregation. + $queryRaw: vi.fn(), measurement: { findMany: vi.fn(), count: vi.fn() }, medication: { findMany: vi.fn() }, medicationIntakeEvent: { findMany: vi.fn(), count: vi.fn() }, @@ -31,6 +34,14 @@ vi.mock("@/lib/db", () => ({ createMany: vi.fn(), }, auditEvent: { findMany: vi.fn() }, + // v1.28.25 — the achievements aggregation previously aborted mid-flight + // on this mock (no `auditLog` model), which the old assertion masked + // because the vitals findMany fired before the abort. With the vitals + // read on `$queryRaw`, the sleep findMany the assertion now pins runs + // AFTER these models — mock them so the aggregation completes. + auditLog: { findMany: vi.fn() }, + userHealthProfile: { findUnique: vi.fn() }, + illnessEpisode: { findMany: vi.fn() }, }, })); @@ -95,6 +106,7 @@ beforeEach(() => { remaining: 9, resetAt: Date.now() + 3_600_000, }); + vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); vi.mocked(prisma.measurement.count).mockResolvedValue(0); vi.mocked(prisma.medication.findMany).mockResolvedValue([] as never); @@ -107,6 +119,12 @@ beforeEach(() => { vi.mocked(prisma.user.findUnique).mockResolvedValue({ heightCm: null, } as never); + vi.mocked(prisma.passkey.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.auditLog.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.userHealthProfile.findUnique).mockResolvedValue( + null as never, + ); + vi.mocked(prisma.illnessEpisode.findMany).mockResolvedValue([] as never); }); describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => { @@ -154,5 +172,14 @@ describe("v1.4.41 W-DELETED-2 — soft-delete invisibility", () => { where: expect.objectContaining({ deletedAt: null }), }); } + // v1.28.25 — the vitals read moved to a raw (day, hour, type) bucket + // aggregation; the soft-delete scope must survive on that path too. + const rawCalls = vi.mocked(prisma.$queryRaw).mock.calls; + expect(rawCalls.length).toBeGreaterThan(0); + for (const [strings] of rawCalls) { + expect((strings as unknown as readonly string[]).join("?")).toContain( + `"deleted_at" IS NULL`, + ); + } }); }); diff --git a/src/app/api/export/measurements/route.ts b/src/app/api/export/measurements/route.ts index 3a5592596..1c6eadff6 100644 --- a/src/app/api/export/measurements/route.ts +++ b/src/app/api/export/measurements/route.ts @@ -23,6 +23,7 @@ import { auditLog } from "@/lib/auth/audit"; import { apiError, getClientIp } from "@/lib/api-response"; import { checkRateLimit } from "@/lib/rate-limit"; import { toCSV, formatMeasurementsForExport } from "@/lib/export"; +import { findMeasurementsPaged } from "@/lib/export/paged-measurements"; import { shapeMeasurementNotes } from "@/lib/crypto/note-cipher"; import { resolveUserTimezone } from "@/lib/tz/resolver"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; @@ -49,9 +50,24 @@ export const GET = apiHandler(async (request: NextRequest) => { const [measurements, userTz, sourcePriorityJson, profile] = await Promise.all( [ - prisma.measurement.findMany({ - where, - orderBy: { measuredAt: "desc" }, + // v1.28.25 — keyset-paginated read with a narrow select. The route + // keeps its contract (an absent `since`/`until` means the full + // history), but the read is now chunked so an unbounded window on a + // CGM / per-sample-HR account never materialises the whole table in + // one query. The select is exactly the formatter's `ExportMeasurement` + // shape + note decryption + the `id` cursor key. + findMeasurementsPaged(prisma, where, { + id: true, + type: true, + value: true, + unit: true, + measuredAt: true, + source: true, + notes: true, + notesEncrypted: true, + glucoseContext: true, + sleepStage: true, + deviceType: true, }), resolveUserTimezone(user.id), loadUserSourcePriority(user.id), diff --git a/src/app/api/export/route.ts b/src/app/api/export/route.ts index 41534be3b..fb93945bb 100644 --- a/src/app/api/export/route.ts +++ b/src/app/api/export/route.ts @@ -12,6 +12,7 @@ import { formatMoodEntriesForExport, } from "@/lib/export"; import { shapeMeasurementNotes, shapeMoodNote } from "@/lib/crypto/note-cipher"; +import { findMeasurementsPaged } from "@/lib/export/paged-measurements"; import { resolveUserTimezone } from "@/lib/tz/resolver"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; import { NextRequest, NextResponse } from "next/server"; @@ -59,13 +60,33 @@ export const GET = apiHandler(async (request: NextRequest) => { const data: Record = {}; if (type === "measurements" || type === "all") { - const measurements = await prisma.measurement.findMany({ + // v1.28.25 — keyset-paginated read with a narrow select. The legacy + // export has no date window, so a CGM / per-sample-HR account holds a + // six-figure row set; one unbounded full-width `findMany` here was the + // scale hazard. The select carries exactly the fields the formatter + + // note decryption consume (`ExportMeasurement` + `shapeMeasurementNotes` + // + the `id` cursor key) — no Bytes column beyond the note ciphertext + // the export must decrypt. + const measurements = await findMeasurementsPaged( + prisma, // v1.4.41 W-DELETED-2 — exclude soft-deleted measurements from // the legacy /api/export endpoint so CSV + JSON downloads stay // consistent with the live measurement reads. - where: { userId, deletedAt: null }, - orderBy: { measuredAt: "desc" }, - }); + { userId, deletedAt: null }, + { + id: true, + type: true, + value: true, + unit: true, + measuredAt: true, + source: true, + notes: true, + notesEncrypted: true, + glucoseContext: true, + sleepStage: true, + deviceType: true, + }, + ); // v1.11.5 — sleep collapses to one row per night by default (the // formatter's `night` granularity), matching the per-type CSV route. // Thread the user's source-priority so a multi-source night dedups to diff --git a/src/app/api/gamification/achievements/__tests__/hidden-redaction.test.ts b/src/app/api/gamification/achievements/__tests__/hidden-redaction.test.ts index deb5464b0..3f5d16626 100644 --- a/src/app/api/gamification/achievements/__tests__/hidden-redaction.test.ts +++ b/src/app/api/gamification/achievements/__tests__/hidden-redaction.test.ts @@ -3,6 +3,9 @@ import { NextRequest } from "next/server"; vi.mock("@/lib/db", () => ({ prisma: { + // v1.28.25 — the achievements vitals read is a raw (day, hour, type) + // bucket aggregation. + $queryRaw: vi.fn(), measurement: { findMany: vi.fn() }, medicationIntakeEvent: { findMany: vi.fn() }, medication: { findMany: vi.fn() }, @@ -84,6 +87,7 @@ beforeEach(() => { // v1.4.34 IW-G — reset achievement LRU between tests so each case // observes a cold cache. __resetAllCachesForTests(); + vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); vi.mocked(prisma.medicationIntakeEvent.findMany).mockResolvedValue( [] as never, diff --git a/src/app/api/gamification/achievements/__tests__/ios-format.test.ts b/src/app/api/gamification/achievements/__tests__/ios-format.test.ts index 3393acc51..8bccf3bb0 100644 --- a/src/app/api/gamification/achievements/__tests__/ios-format.test.ts +++ b/src/app/api/gamification/achievements/__tests__/ios-format.test.ts @@ -3,6 +3,9 @@ import { NextRequest } from "next/server"; vi.mock("@/lib/db", () => ({ prisma: { + // v1.28.25 — the achievements vitals read is a raw (day, hour, type) + // bucket aggregation. + $queryRaw: vi.fn(), measurement: { findMany: vi.fn() }, medicationIntakeEvent: { findMany: vi.fn() }, medication: { findMany: vi.fn() }, @@ -70,6 +73,7 @@ beforeEach(() => { // v1.4.34 IW-G — reset achievement LRU between tests so each case // observes a cold cache. __resetAllCachesForTests(); + vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); vi.mocked(prisma.medicationIntakeEvent.findMany).mockResolvedValue( [] as never, diff --git a/src/app/api/gamification/achievements/__tests__/module-gate.test.ts b/src/app/api/gamification/achievements/__tests__/module-gate.test.ts index 902ce62a5..76842cf9e 100644 --- a/src/app/api/gamification/achievements/__tests__/module-gate.test.ts +++ b/src/app/api/gamification/achievements/__tests__/module-gate.test.ts @@ -16,6 +16,9 @@ import { NextRequest } from "next/server"; vi.mock("@/lib/db", () => ({ prisma: { + // v1.28.25 — the achievements vitals read is a raw (day, hour, type) + // bucket aggregation. + $queryRaw: vi.fn(), measurement: { findMany: vi.fn() }, medicationIntakeEvent: { findMany: vi.fn() }, medication: { findMany: vi.fn() }, @@ -76,6 +79,7 @@ const SESSION_OK = { beforeEach(() => { vi.resetAllMocks(); __resetAllCachesForTests(); + vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); vi.mocked(prisma.medicationIntakeEvent.findMany).mockResolvedValue( [] as never, @@ -123,6 +127,7 @@ describe("GET /api/gamification/achievements — achievements module gate", () = expect(body.meta.module).toBe("achievements"); // The disappearance is total: no badge aggregation ran. + expect(prisma.$queryRaw).not.toHaveBeenCalled(); expect(prisma.measurement.findMany).not.toHaveBeenCalled(); expect(prisma.userAchievement.findMany).not.toHaveBeenCalled(); expect(prisma.userAchievement.createMany).not.toHaveBeenCalled(); diff --git a/src/app/api/google-health/sync/route.ts b/src/app/api/google-health/sync/route.ts index f15a8fc23..f3e887e4b 100644 --- a/src/app/api/google-health/sync/route.ts +++ b/src/app/api/google-health/sync/route.ts @@ -55,6 +55,6 @@ export const POST = apiHandler(async (request: NextRequest) => { } } - const imported = await syncUserGoogleHealth(user.id, { fullSync }); + const { imported } = await syncUserGoogleHealth(user.id, { fullSync }); return apiSuccess({ imported, fullSync }); }); diff --git a/src/app/api/import/csv/__tests__/route.test.ts b/src/app/api/import/csv/__tests__/route.test.ts index a1bb577a0..72daea91c 100644 --- a/src/app/api/import/csv/__tests__/route.test.ts +++ b/src/app/api/import/csv/__tests__/route.test.ts @@ -103,7 +103,11 @@ beforeEach(() => { updateMany: ReturnType; }; measurement.findMany.mockResolvedValue([]); - measurement.createMany.mockResolvedValue({ count: 0 }); + // Echo the attempted chunk size, matching Postgres semantics when no row + // collides — the route now reconciles `inserted` against this count. + measurement.createMany.mockImplementation( + async (arg: { data: unknown[] }) => ({ count: arg.data.length }), + ); measurement.updateMany.mockResolvedValue({ count: 0 }); ( prisma.$transaction as unknown as { @@ -226,6 +230,29 @@ describe("POST /api/import/csv — batched write + per-row envelope", () => { expect(mMeasurement().createMany).not.toHaveBeenCalled(); }); + it("resurrects a tombstoned externalId row on re-import (deletedAt: null in update)", async () => { + // The ext-probe is deliberately deletedAt-less, so a tombstoned IMPORT + // row matches like a live one; the update must carry the resurrection + // (IMPORT rows are re-importable by design). + mMeasurement().findMany.mockResolvedValue([ + { type: "WEIGHT", externalId: "ext-tomb" }, + ]); + + const res = await POST( + csvRequest( + [HEADER, "WEIGHT,81.0,kg,2026-05-01T08:00:00Z,,,ext-tomb"].join("\n"), + ), + ); + const body = (await res.json()) as CsvEnvelope; + expect(body.data?.updated).toBe(1); + expect(body.data?.rows[0].status).toBe("updated"); + const updateArg = mMeasurement().updateMany.mock.calls[0][0] as { + data: { value: number; deletedAt: Date | null }; + }; + expect(updateArg.data.value).toBe(81.0); + expect(updateArg.data.deletedAt).toBeNull(); + }); + it("inserts an externalId row when it does not exist yet", async () => { mMeasurement().findMany.mockResolvedValue([]); const res = await POST( @@ -274,4 +301,33 @@ describe("POST /api/import/csv — batched write + per-row envelope", () => { // Only the first survivor reaches the bulk insert. expect(mMeasurement().createMany.mock.calls[0][0].data).toHaveLength(1); }); + + it("reconciles `inserted` against the createMany count when skipDuplicates absorbs a race", async () => { + // Two fresh rows attempted, but a concurrent double-submit already + // landed one of them — `skipDuplicates` absorbs the conflict and the + // count comes back short. The envelope must sum to the DB truth: one + // inserted, one downgraded to skipped/duplicate. + mMeasurement().createMany.mockImplementation( + async (arg: { data: unknown[] }) => ({ count: arg.data.length - 1 }), + ); + + const res = await POST( + csvRequest( + [ + HEADER, + "WEIGHT,80.5,kg,2026-05-01T08:00:00Z,,,", + "WEIGHT,81.5,kg,2026-05-01T09:00:00Z,,,", + ].join("\n"), + ), + ); + expect(res.status).toBe(200); + const body = (await res.json()) as CsvEnvelope; + expect(body.data?.inserted).toBe(1); + expect(body.data?.skipped).toBe(1); + const statuses = body.data?.rows.map((r) => r.status).sort(); + expect(statuses).toEqual(["inserted", "skipped"]); + expect(body.data?.rows.find((r) => r.status === "skipped")?.reason).toBe( + "duplicate", + ); + }); }); diff --git a/src/app/api/import/csv/route.ts b/src/app/api/import/csv/route.ts index 7fdd868d7..34cf2dc18 100644 --- a/src/app/api/import/csv/route.ts +++ b/src/app/api/import/csv/route.ts @@ -239,12 +239,14 @@ export const POST = apiHandler(async (request: NextRequest) => { ...plainInserts.map(buildCreateData), ]; const CREATE_CHUNK = 200; + let createdCount = 0; await prisma.$transaction(async (tx) => { for (let i = 0; i < toCreate.length; i += CREATE_CHUNK) { - await tx.measurement.createMany({ + const res = await tx.measurement.createMany({ data: toCreate.slice(i, i + CREATE_CHUNK), skipDuplicates: true, }); + createdCount += res.count; } for (const m of extUpdates.values()) { await tx.measurement.updateMany({ @@ -262,11 +264,37 @@ export const POST = apiHandler(async (request: NextRequest) => { notesEncrypted: encryptNote(m.notes ?? null), glucoseContext: (m.glucoseContext as GlucoseContext | undefined) ?? null, + // No-op on a live row, a deliberate RESURRECTION on a + // tombstoned one — IMPORT rows are re-importable by design, so + // a re-imported externalId brings the row back (mirrors the + // source-owned sync resurrect rule). + deletedAt: null, }, }); } }); + // Reconcile `inserted` against what `createMany` actually wrote + // (mirrors the batch route's raced-duplicate downgrade). Under + // `skipDuplicates` a conflicting row is silently absorbed — the key is + // present in the table either way, so we cannot identify the SPECIFIC + // raced lines; we only need the counters and per-line statuses to sum + // to the truth. Downgrade enough `inserted` lines to `duplicate` so + // the envelope matches the DB write count. + const racedDuplicates = toCreate.length - createdCount; + if (racedDuplicates > 0) { + let downgraded = 0; + for (const [line, outcome] of writeOutcome) { + if (downgraded >= racedDuplicates) break; + if (outcome === "inserted") { + writeOutcome.set(line, "duplicate"); + inserted--; + skipped++; + downgraded++; + } + } + } + // One bounded rollup re-fold per touched (type, day) — a 10 000-row CSV // pays at most ~N (type, day) recomputes, not 10 000 per-row hooks. // Best-effort: a populator hiccup never fails the importer. diff --git a/src/app/api/insights/generate/__tests__/route.test.ts b/src/app/api/insights/generate/__tests__/route.test.ts index 550817bd4..507a29e51 100644 --- a/src/app/api/insights/generate/__tests__/route.test.ts +++ b/src/app/api/insights/generate/__tests__/route.test.ts @@ -104,6 +104,10 @@ vi.mock("@/lib/jobs/insight-pregenerate-shared", () => ({ // enqueue pipeline itself is covered by the comprehensive-generate tests. vi.mock("@/lib/insights/comprehensive-generate", () => ({ enqueueStatusRefillForUser: vi.fn(async () => 7), + // v1.28.25 — the route shares the lib's comparison-snapshot builder + // (its private near-copy was deleted). Null = comparison toggle off, + // matching the pre-existing fixtures (no dashboardWidgetsJson set). + buildComparisonSnapshotForUser: vi.fn(async () => null), })); vi.mock("@/lib/logging/context", () => ({ diff --git a/src/app/api/insights/generate/route.ts b/src/app/api/insights/generate/route.ts index 9a16bad10..2910a606b 100644 --- a/src/app/api/insights/generate/route.ts +++ b/src/app/api/insights/generate/route.ts @@ -30,10 +30,7 @@ import { buildDerivedBriefingPrompt, } from "@/lib/insights/derived-briefing"; import { getAgeFromDateOfBirth } from "@/lib/analytics/pulse-targets"; -import { - buildUserPrompt, - type ComparisonSnapshot, -} from "@/lib/ai/prompts/insight-system-prompt"; +import { buildUserPrompt } from "@/lib/ai/prompts/insight-system-prompt"; import { buildSystemPromptWithReferences, buildBriefingPersonalisationBlock, @@ -43,18 +40,7 @@ import { getSelfContextTextForUser, } from "@/lib/ai/coach/about-me"; import { metricsFromPresentSections } from "@/lib/ai/medical-references"; -import { summarize, type DataPoint } from "@/lib/analytics/trends"; -import { - reconstructSleepNights, - type SleepStageRow, -} from "@/lib/analytics/sleep-night"; -import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; import { resolveUserTimezone } from "@/lib/tz/resolver"; -import { - resolveDashboardLayout, - type ComparisonBaseline, -} from "@/lib/dashboard-layout"; -import type { MeasurementType } from "@/generated/prisma/client"; import { insightResultSchema, singleUserTurn, @@ -88,7 +74,13 @@ import { resolveServerLocale } from "@/lib/i18n/server-locale"; import { hasUsableStatusProvider } from "@/lib/insights/status-provider"; import { hashInsightSnapshot } from "@/lib/insights/snapshot-hash"; import { enqueueForceWarm } from "@/lib/jobs/insight-pregenerate-shared"; -import { enqueueStatusRefillForUser } from "@/lib/insights/comprehensive-generate"; +// v1.28.25 — the comparison-snapshot builder was a private near-copy of +// the lib export (drifted only in comments); the route now shares the +// one implementation the nightly warm pass uses. +import { + buildComparisonSnapshotForUser, + enqueueStatusRefillForUser, +} from "@/lib/insights/comprehensive-generate"; export const dynamic = "force-dynamic"; @@ -120,153 +112,6 @@ export function resolveInsightsRateLimit(): number { return parsed; } -/** - * v1.4.16 phase B8 — fetch the user's persisted comparison toggle and - * build a `ComparisonSnapshot` for the prompt builder. Returns null - * when comparison is off (most users), so the prompt builder skips - * the context block entirely. - * - * Snapshot construction reuses the analytics `summarize()` helper - * (which now emits `avg30LastMonth` + `avg30LastYear`) so the numbers - * the LLM sees match the numbers the dashboard tile renders — no - * second source of truth for the comparison delta. - */ -async function buildComparisonSnapshotForUser( - userId: string, -): Promise { - // The features payload doesn't carry prior-period values — pulling - // a fresh `summarize()` per metric below keeps the snapshot - // self-contained without retrofitting features.ts. - const row = await prisma.user.findUnique({ - where: { id: userId }, - select: { dashboardWidgetsJson: true }, - }); - const layout = resolveDashboardLayout(row?.dashboardWidgetsJson); - const baseline: ComparisonBaseline = layout.comparisonBaseline ?? "none"; - if (baseline === "none") return null; - - // Pull every measurement type once and run summarize() so the - // prior-period averages line up with what the dashboard analytics - // computes. Only include rows where the LLM can sensibly narrate - // a prior comparison (i.e. the metric exists in our snapshot type - // mapping below). - const typeToSnapshotKey: Record = { - WEIGHT: "weight", - BLOOD_PRESSURE_SYS: "bloodPressureSys", - BLOOD_PRESSURE_DIA: "bloodPressureDia", - PULSE: "pulse", - BODY_FAT: "bodyFat", - SLEEP_DURATION: "sleep", - ACTIVITY_STEPS: "steps", - }; - const typeUnits: Record = { - WEIGHT: "kg", - BLOOD_PRESSURE_SYS: "mmHg", - BLOOD_PRESSURE_DIA: "mmHg", - PULSE: "bpm", - BODY_FAT: "%", - SLEEP_DURATION: "h", - ACTIVITY_STEPS: "", - }; - const types = Object.keys(typeToSnapshotKey) as MeasurementType[]; - // The snapshot only feeds summarize()'s avg30 / avg30LastMonth / - // avg30LastYear. The widest of those reaches back into the [365, 395)-day - // window (ageMs < 395 * DAY in meanOfWindow), so nothing older than 395 - // days can change a result. Bound the read at 400 days — a 5-day floor over - // the strict-less-than boundary — instead of scanning the full history, - // which on multi-year accounts is tens of thousands of rows per type on the - // page-blocking generate path and the nightly pregenerate cron. - const sinceMeasuredAt = new Date(Date.now() - 400 * 86_400_000); - const rows = await Promise.all( - types.map(async (type) => { - // SLEEP_DURATION is stored ONE ROW PER STAGE per night; summarising the - // raw stage rows mislabels a per-stage average as a night total (and - // double-counts a bare ASLEEP aggregate against its granular twin). Route - // the comparison through the per-night dedup reconstruction so the - // current/baseline averages are per-night TIME-ASLEEP totals in minutes. - let dataPoints: DataPoint[]; - if (type === "SLEEP_DURATION") { - const [sleepRows, sleepTz, sleepPriority] = await Promise.all([ - prisma.measurement.findMany({ - where: { - userId, - type, - deletedAt: null, - measuredAt: { gte: sinceMeasuredAt }, - }, - orderBy: { measuredAt: "asc" }, - select: { - measuredAt: true, - value: true, - sleepStage: true, - source: true, - // Writer-level collapse: two HealthKit apps behind one - // source (watch stages vs phone in-bed) must not blend. - deviceType: true, - }, - }), - resolveUserTimezone(userId), - loadUserSourcePriority(userId), - ]); - dataPoints = reconstructSleepNights( - sleepRows as SleepStageRow[], - sleepTz, - sleepPriority, - ) - .filter((n) => n.asleepMinutes > 0) - .map((n) => ({ date: n.measuredAt, value: n.asleepMinutes })); - } else { - const measurements = await prisma.measurement.findMany({ - where: { - userId, - type, - deletedAt: null, - measuredAt: { gte: sinceMeasuredAt }, - }, - orderBy: { measuredAt: "asc" }, - select: { measuredAt: true, value: true }, - }); - dataPoints = measurements.map((m): DataPoint => ({ - date: m.measuredAt, - value: m.value, - })); - } - const summary = summarize(dataPoints); - const baselineAvg = - baseline === "lastMonth" - ? (summary.avg30LastMonth ?? null) - : (summary.avg30LastYear ?? null); - const currentAvg = summary.avg30 ?? null; - const delta = - currentAvg !== null && baselineAvg !== null - ? Math.round((currentAvg - baselineAvg) * 100) / 100 - : null; - const deltaPercent = - delta !== null && baselineAvg !== null && baselineAvg !== 0 - ? Math.round((delta / Math.abs(baselineAvg)) * 100 * 10) / 10 - : null; - return { - type: typeToSnapshotKey[type] ?? type, - currentAvg, - baselineAvg, - delta, - deltaPercent, - unit: typeUnits[type] ?? "", - }; - }), - ); - - // Drop fully-empty rows (current AND baseline missing) so the prompt - // doesn't pad with noise. Keep partial rows (one side null) — the - // block renderer flags them with "no prior-period data available" - // which is useful context for the model. - const metrics = rows.filter( - (row) => row.currentAvg !== null || row.baselineAvg !== null, - ); - - return { baseline, metrics }; -} - /** * GET /api/insights/generate — read-only advisor read. * diff --git a/src/app/api/insights/targets/__tests__/route.test.ts b/src/app/api/insights/targets/__tests__/route.test.ts index 1d198ef27..3453c7592 100644 --- a/src/app/api/insights/targets/__tests__/route.test.ts +++ b/src/app/api/insights/targets/__tests__/route.test.ts @@ -145,6 +145,9 @@ beforeEach(() => { ( prisma.moodEntryRollup.findMany as ReturnType ).mockResolvedValue([]); + // v1.28.25 — the latest-ever-per-type read is a raw `DISTINCT ON` + // (Prisma's `distinct` dedups client-side after pulling every row). + (prisma.$queryRawUnsafe as ReturnType).mockResolvedValue([]); }); describe("GET /api/insights/targets — mood-rollup tier swap", () => { @@ -291,36 +294,33 @@ describe("GET /api/insights/targets — mood-rollup tier swap", () => { expect(moodTarget).toBeUndefined(); }); - it("floors the `findMany distinct` latest-ever-per-type read at one year", async () => { - // Audit High finding 3: the `findMany({orderBy: measuredAt desc, - // distinct: ["type"]})` for "absolute latest measurement per type" - // used to run WITHOUT a `measuredAt >=` floor. On a 347 k-row - // tenant Postgres still has to sort the full set before applying - // DISTINCT ON because Prisma's `distinct` does not compile to PG's - // `DISTINCT ON` — it dedups in the driver. v1.4.40 adds a - // 365-day floor so the planner-side walk is bounded. - (prisma.measurement.findMany as ReturnType).mockResolvedValue( - [], - ); - + it("reads latest-ever-per-type via raw DISTINCT ON with the one-year floor", async () => { + // Audit High finding 3 established the 365-day floor; v1.28.25 moves + // the read onto a real Postgres `DISTINCT ON` (Prisma's `distinct` + // dedups in the driver AFTER pulling every row in the window, which + // on a dense-type tenant shipped a year of raw rows to Node just to + // keep seven). Pin the raw query: DISTINCT ON, per-type descending + // walk, user id + floored date as bound parameters. await callGet(makeReq()); - // Find the `distinct: ["type"]` call. - const calls = ( - prisma.measurement.findMany as ReturnType - ).mock.calls.map((c) => c[0]); - const distinctCall = calls.find( - (c) => - c && - Array.isArray(c.distinct) && - c.distinct[0] === "type" && - c.orderBy?.measuredAt === "desc", + const rawCalls = (prisma.$queryRawUnsafe as ReturnType).mock + .calls; + const distinctCall = rawCalls.find( + (c) => typeof c[0] === "string" && c[0].includes("DISTINCT ON"), ); expect(distinctCall).toBeDefined(); - // The `measuredAt >= oneYearAgo` floor must be present. - expect(distinctCall?.where?.measuredAt).toBeDefined(); - expect(distinctCall?.where?.measuredAt?.gte).toBeInstanceOf(Date); - const flooredAt = distinctCall?.where?.measuredAt?.gte as Date; + const [sql, boundUserId, flooredAt] = distinctCall as [ + string, + string, + Date, + ]; + expect(sql).toContain(`DISTINCT ON (m."type")`); + expect(sql).toContain(`ORDER BY m."type" ASC, m."measured_at" DESC`); + // Parameter-bound, never spliced: user id + date ride $1/$2. + expect(sql).toContain("$1"); + expect(sql).toContain("$2"); + expect(boundUserId).toBe("user-targets-1"); + expect(flooredAt).toBeInstanceOf(Date); const oneYearMs = 365 * 24 * 60 * 60 * 1000; const ageMs = Date.now() - flooredAt.getTime(); // Floor is somewhere around one year ago — give a ±1-hour window diff --git a/src/app/api/integrations/moodlog/webhook/route.ts b/src/app/api/integrations/moodlog/webhook/route.ts index 827a66fcf..64e699731 100644 --- a/src/app/api/integrations/moodlog/webhook/route.ts +++ b/src/app/api/integrations/moodlog/webhook/route.ts @@ -133,6 +133,11 @@ export const POST = apiHandler(async (request: NextRequest) => { source: "MOODLOG", date: entry.date, moodLoggedAt, + // No-op on a live row, a deliberate RESURRECTION on a tombstoned + // one — moodLog is the source of truth for the rows it minted; a + // created/updated event means the entry exists upstream (deletion + // arrives as its own `mood.deleted` event). + deletedAt: null, }, create: { userId: user.id, @@ -160,6 +165,8 @@ export const POST = apiHandler(async (request: NextRequest) => { score: entry.score, tags: entry.tags ? JSON.stringify(entry.tags) : null, source: "MOODLOG", + // Same resurrection rule as the externalId-keyed upsert above. + deletedAt: null, }, create: { userId: user.id, diff --git a/src/app/api/measurements/batch/__tests__/hourly-hr-wire.test.ts b/src/app/api/measurements/batch/__tests__/hourly-hr-wire.test.ts index 839e85d2f..866bffc84 100644 --- a/src/app/api/measurements/batch/__tests__/hourly-hr-wire.test.ts +++ b/src/app/api/measurements/batch/__tests__/hourly-hr-wire.test.ts @@ -308,3 +308,69 @@ describe("POST /api/measurements/batch — hourly HR wire contract (iOS #34)", ( expect(data.entries[0].status).toBe("updated"); }); }); + +describe("POST /api/measurements/batch — stats tombstone resurrection", () => { + it("resurrects a tombstoned stats: day-total on re-post (deletedAt: null, status updated)", async () => { + const STEP_STATS_ID = "stats:HKQuantityTypeIdentifierStepCount:2026-06-21"; + // The existence probe is deliberately deletedAt-less, so a tombstoned + // day-total row matches exactly like a live one and routes into the + // overwrite branch. + vi.mocked(prisma.measurement.findMany).mockResolvedValue([ + { + type: "ACTIVITY_STEPS", + source: "APPLE_HEALTH", + externalId: STEP_STATS_ID, + }, + ] as never); + + const res = await POST( + makeRequest({ + entries: [ + { + hkIdentifier: "HKQuantityTypeIdentifierStepCount", + value: 9001, + unit: "count", + startDate: "2026-06-21T00:00:00.000Z", + endDate: "2026-06-21T23:59:59.000Z", + externalId: STEP_STATS_ID, + }, + ], + }), + ); + expect(res.status).toBe(200); + const { data } = await readJson(res); + expect(data.updated).toBe(1); + expect(data.entries[0].status).toBe("updated"); + + // The overwrite carries the resurrection: the observer re-posts the + // day's canonical total, so the update data pins `deletedAt: null`. + expect(prisma.measurement.updateMany).toHaveBeenCalledTimes(1); + const updateArg = vi.mocked(prisma.measurement.updateMany).mock.calls[0][0]; + const updateData = ( + updateArg as { data: { value: number; deletedAt: Date | null } } + ).data; + expect(updateData.value).toBe(9001); + expect(updateData.deletedAt).toBeNull(); + }); + + it("keeps a tombstoned SAMPLE-grain row suppressed (duplicate, no resurrect write)", async () => { + // Apple LWW contract: the iOS reconciler propagates HealthKit + // deletions for sample-grain rows, so a re-post of a tombstoned + // sample stays a checkpointing `duplicate` — no update runs at all. + const SAMPLE_ID = "D00DAD00-0000-4000-8000-000000000000"; + vi.mocked(prisma.measurement.findMany).mockResolvedValue([ + { type: "PULSE", source: "APPLE_HEALTH", externalId: SAMPLE_ID }, + ] as never); + + const res = await POST( + makeRequest({ entries: [hrBucketEntry(SAMPLE_ID, 61)] }), + ); + expect(res.status).toBe(200); + const { data } = await readJson(res); + expect(data.duplicates).toBe(1); + expect(data.updated).toBe(0); + expect(data.entries[0].status).toBe("duplicate"); + expect(prisma.measurement.updateMany).not.toHaveBeenCalled(); + expect(prisma.measurement.createMany).not.toHaveBeenCalled(); + }); +}); diff --git a/src/app/api/measurements/batch/route.ts b/src/app/api/measurements/batch/route.ts index 3748b429c..354c68d60 100644 --- a/src/app/api/measurements/batch/route.ts +++ b/src/app/api/measurements/batch/route.ts @@ -545,6 +545,14 @@ async function postBatch(request: NextRequest): Promise { string | null, deviceType: p.row.deviceType as Prepared["row"]["deviceType"], sleepStage: p.row.sleepStage as Prepared["row"]["sleepStage"], + // No-op on a live row, a deliberate RESURRECTION on a + // tombstoned one — the observer re-posts the day's canonical + // total, so a `stats:` day-total row follows the + // source-owned resurrect rule (mirrors Google / Fitbit). + // SAMPLE-grain rows never reach this branch: their + // tombstones stick (the iOS LWW reconciler propagates + // HealthKit deletions and reports them as `duplicate`). + deletedAt: null, }, }); results[p.index] = { index: p.index, status: "updated" }; diff --git a/src/app/api/measurements/series/__tests__/route.test.ts b/src/app/api/measurements/series/__tests__/route.test.ts index e4070e049..fbeaa46a8 100644 --- a/src/app/api/measurements/series/__tests__/route.test.ts +++ b/src/app/api/measurements/series/__tests__/route.test.ts @@ -6,6 +6,9 @@ vi.mock("@/lib/db", () => ({ measurement: { findMany: vi.fn() }, user: { findUnique: vi.fn() }, auditLog: { create: vi.fn() }, + // v1.28.25 — dense kinds (glucose / pulse) day-bucket in SQL for + // windows beyond the 90-day raw cap. + $queryRaw: vi.fn(), }, })); @@ -65,6 +68,7 @@ function req(query: string): NextRequest { beforeEach(() => { vi.resetAllMocks(); vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); + vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); vi.mocked(prisma.auditLog.create).mockResolvedValue({} as never); vi.mocked(prisma.user.findUnique).mockResolvedValue({ glucoseUnit: "mg/dL", @@ -122,6 +126,48 @@ describe("GET /api/measurements/series", () => { expect(body.data.stats.count).toBe(1); }); + it("pairs BP via the sorted two-pointer merge exactly like the old scan (v1.28.25)", async () => { + // Pins the O(sys + dia) merge against the previous per-systolic full + // rescan: nearest diastolic within ±5 min wins, a tie keeps the + // EARLIER diastolic row, an out-of-window systolic pairs null. + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + const at = (iso: string) => new Date(iso); + const findManyMock = vi.mocked(prisma.measurement.findMany); + findManyMock.mockImplementation(((args: unknown) => { + const a = args as { where: { type: string } }; + if (a.where.type === "BLOOD_PRESSURE_SYS") { + return Promise.resolve([ + { id: "s1", value: 120, measuredAt: at("2026-05-01T10:00:00Z") }, + // Equidistant (±2 min) between the 10:03 and 10:07 dia rows. + { id: "s2", value: 124, measuredAt: at("2026-05-01T10:05:00Z") }, + { id: "s3", value: 130, measuredAt: at("2026-05-01T10:10:00Z") }, + // > 5 min from every diastolic row — stays unpaired. + { id: "s4", value: 140, measuredAt: at("2026-05-01T11:00:00Z") }, + ]) as never; + } + if (a.where.type === "BLOOD_PRESSURE_DIA") { + return Promise.resolve([ + { id: "d1", value: 80, measuredAt: at("2026-05-01T10:01:00Z") }, + { id: "d2", value: 82, measuredAt: at("2026-05-01T10:03:00Z") }, + { id: "d3", value: 84, measuredAt: at("2026-05-01T10:07:00Z") }, + { id: "d4", value: 86, measuredAt: at("2026-05-01T10:11:00Z") }, + ]) as never; + } + return Promise.resolve([]) as never; + }) as never); + const res = await GET(req("kind=bloodPressure&days=30")); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { points: Array<{ id: string; secondary: number | null }> }; + }; + expect(body.data.points.map((p) => [p.id, p.secondary])).toEqual([ + ["s1", 80], // nearest is d1 (1 min) + ["s2", 82], // d2/d3 tie at 2 min — earlier row wins + ["s3", 86], // nearest is d4 (1 min) + ["s4", null], // nothing within ±5 min + ]); + }); + it("returns single-value series for kind=weight", async () => { vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); vi.mocked(prisma.measurement.findMany).mockResolvedValue([ @@ -377,6 +423,184 @@ describe("GET /api/measurements/series", () => { ); }); +describe("GET /api/measurements/series — dense-kind day-bucketing (v1.28.25)", () => { + it("day-buckets glucose in SQL beyond the 90-day raw window", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.$queryRaw) + // 1st query — day buckets. + .mockResolvedValueOnce([ + { + bucket_start: new Date("2025-01-01T00:00:00Z"), + mean: 100, + min_value: 70, + max_value: 180, + }, + { + bucket_start: new Date("2025-01-02T00:00:00Z"), + mean: 126, + min_value: 80, + max_value: 200, + }, + ] as never) + // 2nd query — raw-row aggregate for the stats strip. + .mockResolvedValueOnce([ + { n: 576, mean: 113.04, min: 70, max: 200, sd: 22.34 }, + ] as never); + + const res = await GET(req("kind=glucose&days=365")); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { + unit: string; + points: Array<{ + id: string; + at: string; + value: number; + secondary: number | null; + }>; + stats: { + mean: number; + min: number; + max: number; + stdDev: number; + count: number; + }; + }; + }; + // No raw row walk — the read is SQL-aggregate only. + expect(prisma.measurement.findMany).not.toHaveBeenCalled(); + expect(prisma.$queryRaw).toHaveBeenCalledTimes(2); + expect(body.data.unit).toBe("mg/dL"); + expect(body.data.points).toEqual([ + { + id: "day:2025-01-01", + at: "2025-01-01T00:00:00.000Z", + value: 100, + secondary: null, + }, + { + id: "day:2025-01-02", + at: "2025-01-02T00:00:00.000Z", + value: 126, + secondary: null, + }, + ]); + // Stats come from the RAW-row aggregate (count = raw samples, not + // day buckets), matching the raw path's mg/dL parity convention. + expect(body.data.stats).toEqual({ + mean: 113, + min: 70, + max: 200, + stdDev: 22, + count: 576, + }); + }); + + it("converts day-bucketed glucose for a mmol/L-preference user", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.user.findUnique).mockResolvedValue({ + glucoseUnit: "mmol/L", + } as never); + vi.mocked(prisma.$queryRaw) + .mockResolvedValueOnce([ + { + bucket_start: new Date("2025-01-01T00:00:00Z"), + mean: 100, + min_value: 70, + max_value: 180, + }, + ] as never) + .mockResolvedValueOnce([ + { n: 288, mean: 100, min: 70, max: 180, sd: 18 }, + ] as never); + + const res = await GET(req("kind=glucose&days=3650")); + const body = (await res.json()) as { + data: { + unit: string; + points: Array<{ value: number }>; + stats: { mean: number }; + }; + }; + expect(body.data.unit).toBe("mmol/L"); + expect(body.data.points[0].value).toBe(5.5); // 100 mg/dL → 5.5 mmol/L + expect(body.data.stats.mean).toBe(5.5); + }); + + it("day-buckets pulse beyond 90 days with the day's min/max band", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.$queryRaw) + .mockResolvedValueOnce([ + { + bucket_start: new Date("2025-06-01T00:00:00Z"), + mean: 71.666, + min_value: 52, + max_value: 143, + }, + ] as never) + .mockResolvedValueOnce([ + { n: 24, mean: 71.666, min: 58, max: 132, sd: 9.876 }, + ] as never); + + const res = await GET(req("kind=pulse&days=365")); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { + points: Array<{ + id: string; + value: number; + valueMin: number | null; + valueMax: number | null; + }>; + stats: { mean: number; stdDev: number; count: number }; + }; + }; + expect(prisma.measurement.findMany).not.toHaveBeenCalled(); + expect(body.data.points).toEqual([ + { + id: "day:2025-06-01", + at: "2025-06-01T00:00:00.000Z", + value: 71.67, + secondary: null, + valueMin: 52, + valueMax: 143, + }, + ]); + expect(body.data.stats.mean).toBe(71.67); + expect(body.data.stats.stdDev).toBe(9.88); + expect(body.data.stats.count).toBe(24); + }); + + it("keeps the 90-day window on the raw path (byte-identical short ranges)", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([ + { id: "g1", value: 100, measuredAt: new Date("2026-06-01T08:00:00Z") }, + ] as never); + const res = await GET(req("kind=glucose&days=90")); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { points: Array<{ id: string }> }; + }; + // Raw read, individual row ids — the SQL bucket path never fires. + expect(prisma.$queryRaw).not.toHaveBeenCalled(); + expect(body.data.points[0].id).toBe("g1"); + }); + + it("keeps sparse kinds raw even on the ten-year window", async () => { + vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); + vi.mocked(prisma.measurement.findMany).mockResolvedValue([ + { id: "w1", value: 78.4, measuredAt: new Date() }, + ] as never); + const res = await GET(req("kind=weight&days=3650")); + expect(res.status).toBe(200); + const body = (await res.json()) as { + data: { points: Array<{ id: string }> }; + }; + expect(prisma.$queryRaw).not.toHaveBeenCalled(); + expect(body.data.points[0].id).toBe("w1"); + }); +}); + describe("GET /api/measurements/series — 422 multi-issue (v1.4.43 W6)", () => { it("surfaces TWO simultaneous validation errors", async () => { vi.mocked(getSession).mockResolvedValue(SESSION_OK as never); diff --git a/src/app/api/measurements/series/route.ts b/src/app/api/measurements/series/route.ts index ca27049ec..8dfc62b1e 100644 --- a/src/app/api/measurements/series/route.ts +++ b/src/app/api/measurements/series/route.ts @@ -35,6 +35,21 @@ import { convertGlucose, resolveGlucoseUnit } from "@/lib/glucose"; */ const SLEEP_SERIES_MAX_DAYS = 365; +/** + * v1.28.25 — sample-dense kinds read raw rows only up to this window. + * A CGM account stores ~288 BLOOD_GLUCOSE rows/day (105k+ rows/year) and + * per-sample PULSE piles up similarly, so the 365 / 3650-day requests + * walked a six-figure row set into JS, mapped it twice, and serialized + * everything. Mirrors the sleep precedent above, but instead of silently + * clamping the window, requests beyond the cap keep their full range and + * are day-bucketed in SQL (one point per day, mean + min/max band) — the + * chart still shows the whole requested history, just at daily grain. + * The short windows users actually browse (7 / 30 / 90 d) stay raw and + * byte-identical. Sparse kinds (weight, BP, …) are untouched. + */ +const DENSE_SERIES_RAW_WINDOW_DAYS = 90; +const DENSE_SERIES_KINDS: ReadonlySet = new Set(["glucose", "pulse"]); + const kindEnum = z.enum([ "weight", "bloodPressure", @@ -204,7 +219,9 @@ export const GET = apiHandler(async (request: NextRequest) => { // v1.16.16 — when glucose stats are computed over RAW mg/dL (parity with the // detail page + FHIR), the branch fills this so the response below skips the // generic over-points path that would double-round mmol/L figures. - let glucoseStats: { + // v1.28.25 — the dense day-bucket branch also fills it (stats stay + // aggregated over the RAW rows in SQL, not over the day-bucket points). + let statsOverride: { mean: number; min: number; max: number; @@ -306,24 +323,155 @@ export const GET = apiHandler(async (request: NextRequest) => { }), ]); - // Pair by closest timestamp within ±5 minutes. + // Pair by closest timestamp within ±5 minutes. Both reads arrive + // sorted ascending, so the nearest-diastolic index is non-decreasing + // across the systolic walk — a two-pointer merge finds each pair in + // O(sys + dia) instead of the previous full diastolic rescan per + // systolic row (O(sys × dia) — quadratic on dense imports). Ties keep + // the earlier diastolic row, matching the old scan's strict-`<` keep. const PAIR_WINDOW_MS = 5 * 60_000; + let diaIdx = 0; points = sys.map((s) => { - let bestDia: { value: number; dt: number } | null = null; - for (const d of dia) { - const dt = Math.abs(d.measuredAt.getTime() - s.measuredAt.getTime()); - if (dt > PAIR_WINDOW_MS) continue; - if (!bestDia || dt < bestDia.dt) { - bestDia = { value: d.value, dt }; - } + const sysMs = s.measuredAt.getTime(); + while ( + diaIdx + 1 < dia.length && + Math.abs(dia[diaIdx + 1].measuredAt.getTime() - sysMs) < + Math.abs(dia[diaIdx].measuredAt.getTime() - sysMs) + ) { + diaIdx += 1; } + const best = dia[diaIdx]; + const paired = + best !== undefined && + Math.abs(best.measuredAt.getTime() - sysMs) <= PAIR_WINDOW_MS; return { id: s.id, at: s.measuredAt.toISOString(), value: s.value, - secondary: bestDia?.value ?? null, + secondary: paired ? best.value : null, }; }); + } else if ( + DENSE_SERIES_KINDS.has(kind) && + days > DENSE_SERIES_RAW_WINDOW_DAYS + ) { + // v1.28.25 — long-window read of a sample-dense kind (CGM glucose, + // per-sample / hourly pulse). Day-bucket in SQL instead of walking + // every raw row into JS: one aggregate pass in Postgres returns at + // most `days` rows regardless of sample density. Parameter-bound + // tagged-template `$queryRaw`, mirroring the rollup tier's + // `date_trunc(... ) GROUP BY` aggregate (measurement-rollups.ts). + const type = KIND_TO_TYPE[kind]; + const bucketRows = await prisma.$queryRaw< + Array<{ + bucket_start: Date; + mean: number; + min_value: number; + max_value: number; + }> + >` + SELECT + date_trunc('day', m."measured_at") AS bucket_start, + AVG(m."value")::double precision AS mean, + MIN(COALESCE(m."value_min", m."value"))::double precision AS min_value, + MAX(COALESCE(m."value_max", m."value"))::double precision AS max_value + FROM measurements m + WHERE m."user_id" = ${user.id} + AND m."type" = ${type}::"measurement_type" + AND m."measured_at" >= ${since} + AND m."deleted_at" IS NULL + GROUP BY date_trunc('day', m."measured_at") + ORDER BY bucket_start ASC + `; + // Stats stay aggregated over the RAW rows (not the day buckets) so the + // strip's mean/min/max/stdDev/count are the same figures the raw path + // computed: `AVG`/`MIN`/`MAX` over `value`, `STDDEV_POP` matching the + // JS population-variance helper, exact row count. Aggregate-only — + // no row transfer. + const [rawAgg] = await prisma.$queryRaw< + Array<{ + n: number; + mean: number | null; + min: number | null; + max: number | null; + sd: number | null; + }> + >` + SELECT + COUNT(*)::int AS n, + AVG(m."value")::double precision AS mean, + MIN(m."value")::double precision AS min, + MAX(m."value")::double precision AS max, + STDDEV_POP(m."value")::double precision AS sd + FROM measurements m + WHERE m."user_id" = ${user.id} + AND m."type" = ${type}::"measurement_type" + AND m."measured_at" >= ${since} + AND m."deleted_at" IS NULL + `; + const round2 = (v: number) => Math.round(v * 100) / 100; + const dayId = (d: Date) => `day:${d.toISOString().slice(0, 10)}`; + if (kind === "glucose") { + // Same unit engine as the raw glucose branch: canonical mg/dL in + // the DB, converted ONCE at serialization to the user's preference. + const profile = await prisma.user.findUnique({ + where: { id: user.id }, + select: { glucoseUnit: true }, + }); + const glucoseUnit = resolveGlucoseUnit(profile?.glucoseUnit ?? null); + unit = glucoseUnit; + points = bucketRows.map((r) => ({ + id: dayId(r.bucket_start), + at: r.bucket_start.toISOString(), + value: convertGlucose(r.mean, glucoseUnit), + secondary: null, + })); + if ( + rawAgg !== undefined && + rawAgg.n > 0 && + rawAgg.mean !== null && + rawAgg.min !== null && + rawAgg.max !== null + ) { + // Mirrors the raw branch's v1.16.16 parity: round the mg/dL + // aggregate to 2 decimals (summarize()'s convention), then + // convert each figure once. + statsOverride = { + mean: convertGlucose(round2(rawAgg.mean), glucoseUnit), + min: convertGlucose(rawAgg.min, glucoseUnit), + max: convertGlucose(rawAgg.max, glucoseUnit), + stdDev: convertGlucose(round2(rawAgg.sd ?? 0), glucoseUnit), + count: rawAgg.n, + }; + } + } else { + // pulse — `value` is the day's average; valueMin/valueMax carry the + // day's low/high band (folding each hourly bucket's own spread via + // the COALESCE above), same band semantics as the per-hour shape. + points = bucketRows.map((r) => ({ + id: dayId(r.bucket_start), + at: r.bucket_start.toISOString(), + value: round2(r.mean), + secondary: null, + valueMin: r.min_value, + valueMax: r.max_value, + })); + if ( + rawAgg !== undefined && + rawAgg.n > 0 && + rawAgg.mean !== null && + rawAgg.min !== null && + rawAgg.max !== null + ) { + statsOverride = { + mean: round2(rawAgg.mean), + min: rawAgg.min, + max: rawAgg.max, + stdDev: round2(rawAgg.sd ?? 0), + count: rawAgg.n, + }; + } + } } else { const type = KIND_TO_TYPE[kind]; // v1.19.2 (iOS #34 extension) — pull the per-bucket spread for the @@ -371,7 +519,7 @@ export const GET = apiHandler(async (request: NextRequest) => { // already-converted+rounded points double-rounds the mean and stdDev // for mmol/L users, drifting the last decimal off the detail page. // Single-reading sets stay identical (100 → 5.5). Hand back stats here - // so the shared block below leaves a populated `glucoseStats` alone. + // so the shared block below leaves a populated `statsOverride` alone. const rawDataPoints: DataPoint[] = rows.map((r) => ({ date: r.measuredAt, value: r.value, @@ -379,7 +527,7 @@ export const GET = apiHandler(async (request: NextRequest) => { if (rawDataPoints.length > 0) { const rawSummary = summarize(rawDataPoints); const rawValues = rawDataPoints.map((p) => p.value); - glucoseStats = { + statsOverride = { mean: rawSummary.mean === null ? 0 @@ -443,7 +591,7 @@ export const GET = apiHandler(async (request: NextRequest) => { unit, points, stats: - glucoseStats ?? + statsOverride ?? (summary ? { mean: summary.mean, diff --git a/src/app/api/mood-entries/bulk/__tests__/route.test.ts b/src/app/api/mood-entries/bulk/__tests__/route.test.ts index 1946d171e..8742b60d8 100644 --- a/src/app/api/mood-entries/bulk/__tests__/route.test.ts +++ b/src/app/api/mood-entries/bulk/__tests__/route.test.ts @@ -413,3 +413,71 @@ describe("POST /api/mood-entries/bulk — structured tagKeys (v1.12.0)", () => { expect(json.data.entries[1].status).toBe("inserted"); }); }); + +describe("POST /api/mood-entries/bulk — tombstone suppression", () => { + it("keeps a tombstoned match deleted: true no-op reported as duplicate", async () => { + // A soft-deleted row under the probe key: the sync feed already + // surfaced the deletion to paired clients, so a stale offline re-post + // must NOT resurrect it — and must not churn the hidden row either. + vi.mocked(prisma.moodEntry.findUnique).mockResolvedValue({ + id: "entry-tomb", + deletedAt: new Date("2026-05-01T00:00:00.000Z"), + } as never); + const res = await POST( + postReq({ + entries: [ + { + mood: "GUT", + moodLoggedAt: "2026-05-16T08:00:00.000Z", + externalId: "ios-uuid-tomb", + tagKeys: ["movies"], + }, + ], + }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { + data: { + inserted: number; + duplicates: number; + entries: Array<{ status: string; id?: string; externalId?: string }>; + }; + }; + expect(json.data.inserted).toBe(0); + expect(json.data.duplicates).toBe(1); + expect(json.data.entries[0].status).toBe("duplicate"); + expect(json.data.entries[0].id).toBe("entry-tomb"); + expect(json.data.entries[0].externalId).toBe("ios-uuid-tomb"); + // No write at all on the hidden row: no upsert (value churn / + // updatedAt bump), no tag-link insert. + expect(prisma.moodEntry.upsert).not.toHaveBeenCalled(); + expect(createTagLinks).not.toHaveBeenCalled(); + }); + + it("still refreshes a LIVE match in place (duplicate + upsert runs)", async () => { + vi.mocked(prisma.moodEntry.findUnique).mockResolvedValue({ + id: "entry-live", + deletedAt: null, + } as never); + vi.mocked(prisma.moodEntry.upsert).mockResolvedValue({ + id: "entry-live", + } as never); + const res = await POST( + postReq({ + entries: [ + { + mood: "OKAY", + moodLoggedAt: "2026-05-16T08:00:00.000Z", + externalId: "ios-uuid-live", + }, + ], + }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { + data: { duplicates: number }; + }; + expect(json.data.duplicates).toBe(1); + expect(prisma.moodEntry.upsert).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/app/api/mood-entries/bulk/route.ts b/src/app/api/mood-entries/bulk/route.ts index 23779e338..7afe11bd4 100644 --- a/src/app/api/mood-entries/bulk/route.ts +++ b/src/app/api/mood-entries/bulk/route.ts @@ -227,9 +227,28 @@ async function postBulk(request: NextRequest): Promise { // grows. const existing = await prisma.moodEntry.findUnique({ where: probeWhere, - select: { id: true }, + select: { id: true, deletedAt: true }, }); + // Tombstone suppression: a soft-deleted match stays deleted. The + // user-facing DELETE route flips `deletedAt` so the `/api/sync/changes` + // feed surfaces the deletion to paired clients offline at delete time + // (see the MoodEntry schema note) — an offline client's later re-post + // of the same entry is stale state, not a recreation, and must not + // resurrect the row. True no-op: no value churn, no `updatedAt` bump, + // no tag-link writes on the hidden row; report `duplicate` so the + // client checkpoints past it exactly like a live-row match. + if (existing?.deletedAt) { + duplicates += 1; + results.push({ + index: i, + status: "duplicate", + id: existing.id, + ...(entry.externalId ? { externalId: entry.externalId } : {}), + }); + continue; + } + const result = await prisma.moodEntry.upsert({ where: probeWhere, create: { diff --git a/src/lib/doctor-report-data.ts b/src/lib/doctor-report-data.ts index 906e70458..5c0e0734d 100644 --- a/src/lib/doctor-report-data.ts +++ b/src/lib/doctor-report-data.ts @@ -866,6 +866,40 @@ export async function collectDoctorReportData( const recoveryEnabled = moduleMap.recovery !== false; const workoutsEnabled = moduleMap.workouts !== false; + // v1.28.25 — push the section / module gates INTO the measurement query. + // `filterMeasurementKeys` below strips gated types from the returned + // payload anyway, so rows of a disabled section / module were fetched only + // to be dropped — pure dead weight, and the heaviest series (BLOOD_GLUCOSE + // minute-grain CGM, gated by the glucose module) is exactly the one that + // runs to six figures over a 730-day window. Excluding them at the query + // is output-identical by construction. + // + // WEIGHT is deliberately NEVER excluded: the BMI figure (gated by + // `sections.bmi`, not `sections.weight`) and the GLP-1 weight-delta both + // read the PRE-filter `byType` / `stats` maps, so weight rows must be + // present even when the weight section itself is toggled off. No other + // gated type is read before the filter (each remaining pre-filter read — + // sleep nights, glucose panel, recovery summary — sits behind the same + // gate that drives its exclusion here). + // + // NOTE a closed include-list is NOT possible here: every ungated type in + // the window lands in `stats` / `measurements`, which the clinician share + // view and the MCP doctor-visit summary iterate wholesale + // (`Object.entries(data.stats)`). Dense ENABLED types (BLOOD_GLUCOSE + // minute-grain for a glucose-module user, hourly PULSE) therefore remain a + // known load — the per-day bucket tier is the follow-up for that. + const excludedMeasurementTypes: MeasurementType[] = []; + if (sections.bp === false) { + excludedMeasurementTypes.push("BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA"); + } + if (sections.pulse === false) excludedMeasurementTypes.push("PULSE"); + if (sections.sleep === false) excludedMeasurementTypes.push("SLEEP_DURATION"); + for (const [type, moduleKey] of Object.entries(MEASUREMENT_TYPE_MODULE)) { + if (moduleMap[moduleKey] === false) { + excludedMeasurementTypes.push(type as MeasurementType); + } + } + const [measurements, medications, intakeEvents, moodEntries, userProfile] = await Promise.all([ prisma.measurement.findMany({ @@ -876,8 +910,27 @@ export async function collectDoctorReportData( userId, measuredAt: { gte: start, lte: end }, deletedAt: null, + ...(excludedMeasurementTypes.length > 0 + ? { type: { notIn: excludedMeasurementTypes } } + : {}), }, orderBy: { measuredAt: "asc" }, + // v1.28.25 — narrow select. The collector reads exactly these seven + // fields (canonical-source collapse: type / value / measuredAt / + // source / deviceType; sleep-night reconstruction adds sleepStage; + // the glucose panel adds glucoseContext). The full-width read pulled + // every column — including the notesEncrypted Bytes — across up to + // 730 days of rows, which on a CGM + per-sample-HR account is the + // v1.28.2x six-figure-row incident class. + select: { + type: true, + value: true, + measuredAt: true, + source: true, + deviceType: true, + sleepStage: true, + glucoseContext: true, + }, }), prisma.medication.findMany({ where: { userId, active: true }, diff --git a/src/lib/export/full-backup-payload.ts b/src/lib/export/full-backup-payload.ts index 27357f1e6..2213f36e2 100644 --- a/src/lib/export/full-backup-payload.ts +++ b/src/lib/export/full-backup-payload.ts @@ -12,6 +12,7 @@ */ import type { PrismaClient } from "@/generated/prisma/client"; import { readNote } from "@/lib/crypto/note-cipher"; +import { findMeasurementsPaged } from "@/lib/export/paged-measurements"; import { BACKUP_SCHEMA_VERSION } from "@/lib/validations/backup"; import { buildCycleBackupSection } from "@/lib/cycle/backup"; @@ -39,10 +40,28 @@ export async function buildFullBackupPayload( ): Promise { const [measurements, medications, intakeEvents, moodEntries, cycle] = await Promise.all([ - prisma.measurement.findMany({ - where: { userId, deletedAt: null }, - orderBy: { measuredAt: "desc" }, - }), + // v1.28.25 — keyset-paginated read with a narrow select. The backup + // is inherently whole-history, so on a CGM / per-sample-HR account the + // previous single full-width `findMany` materialised a six-figure row + // set in one shot. The chunked read accumulates the same rows in the + // same `measuredAt desc` order; the payload FORMAT below is untouched + // (restore compatibility). The select is exactly the fields the + // serializer reads: type / value / unit / measuredAt / source + the + // note columns `readNote` decrypts + the `id` cursor key. + findMeasurementsPaged( + prisma, + { userId, deletedAt: null }, + { + id: true, + type: true, + value: true, + unit: true, + measuredAt: true, + source: true, + notes: true, + notesEncrypted: true, + }, + ), prisma.medication.findMany({ where: { userId }, select: { diff --git a/src/lib/export/paged-measurements.ts b/src/lib/export/paged-measurements.ts new file mode 100644 index 000000000..6e1f25dd9 --- /dev/null +++ b/src/lib/export/paged-measurements.ts @@ -0,0 +1,53 @@ +/** + * Keyset-paginated measurement reader for the export surfaces. + * + * The legacy exports (`GET /api/export`, `GET /api/export/measurements`, the + * full-backup builder) read the user's ENTIRE measurement table in one + * `findMany`. On a CGM + per-sample-HR account that is a six-figure row set + * materialised by a single query — the v1.28.2x incident class. This helper + * keeps the output row set (and its `measuredAt desc` order) identical while + * bounding each database round-trip to `MEASUREMENT_EXPORT_PAGE_SIZE` rows: + * the driver never buffers one giant result, and the event loop yields + * between chunks. + * + * Ordering: `measuredAt desc, id desc`. The exports always sorted + * `measuredAt desc`; the `id` tie-breaker makes the keyset cursor total (two + * rows can share a timestamp) — for equal-timestamp rows the previous order + * was unspecified heap order, so pinning it is not a behaviour change any + * consumer could have relied on. + * + * The `select` MUST include `id` (the cursor key); the generic constraint + * enforces it at compile time. + */ +import type { Prisma, PrismaClient } from "@/generated/prisma/client"; + +export const MEASUREMENT_EXPORT_PAGE_SIZE = 10_000; + +export async function findMeasurementsPaged< + S extends Prisma.MeasurementSelect & { id: true }, +>( + db: PrismaClient, + where: Prisma.MeasurementWhereInput, + select: S, + pageSize: number = MEASUREMENT_EXPORT_PAGE_SIZE, +): Promise>> { + const rows: Array> = []; + let cursorId: string | null = null; + for (;;) { + const batch: Array> = + (await db.measurement.findMany({ + where, + orderBy: [{ measuredAt: "desc" }, { id: "desc" }], + take: pageSize, + ...(cursorId !== null ? { cursor: { id: cursorId }, skip: 1 } : {}), + select, + })) as Array>; + // Loop, not `push(...batch)` — a 10k-element spread is fine today, but + // this file exists because of stack-overflow-by-spread regressions on + // large arrays (v1.28.22 class); keep the pattern boring. + for (const row of batch) rows.push(row); + if (batch.length < pageSize) break; + cursorId = (batch[batch.length - 1] as { id: string }).id; + } + return rows; +} diff --git a/src/lib/fitbit/__tests__/client.test.ts b/src/lib/fitbit/__tests__/client.test.ts index 483bf42d5..2297419c5 100644 --- a/src/lib/fitbit/__tests__/client.test.ts +++ b/src/lib/fitbit/__tests__/client.test.ts @@ -538,8 +538,11 @@ describe("sleep mapping (1.2 levels.data)", () => { const sessions = readSleepSessions({ sleep: [ { - // No logId → the anchor falls back to the END instant, which must - // also be tz-resolved so the externalId is stable across syncs. + // No logId → the anchor falls back to the START instant (the + // session's identity; the end moves when Fitbit re-scores a night), + // which must also be tz-resolved so the externalId is stable across + // syncs. endTime is present to prove start wins the fallback. + startTime: "2026-05-10T23:00:00.000", endTime: "2026-05-11T00:30:00.000", levels: { data: [ @@ -556,8 +559,8 @@ describe("sleep mapping (1.2 levels.data)", () => { const out = mapSleepSession(sessions[0]!, "Europe/Berlin"); expect(out).toHaveLength(1); - // anchor = end instant ISO (UTC) → fieldTag starts with that instant. - expect(out[0]!.fieldTag).toBe("2026-05-10T22:30:00.000Z:sleep_rem:0"); + // anchor = start instant ISO (UTC) → fieldTag starts with that instant. + expect(out[0]!.fieldTag).toBe("2026-05-10T21:00:00.000Z:sleep_rem:0"); }); }); diff --git a/src/lib/fitbit/__tests__/sync-upsert-resurrect.test.ts b/src/lib/fitbit/__tests__/sync-upsert-resurrect.test.ts new file mode 100644 index 000000000..c17fff78c --- /dev/null +++ b/src/lib/fitbit/__tests__/sync-upsert-resurrect.test.ts @@ -0,0 +1,123 @@ +/** + * Pins the tombstone-resurrect contract of `upsertFitbitMeasurements`. + * + * The measurements unique index on `(userId, type, source, externalId)` is + * FULL — it covers soft-deleted rows — so a tombstoned row permanently owns + * its key. The probe must therefore match tombstoned rows too and route them + * into the UPDATE branch with `deletedAt: null`: treating them as absent plans + * an insert that `skipDuplicates` drops SILENTLY against the tombstone's key, + * wedging the key forever (the same wedge shipped live on the Google + * transport). + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { findManyMock, createManyMock, updateMock } = vi.hoisted(() => ({ + findManyMock: vi.fn(async () => [] as unknown[]), + createManyMock: vi.fn(async () => ({ count: 0 })), + updateMock: vi.fn(async () => ({})), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { + measurement: { + findMany: findManyMock, + createMany: createManyMock, + update: updateMock, + }, + }, +})); +vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() })); +vi.mock("@/lib/integrations/status", () => ({ + isReauthRequired: vi.fn(async () => false), + recordSyncFailure: vi.fn(async () => {}), + recordSyncSuccess: vi.fn(async () => {}), +})); +vi.mock("@/lib/rollups/measurement-rollups", () => ({ + collapseToTypeDayKeys: vi.fn(() => []), + recomputeBucketsForMeasurement: vi.fn(async () => {}), + recomputeUserRollups: vi.fn(async () => {}), +})); +vi.mock("@/lib/insights/comprehensive-generate", () => ({ + invalidateStatusInsightsForTypes: vi.fn(async () => {}), +})); +vi.mock("@/lib/integrations/oauth-refresh", () => ({ + persistRotatedToken: vi.fn(async () => {}), +})); +vi.mock("../credentials", () => ({ + getUserFitbitCredentials: vi.fn(async () => null), +})); +vi.mock("../client", () => ({ refreshAccessToken: vi.fn() })); + +import { upsertFitbitMeasurements } from "../sync"; + +const STEPS_READING = { + type: "ACTIVITY_STEPS", + value: 8123, + unit: "steps", + measuredAt: new Date("2026-07-08T00:00:00.000Z"), + externalId: "stats:steps:2026-07-08", +}; + +beforeEach(() => { + findManyMock.mockReset().mockResolvedValue([]); + createManyMock.mockReset().mockResolvedValue({ count: 0 }); + updateMock.mockReset().mockResolvedValue({}); +}); + +describe("upsertFitbitMeasurements — tombstones resurrect", () => { + it("probes WITHOUT a deletedAt filter (a tombstoned row must be matched)", async () => { + await upsertFitbitMeasurements("user-1", [STEPS_READING], { + deferRollup: true, + }); + const arg = (findManyMock.mock.calls[0] as unknown[])[0] as { + where: Record; + }; + expect(arg.where).toEqual({ + userId: "user-1", + source: "FITBIT", + externalId: { in: ["stats:steps:2026-07-08"] }, + }); + expect(arg.where).not.toHaveProperty("deletedAt"); + }); + + it("routes a matched (tombstoned) row into an update that clears deletedAt", async () => { + findManyMock.mockResolvedValue([ + { + id: "row-1", + type: "ACTIVITY_STEPS", + externalId: "stats:steps:2026-07-08", + }, + ]); + + const { imported } = await upsertFitbitMeasurements( + "user-1", + [STEPS_READING], + { deferRollup: true }, + ); + + // No insert is planned for an owned key — the update resurrects in place. + expect(createManyMock).not.toHaveBeenCalled(); + expect(updateMock).toHaveBeenCalledTimes(1); + const upd = (updateMock.mock.calls[0] as unknown[])[0] as { + where: { id: string }; + data: Record; + }; + expect(upd.where).toEqual({ id: "row-1" }); + expect(upd.data.deletedAt).toBeNull(); + expect(upd.data.value).toBe(8123); + expect(upd.data.syncVersion).toEqual({ increment: 1 }); + expect(imported).toBe(1); + }); + + it("still creates a genuinely fresh key", async () => { + createManyMock.mockResolvedValue({ count: 1 }); + const { imported } = await upsertFitbitMeasurements( + "user-1", + [STEPS_READING], + { deferRollup: true }, + ); + expect(updateMock).not.toHaveBeenCalled(); + expect(createManyMock).toHaveBeenCalledTimes(1); + expect(imported).toBe(1); + }); +}); diff --git a/src/lib/fitbit/__tests__/sync.test.ts b/src/lib/fitbit/__tests__/sync.test.ts index 6d4047964..18d0b18ac 100644 --- a/src/lib/fitbit/__tests__/sync.test.ts +++ b/src/lib/fitbit/__tests__/sync.test.ts @@ -237,7 +237,7 @@ describe("getValidToken — rotating refresh", () => { }); }); -describe("upsertFitbitMeasurements — batched, tombstone-safe write", () => { +describe("upsertFitbitMeasurements — batched write, tombstones resurrect", () => { it("inserts a fresh reading via createMany with field-by-field data, source pinned to FITBIT", async () => { const measuredAt = new Date("2026-05-10T07:00:00.000Z"); prismaMock.measurement.findMany.mockResolvedValue([]); // nothing live yet @@ -255,16 +255,17 @@ describe("upsertFitbitMeasurements — batched, tombstone-safe write", () => { expect(imported).toBe(1); expect(prismaMock.measurement.update).not.toHaveBeenCalled(); - // The probe filters to LIVE FITBIT rows for the batch's externalIds. + // The probe matches EVERY FITBIT row for the batch's externalIds — live + // AND tombstoned (the full unique index owns the key either way). const probeArg = prismaMock.measurement.findMany.mock.calls[0]![0] as { where: Record; }; expect(probeArg.where).toMatchObject({ userId: "user1", source: "FITBIT", - deletedAt: null, externalId: { in: ["2026-05-10T07:00:00.000Z:weight"] }, }); + expect(probeArg.where).not.toHaveProperty("deletedAt"); const createArg = prismaMock.measurement.createMany.mock.calls[0]![0] as { data: Record[]; @@ -310,11 +311,14 @@ describe("upsertFitbitMeasurements — batched, tombstone-safe write", () => { expect(updateArg.data.syncVersion).toEqual({ increment: 1 }); }); - it("does NOT resurrect a soft-deleted row — a tombstone is absent from the live probe, so a fresh insert is created", async () => { + it("RESURRECTS a soft-deleted row — the probe matches the tombstone and the update clears deletedAt", async () => { const measuredAt = new Date("2026-05-10T07:00:00.000Z"); - // The user deleted this Fitbit reading: the tombstone (deletedAt set) is - // excluded from the live probe, so findMany returns NOTHING for the key. - prismaMock.measurement.findMany.mockResolvedValue([]); + // The tombstone still owns its key under the FULL unique index, so the + // probe returns it; planning an insert instead would be dropped silently + // by `skipDuplicates` and wedge the key forever. + prismaMock.measurement.findMany.mockResolvedValue([ + { id: "m-dead", type: "WEIGHT", externalId: "stats:weight:2026-05-10" }, + ]); const { imported } = await upsertFitbitMeasurements("user1", [ { @@ -326,12 +330,14 @@ describe("upsertFitbitMeasurements — batched, tombstone-safe write", () => { }, ]); - // The deleted row is NOT updated/resurrected — a fresh insert is attempted - // (the partial unique index `WHERE deleted_at IS NULL` keeps it from - // colliding with the dead row, and skipDuplicates guards a live race). expect(imported).toBe(1); - expect(prismaMock.measurement.update).not.toHaveBeenCalled(); - expect(prismaMock.measurement.createMany).toHaveBeenCalledTimes(1); + expect(prismaMock.measurement.createMany).not.toHaveBeenCalled(); + const updateArg = prismaMock.measurement.update.mock.calls[0]![0] as { + where: { id: string }; + data: Record; + }; + expect(updateArg.where).toEqual({ id: "m-dead" }); + expect(updateArg.data.deletedAt).toBeNull(); }); it("splits a mixed batch: matched-live → update, unmatched → createMany", async () => { diff --git a/src/lib/fitbit/client.ts b/src/lib/fitbit/client.ts index 1ca71cd70..8784e8afa 100644 --- a/src/lib/fitbit/client.ts +++ b/src/lib/fitbit/client.ts @@ -922,14 +922,19 @@ interface FitbitSleepSession { } /** - * The stable session anchor for a sleep externalId — the logId, else the end (or - * start) instant. A re-scored night reuses the same logId → overwrites in place. + * The stable session anchor for a sleep externalId — the logId, else the start + * (or end) instant. A re-scored night reuses the same logId → overwrites in + * place. The fallback prefers `startTime` because it is the session's identity; + * `endTime` moves when Fitbit revises a night's scoring, which would mint a new + * externalId for the same session. Pre-existing rows keyed on the old endTime + * fallback are rare (logId is normally present); a changed key simply creates + * one fresh row. */ function sleepSessionAnchor(s: FitbitSleepSession, tz?: string): string { if (typeof s.logId === "number" && Number.isFinite(s.logId)) { return String(s.logId); } - for (const t of [s.endTime, s.startTime]) { + for (const t of [s.startTime, s.endTime]) { if (typeof t === "string") { const d = parseLocalInstant(t, tz); if (d) return d.toISOString(); diff --git a/src/lib/fitbit/sync.ts b/src/lib/fitbit/sync.ts index 8411831df..0afa8adf8 100644 --- a/src/lib/fitbit/sync.ts +++ b/src/lib/fitbit/sync.ts @@ -382,16 +382,23 @@ const FITBIT_CREATE_CHUNK = 500; * (mirrors the WHOOP / Withings sync tail). Returns the count of rows written * plus the distinct `(type, day)` keys the write touched. * - * TOMBSTONE-SAFE: the live-row probe filters `deletedAt: null`, so a row the - * user soft-deleted (iOS LWW reconciler) is NOT matched — a fresh insert is - * created instead of resurrecting the tombstone on the next hourly sync. The - * partial unique index (`Measurement … WHERE deleted_at IS NULL`) keeps the - * fresh insert from colliding with the live set; the tombstone sits outside it. + * TOMBSTONES RESURRECT: the measurements unique index on + * `(userId, type, source, externalId)` is FULL — it covers soft-deleted rows + * too, so exactly ONE row can ever exist per key, live or tombstoned. An + * earlier revision assumed a partial (live-only) index and skipped tombstoned + * rows in the probe, expecting a fresh insert; in reality `createMany` with + * `skipDuplicates` then hit the tombstoned row's key and dropped the insert + * SILENTLY — once a Fitbit row was soft-deleted, that key could never import + * again (the same wedge shipped live on the Google transport). The probe now + * matches tombstoned rows as well and the update branch clears `deletedAt`: + * Fitbit remains the source of truth for its own rows, so a re-import + * deliberately revives a deleted one. An operator who wants Fitbit data gone + * permanently disconnects the integration rather than deleting rows. * * OVERWRITE CONTRACT preserved: a re-fetched daily summary (`stats:`-keyed and - * every other stable per-point anchor) maps to the SAME live `externalId`, so - * the probe finds the existing live row and takes the in-place update branch - * (bumping `syncVersion` for the iOS LWW reconciler) — never a duplicate. + * every other stable per-point anchor) maps to the SAME `externalId`, so the + * probe finds the existing row and takes the in-place update branch (bumping + * `syncVersion` for the iOS LWW reconciler) — never a duplicate. * * BATCHING: the existence probe is a single `findMany` over the batch's * externalIds; fresh rows go through chunked `createMany` (collapsing the N+1 @@ -411,22 +418,24 @@ export async function upsertFitbitMeasurements( }> { if (readings.length === 0) return { imported: 0, touched: [] }; - // Probe the LIVE rows (deletedAt: null) for every externalId in the batch in - // a single query. A tombstoned row deliberately does NOT appear here, so it is - // treated as absent → a fresh insert, not a resurrecting update. + // Probe EVERY existing row (live AND tombstoned) for the batch's externalIds + // in a single query. The full unique index guarantees at most one row per + // `(type, externalId)` key, and a tombstoned match must take the UPDATE + // branch (which clears `deletedAt`) — treating it as absent would plan an + // insert that `skipDuplicates` silently drops against the tombstone's key, + // permanently wedging the key (see TOMBSTONES RESURRECT above). const externalIds = readings.map((r) => r.externalId); - let liveByKey = new Map(); + let rowByKey = new Map(); try { const existing = await prisma.measurement.findMany({ where: { userId, source: "FITBIT", - deletedAt: null, externalId: { in: externalIds }, }, select: { id: true, type: true, externalId: true }, }); - liveByKey = new Map( + rowByKey = new Map( existing .filter((e) => e.externalId !== null) .map((e) => [ @@ -436,9 +445,9 @@ export async function upsertFitbitMeasurements( ); } catch (err) { // A probe failure must not strand the whole batch; fall back to treating - // every row as fresh (the partial unique index still rejects a genuine - // live-row collision, which the per-create catch swallows). - getEvent()?.addWarning(`Fitbit: live-row probe failed: ${err}`); + // every row as fresh (`skipDuplicates` absorbs the collision on the full + // unique index — those rows simply retry on the next sync tick). + getEvent()?.addWarning(`Fitbit: row probe failed: ${err}`); } const toCreate: Array<{ @@ -458,7 +467,7 @@ export async function upsertFitbitMeasurements( for (const r of readings) { const type = r.type as MeasurementType; const key = `${type}${DEDUP_KEY_DELIM}${r.externalId}`; - const live = liveByKey.get(key); + const live = rowByKey.get(key); if (live) { toUpdate.push({ id: live.id, r }); } else if (!plannedCreateKeys.has(key)) { @@ -520,8 +529,11 @@ export async function upsertFitbitMeasurements( } } - // Live-row overwrites: per-row update (differing values) on the live id, so - // the re-fetched daily summary overwrites in place and bumps `syncVersion`. + // Existing-row overwrites: per-row update (differing values) on the matched + // id, so the re-fetched daily summary overwrites in place and bumps + // `syncVersion`. `deletedAt: null` rides along unconditionally — a no-op on + // a live row, a deliberate RESURRECTION on a tombstoned one (Fitbit is the + // source of truth for its own rows; see TOMBSTONES RESURRECT above). for (const { id, r } of toUpdate) { try { await prisma.measurement.update({ @@ -531,6 +543,7 @@ export async function upsertFitbitMeasurements( unit: r.unit, measuredAt: r.measuredAt, sleepStage: r.sleepStage ?? null, + deletedAt: null, // Surface the server-side mutation to the iOS LWW reconciler. syncVersion: { increment: 1 }, }, @@ -551,7 +564,15 @@ export async function upsertFitbitMeasurements( // path keeps the inline per-day hook here (small touched set, warm next read). if (opts.deferRollup) { const tracker = rollupDeferStorage.getStore(); - if (tracker) tracker.keys.push(...touched); + // Plain loop, never `push(...touched)`: `touched` carries ONE entry PER + // READING, and a full-history batch on a sample-dense account (a + // multi-year heart-rate series is six figures of rows) blows the call + // stack when spread into a single call — the engine caps argument counts. + // The RangeError aborted the whole metrics collection on exactly the + // accounts with the most data. + if (tracker) { + for (const t of touched) tracker.keys.push(t); + } return { imported, touched }; } diff --git a/src/lib/gamification/achievements-result.ts b/src/lib/gamification/achievements-result.ts index 66c53b0ac..e6da30bc1 100644 --- a/src/lib/gamification/achievements-result.ts +++ b/src/lib/gamification/achievements-result.ts @@ -72,10 +72,29 @@ type MedicationScheduleRecord = { daysOfWeek: string | null; }; -type MeasurementRecord = { +/** + * v1.28.25 — one SQL-aggregated vitals bucket: every non-deleted, non-import + * row of `type` inside the Berlin-local hour `hour` of day `dayKey`, folded + * to a row count + value sum by the database. The vitals window is + * rollout-date → now and grows forever, and PULSE can be per-sample heart + * rate (six figures on a long-tenured wearable account) — reading raw rows + * here was the scale hazard. Every consumer's semantics survive the fold: + * green-day classification needs the per-day per-type MEAN (= Σsum / Σcount + * across the day's hour buckets — identical to the raw mean up to + * floating-point summation order), the count badges need row counts, the + * engagement / consistency series need day presence, and the hidden + * night-owl / early-bird / leap-day tallies need per-hour row counts. + */ +type VitalsHourBucket = { type: "WEIGHT" | "BLOOD_PRESSURE_SYS" | "BLOOD_PRESSURE_DIA" | "PULSE"; - value: number; - measuredAt: Date; + /** Berlin-local YYYY-MM-DD. */ + dayKey: string; + /** Berlin-local hour 0–23. */ + hour: number; + /** Number of raw rows folded into this bucket. */ + rowCount: number; + /** Sum of the raw rows' values (feeds the per-day mean). */ + valueSum: number; }; function maxDate(a: Date, b: Date): Date { @@ -123,11 +142,6 @@ function dayKeyToDate(dayKey: string): Date { return new Date(Date.UTC(year, month - 1, day, 12, 0, 0)); } -function mean(values: number[]): number { - if (values.length === 0) return 0; - return values.reduce((total, value) => total + value, 0) / values.length; -} - function findCountCompletionDate(dates: Date[], target: number): Date | null { if (dates.length < target) return null; const sorted = [...dates].sort((a, b) => a.getTime() - b.getTime()); @@ -195,66 +209,64 @@ function getEventDaySeries(dates: Date[]): { }; } +// v1.28.25 — consumes the SQL hour buckets instead of raw rows. The per-day +// per-type mean is Σ(valueSum) / Σ(rowCount) over the day's buckets — the +// same mean the raw-row version computed (up to floating-point summation +// order), feeding the identical classify → green-day → streak pipeline. function getHealthGreenDaySeries( - measurements: MeasurementRecord[], + buckets: VitalsHourBucket[], heightCm: number | null, - // v1.18.11 (W5 perf) — Berlin day-keys precomputed once per vitals row by - // the caller and passed in parallel to `measurements`. Falls back to a - // per-row `toBerlinDayKey` when absent so the standalone callers / tests - // keep working; the result is byte-identical either way. - dayKeys?: string[], ) { + type Acc = { sum: number; count: number }; const dayStats = new Map< string, - { - weight: number[]; - bpSys: number[]; - bpDia: number[]; - pulse: number[]; - } + { weight: Acc; bpSys: Acc; bpDia: Acc; pulse: Acc } >(); - for (let i = 0; i < measurements.length; i++) { - const measurement = measurements[i]; - const dayKey = dayKeys?.[i] ?? toBerlinDayKey(measurement.measuredAt); - const bucket = dayStats.get(dayKey) ?? { - weight: [], - bpSys: [], - bpDia: [], - pulse: [], + for (const b of buckets) { + const slot = dayStats.get(b.dayKey) ?? { + weight: { sum: 0, count: 0 }, + bpSys: { sum: 0, count: 0 }, + bpDia: { sum: 0, count: 0 }, + pulse: { sum: 0, count: 0 }, }; - - if (measurement.type === "WEIGHT") bucket.weight.push(measurement.value); - if (measurement.type === "BLOOD_PRESSURE_SYS") - bucket.bpSys.push(measurement.value); - if (measurement.type === "BLOOD_PRESSURE_DIA") - bucket.bpDia.push(measurement.value); - if (measurement.type === "PULSE") bucket.pulse.push(measurement.value); - - dayStats.set(dayKey, bucket); + const acc = + b.type === "WEIGHT" + ? slot.weight + : b.type === "BLOOD_PRESSURE_SYS" + ? slot.bpSys + : b.type === "BLOOD_PRESSURE_DIA" + ? slot.bpDia + : slot.pulse; + acc.sum += b.valueSum; + acc.count += b.rowCount; + dayStats.set(b.dayKey, slot); } const bmiGreenDays: string[] = []; const bpGreenDays: string[] = []; const pulseGreenDays: string[] = []; - for (const [dayKey, bucket] of dayStats.entries()) { - if (heightCm && bucket.weight.length > 0) { - const bmi = mean(bucket.weight) / (heightCm / 100) ** 2; + for (const [dayKey, slot] of dayStats.entries()) { + if (heightCm && slot.weight.count > 0) { + const bmi = slot.weight.sum / slot.weight.count / (heightCm / 100) ** 2; if (classifyBMI(bmi).severity === "normal") { bmiGreenDays.push(dayKey); } } - if (bucket.bpSys.length > 0 && bucket.bpDia.length > 0) { - const bpClass = classifyBP(mean(bucket.bpSys), mean(bucket.bpDia)); + if (slot.bpSys.count > 0 && slot.bpDia.count > 0) { + const bpClass = classifyBP( + slot.bpSys.sum / slot.bpSys.count, + slot.bpDia.sum / slot.bpDia.count, + ); if (bpClass.severity === "normal") { bpGreenDays.push(dayKey); } } - if (bucket.pulse.length > 0) { - const pulseClass = classifyPulse(mean(bucket.pulse)); + if (slot.pulse.count > 0) { + const pulseClass = classifyPulse(slot.pulse.sum / slot.pulse.count); if (pulseClass.severity === "normal") { pulseGreenDays.push(dayKey); } @@ -528,7 +540,7 @@ export async function buildAchievementsResult( const startDate = maxDate(user.createdAt, GAMIFICATION_ROLLOUT_AT); const [ - measurements, + vitalsBuckets, intakeEvents, medications, passkeys, @@ -538,25 +550,42 @@ export async function buildAchievementsResult( healthProfile, illnessEpisodes, ] = await Promise.all([ - prisma.measurement.findMany({ - where: { - userId, - measuredAt: { gte: startDate, lte: now }, - source: { not: "IMPORT" }, - type: { - in: ["WEIGHT", "BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA", "PULSE"], - }, - // v1.4.41 W-DELETED-2 — soft-deleted measurements are excluded - // from achievement progress so deleting a row immediately rolls - // back any streak / count badge it earned. - deletedAt: null, - }, - select: { - type: true, - value: true, - measuredAt: true, - }, - }), + // v1.28.25 — vitals folded to (type, Berlin day, Berlin hour) buckets in + // SQL instead of one row per sample. The window (rollout → now) grows + // forever and PULSE can be per-sample heart rate, so the raw read ran to + // six figures on long-tenured wearable accounts; the bucket set is + // bounded by 4 types × 24 hours × window days and every downstream + // metric derives exactly from (count, sum) per bucket — see the + // `VitalsHourBucket` doc for the semantics argument. Filters mirror the + // previous findMany exactly: the type allowlist is a closed literal set + // (never user input), source ≠ IMPORT, and soft-deleted rows stay + // excluded (v1.4.41 W-DELETED-2) so deleting a row immediately rolls + // back any streak / count badge it earned. `AT TIME ZONE 'UTC' AT TIME + // ZONE 'Europe/Berlin'` re-tags the naive UTC `measured_at` and converts + // it to the same Berlin wall clock `toBerlinDayKey` / `berlinHour` + // derive via Intl. Ordering pins day → hour → type for determinism. + prisma.$queryRaw` + SELECT + m."type"::text AS "type", + to_char( + (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE 'Europe/Berlin', + 'YYYY-MM-DD' + ) AS "dayKey", + EXTRACT( + HOUR FROM (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE 'Europe/Berlin' + )::int AS "hour", + COUNT(*)::int AS "rowCount", + SUM(m."value")::double precision AS "valueSum" + FROM "measurements" m + WHERE m."user_id" = ${userId} + AND m."measured_at" >= ${startDate} + AND m."measured_at" <= ${now} + AND m."source"::text <> 'IMPORT' + AND m."type"::text IN ('WEIGHT', 'BLOOD_PRESSURE_SYS', 'BLOOD_PRESSURE_DIA', 'PULSE') + AND m."deleted_at" IS NULL + GROUP BY 1, 2, 3 + ORDER BY 2, 3, 1 + `, prisma.medicationIntakeEvent.findMany({ where: { userId, @@ -711,21 +740,25 @@ export async function buildAchievementsResult( ...passwordLoginDates, ]); - // v1.18.11 (W5 perf) — the vitals array runs through several independent - // full-array passes, each previously re-deriving a Berlin day-key per row - // via `Intl.DateTimeFormat.formatToParts` (~16.8k rows × ~3 passes on a - // power-user account dominated the cold build). Compute each row's day-key - // ONCE here and thread the parallel array into every consumer; the values - // are identical to the per-pass derivation, so the output is byte-identical. - const vitalsDayKeys = (measurements as MeasurementRecord[]).map((m) => - toBerlinDayKey(m.measuredAt), - ); + // v1.28.25 — the buckets already carry the Berlin day-key + hour straight + // from SQL (superseding the v1.18.11 Intl-once optimisation: no Intl pass + // over the vitals remains at all). The parallel arrays thread into every + // consumer exactly as the per-row keys did; day-presence consumers dedup + // by key, so bucket-level duplicates (several hours / types on one day) + // are harmless. + const vitalsDayKeys = vitalsBuckets.map((b) => b.dayKey); + const vitalsHours = vitalsBuckets.map((b) => b.hour); + // Weighted records for the expansion metrics: one record per bucket, + // `count` carrying the folded row multiplicity. The `measuredAt` anchor is + // never read on this path (day-keys + hours ride the parallel arrays + // above) — it only satisfies the record shape. + const vitalsRecords = vitalsBuckets.map((b) => ({ + type: b.type, + measuredAt: dayKeyToDate(b.dayKey), + count: b.rowCount, + })); - const healthSeries = getHealthGreenDaySeries( - measurements as MeasurementRecord[], - user.heightCm, - vitalsDayKeys, - ); + const healthSeries = getHealthGreenDaySeries(vitalsBuckets, user.heightCm); const onTimeSeries = getOnTimePerfectDaySeries( intakeEvents, schedulesByMedicationId, @@ -746,8 +779,8 @@ export async function buildAchievementsResult( sleepMeasurements.map((m) => m.measuredAt), ); const weeklyConsistency = getWeeklyConsistency( - // v1.18.11 (W5 perf) — reuse the day-keys computed once above instead of - // a fresh per-row `toBerlinDayKey` pass over the full vitals array. + // Bucket day-keys — `getWeeklyConsistency` dedups to distinct days, so + // the per-hour multiplicity is irrelevant. vitalsDayKeys, MEASUREMENT_CONSISTENCY_MIN_DAYS_PER_WEEK, MEASUREMENT_CONSISTENCY_TARGET_WEEKS, @@ -762,14 +795,15 @@ export async function buildAchievementsResult( : 0; const expansionValues = buildExpansionMetricValues({ - measurements, + measurements: vitalsRecords, moodEntries, intakeEvents, auditEvents, - // v1.18.11 (W5 perf) — reuse the vitals day-keys computed once above so - // the engagement + hidden passes don't re-walk the full vitals array - // through `Intl.DateTimeFormat` a second and third time. + // The SQL-derived Berlin day-keys + hours ride in parallel to the + // weighted bucket records — the expansion passes never touch a bucket's + // representative `measuredAt`. measurementDayKeys: vitalsDayKeys, + measurementHours: vitalsHours, }); const earnability = getEarnabilityFlags({ hasMedication: medications.length > 0, diff --git a/src/lib/gamification/expansion-metrics.ts b/src/lib/gamification/expansion-metrics.ts index e96aa399c..34285f2b9 100644 --- a/src/lib/gamification/expansion-metrics.ts +++ b/src/lib/gamification/expansion-metrics.ts @@ -47,6 +47,17 @@ export interface MoodEntryRecord { export interface MeasurementRecord { type: string; measuredAt: Date; + /** + * v1.28.25 — row multiplicity. The achievements builder now reads vitals + * as SQL (day, hour, type) buckets instead of one row per sample (a + * per-sample PULSE history runs to six figures), so one record here can + * stand for N underlying rows. Every count-semantics consumer + * (`countMeasurementsByType`, the hidden-metrics tallies) weighs the + * record by `count`; day-presence consumers dedup by day key and are + * multiplicity-blind. Absent means 1 — raw per-sample callers and the + * existing tests are unchanged. + */ + count?: number; } export interface IntakeEventRecord { @@ -79,9 +90,10 @@ export function countMeasurementsByType( let bp = 0; let pulse = 0; for (const m of measurements) { - if (m.type === "WEIGHT") weight += 1; - else if (m.type === "BLOOD_PRESSURE_SYS") bp += 1; - else if (m.type === "PULSE") pulse += 1; + const n = m.count ?? 1; + if (m.type === "WEIGHT") weight += n; + else if (m.type === "BLOOD_PRESSURE_SYS") bp += n; + else if (m.type === "PULSE") pulse += n; } return { weightCount: weight, bpCount: bp, pulseCount: pulse }; } @@ -289,18 +301,22 @@ export function getHiddenMetrics(input: { let earlyBird = 0; let leapDay = 0; - const tally = (hour: number, dayKey: string): void => { - if (hour >= 2 && hour < 4) nightOwl += 1; - if (hour >= 4 && hour < 6) earlyBird += 1; + // v1.28.25 — `n` is the row multiplicity (`MeasurementRecord.count`): a + // bucketed vitals record stands for N raw rows, and these are COUNT + // metrics, so each contributes N. Mood / intake tallies stay per-row (n=1). + const tally = (hour: number, dayKey: string, n = 1): void => { + if (hour >= 2 && hour < 4) nightOwl += n; + if (hour >= 4 && hour < 6) earlyBird += n; // Feb 29 — only valid in leap years. - if (dayKey.endsWith("-02-29")) leapDay += 1; + if (dayKey.endsWith("-02-29")) leapDay += n; }; for (let i = 0; i < input.measurements.length; i++) { - const ts = input.measurements[i].measuredAt; + const m = input.measurements[i]; tally( - input.measurementHours?.[i] ?? berlinHour(ts), - input.measurementDayKeys?.[i] ?? toBerlinDayKey(ts), + input.measurementHours?.[i] ?? berlinHour(m.measuredAt), + input.measurementDayKeys?.[i] ?? toBerlinDayKey(m.measuredAt), + m.count ?? 1, ); } for (const e of input.moodEntries) { @@ -387,15 +403,26 @@ export function buildExpansionMetricValues(input: { * hidden passes so the full vitals array is `Intl`-walked once, not ~3×. */ measurementDayKeys?: string[]; + /** + * v1.28.25 — Berlin hours for `measurements`, precomputed by the caller. + * The bucketed-vitals path (SQL `(day, hour, type)` buckets) carries the + * Berlin hour straight from the query; deriving it here from a bucket's + * representative `measuredAt` would be wrong, so the caller MUST supply + * this alongside bucketed records. Raw per-row callers omit it and the + * hours derive from `measuredAt` exactly as before. + */ + measurementHours?: number[]; }): ExpansionMetrics { const counts = countMeasurementsByType(input.measurements); const mood = getMoodMetrics(input.moodEntries); // The hidden pass also needs the per-row Berlin hour; derive it once here // (only when the caller supplied day-keys, i.e. on the hot achievements // path) so a single hour pass replaces the per-pass re-derivation. - const measurementHours = input.measurementDayKeys - ? input.measurements.map((m) => berlinHour(m.measuredAt)) - : undefined; + const measurementHours = + input.measurementHours ?? + (input.measurementDayKeys + ? input.measurements.map((m) => berlinHour(m.measuredAt)) + : undefined); const engagement = getEngagementMetrics({ measurements: input.measurements, moodEntries: input.moodEntries, diff --git a/src/lib/google-health/__tests__/sync-failsoft.test.ts b/src/lib/google-health/__tests__/sync-failsoft.test.ts index e6c67f096..645d52efa 100644 --- a/src/lib/google-health/__tests__/sync-failsoft.test.ts +++ b/src/lib/google-health/__tests__/sync-failsoft.test.ts @@ -4,19 +4,51 @@ * failure and RETURNS — it must not rethrow, because every call site sits in a * per-collection catch and a rethrow would abort the sibling collections * (live: steps 400ing suppressed distance / floors / active-energy entirely). + * + * Also pins the dead-token verdict: `getValidToken`'s failure paths + * (credentials missing, refresh failure) must register on the cycle's + * hard-fail ledger, and a cycle whose token is dead must NOT stamp + * `markSynced` / `recordSyncSuccess` — otherwise every resource returns 0 + * without failures, the cycle reads as clean, `recordSyncSuccess` un-parks + * `error_reauth`, and the hourly cohort hammers the dead refresh token forever + * while `lastSyncedAt` advances past real data. */ import { beforeEach, describe, expect, it, vi } from "vitest"; -const { recordSyncFailureMock } = vi.hoisted(() => ({ +const { + recordSyncFailureMock, + recordSyncSuccessMock, + prismaMock, + getUserGoogleHealthCredentialsMock, + refreshAccessTokenMock, + resourceFake, +} = vi.hoisted(() => ({ recordSyncFailureMock: vi.fn(async () => {}), + recordSyncSuccessMock: vi.fn(async () => {}), + prismaMock: { + googleHealthConnection: { + findUnique: vi.fn(), + update: vi.fn(async () => ({})), + }, + }, + getUserGoogleHealthCredentialsMock: vi.fn(async (): Promise => null), + refreshAccessTokenMock: vi.fn(), + // The dead-token cycle test drives the real `syncUserGoogleHealth`; each + // resource module is mocked to do exactly what the real ones do first — + // resolve the token — so the ledger registration happens inside the cycle's + // ALS scope. + resourceFake: async (userId: string): Promise => { + const { getValidToken } = await import("../sync"); + return (await getValidToken(userId)) ? 1 : 0; + }, })); -vi.mock("@/lib/db", () => ({ prisma: {} })); +vi.mock("@/lib/db", () => ({ prisma: prismaMock })); vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() })); vi.mock("@/lib/integrations/status", () => ({ isReauthRequired: vi.fn(async () => false), recordSyncFailure: recordSyncFailureMock, - recordSyncSuccess: vi.fn(async () => {}), + recordSyncSuccess: recordSyncSuccessMock, })); vi.mock("@/lib/rollups/measurement-rollups", () => ({ collapseToTypeDayKeys: vi.fn(() => []), @@ -27,15 +59,44 @@ vi.mock("@/lib/insights/comprehensive-generate", () => ({ invalidateStatusInsightsForTypes: vi.fn(async () => {}), })); vi.mock("../credentials", () => ({ - getUserGoogleHealthCredentials: vi.fn(async () => null), + getUserGoogleHealthCredentials: getUserGoogleHealthCredentialsMock, })); -vi.mock("../client", () => ({ refreshAccessToken: vi.fn() })); +vi.mock("../client", () => ({ refreshAccessToken: refreshAccessTokenMock })); -import { handleCollectionFetchError } from "../sync"; +vi.mock("../sync-metrics", () => ({ syncUserMetrics: resourceFake })); +vi.mock("../sync-activity", () => ({ syncUserActivity: resourceFake })); +vi.mock("../sync-sleep", () => ({ syncUserSleep: resourceFake })); +vi.mock("../sync-workout", () => ({ syncUserWorkout: resourceFake })); + +import { + GOOGLE_HEALTH_TOKEN_HARD_FAIL, + getValidToken, + handleCollectionFetchError, + runWithGoogleHealthHardFailLedger, + syncUserGoogleHealth, +} from "../sync"; import { GoogleHealthApiError } from "../response-classifier"; +/** A connection whose access token is inside the 5-min refresh buffer. */ +const EXPIRED_CONNECTION = { + id: "conn-1", + userId: "user-1", + googleUserId: "g-user-1", + accessToken: "enc-access", + refreshToken: "enc-refresh", + tokenExpiresAt: new Date(Date.now() - 60_000), + lastSyncedAt: new Date("2026-07-01T00:00:00.000Z"), +}; + beforeEach(() => { recordSyncFailureMock.mockClear(); + recordSyncSuccessMock.mockClear(); + prismaMock.googleHealthConnection.findUnique.mockReset(); + prismaMock.googleHealthConnection.update + .mockReset() + .mockResolvedValue({} as never); + getUserGoogleHealthCredentialsMock.mockReset().mockResolvedValue(null); + refreshAccessTokenMock.mockReset(); }); describe("handleCollectionFetchError — fail-soft grouping", () => { @@ -89,3 +150,85 @@ describe("handleCollectionFetchError — fail-soft grouping", () => { ); }); }); + +describe("getValidToken — dead-token cycle verdict", () => { + it("registers the credentials-missing path on the hard-fail ledger", async () => { + prismaMock.googleHealthConnection.findUnique.mockResolvedValue( + EXPIRED_CONNECTION as never, + ); + + const { result, failures } = await runWithGoogleHealthHardFailLedger(() => + getValidToken("user-1"), + ); + + expect(result).toBeNull(); + expect(failures).toEqual([GOOGLE_HEALTH_TOKEN_HARD_FAIL]); + expect(recordSyncFailureMock).toHaveBeenCalledWith( + expect.objectContaining({ + userId: "user-1", + integration: "google-health", + kind: "reauth_required", + errorCode: "credentials_missing", + }), + ); + }); + + it("registers a refresh failure on the hard-fail ledger", async () => { + prismaMock.googleHealthConnection.findUnique.mockResolvedValue( + EXPIRED_CONNECTION as never, + ); + getUserGoogleHealthCredentialsMock.mockResolvedValue({ + clientId: "id", + clientSecret: "secret", + }); + refreshAccessTokenMock.mockRejectedValue( + new GoogleHealthApiError({ + verb: "refreshToken", + classification: "reauth_required", + httpStatus: 401, + reason: "invalid_grant", + }), + ); + + const { result, failures } = await runWithGoogleHealthHardFailLedger(() => + getValidToken("user-1"), + ); + + expect(result).toBeNull(); + expect(failures).toEqual([GOOGLE_HEALTH_TOKEN_HARD_FAIL]); + expect(recordSyncFailureMock).toHaveBeenCalledWith( + expect.objectContaining({ kind: "reauth_required" }), + ); + }); + + it("a dead-token cycle fails the verdict and never stamps markSynced / recordSyncSuccess", async () => { + // Every resource resolves the token, the refresh fails each time — before + // the ledger registration the cycle read as CLEAN (all resources returned + // 0 without failures), stamped the watermark, and un-parked error_reauth. + prismaMock.googleHealthConnection.findUnique.mockResolvedValue( + EXPIRED_CONNECTION as never, + ); + getUserGoogleHealthCredentialsMock.mockResolvedValue({ + clientId: "id", + clientSecret: "secret", + }); + refreshAccessTokenMock.mockRejectedValue( + new GoogleHealthApiError({ + verb: "refreshToken", + classification: "reauth_required", + httpStatus: 401, + reason: "invalid_grant", + }), + ); + + const res = await syncUserGoogleHealth("user-1"); + + expect(res).toEqual({ imported: 0, failed: true }); + expect(recordSyncSuccessMock).not.toHaveBeenCalled(); + // No connection write may carry the watermark stamp. + for (const call of prismaMock.googleHealthConnection.update.mock.calls) { + const arg = (call as unknown[])[0] as { data: Record }; + expect(arg.data).not.toHaveProperty("lastSyncedAt"); + } + }); +}); diff --git a/src/lib/google-health/__tests__/sync-upsert-resurrect.test.ts b/src/lib/google-health/__tests__/sync-upsert-resurrect.test.ts index d028ac07b..6df9b2e7a 100644 --- a/src/lib/google-health/__tests__/sync-upsert-resurrect.test.ts +++ b/src/lib/google-health/__tests__/sync-upsert-resurrect.test.ts @@ -118,3 +118,85 @@ describe("upsertGoogleHealthMeasurements — tombstones resurrect", () => { expect(imported).toBe(1); }); }); + +describe("upsertGoogleHealthMeasurements — no-op overwrite skip", () => { + it("skips the update entirely for an identical LIVE row (no syncVersion churn)", async () => { + findManyMock.mockResolvedValue([ + { + id: "row-1", + type: "ACTIVITY_STEPS", + externalId: "stats:steps:2026-07-08", + value: 8123, + unit: "steps", + measuredAt: new Date("2026-07-08T00:00:00.000Z"), + sleepStage: null, + deletedAt: null, + }, + ]); + + const { imported } = await upsertGoogleHealthMeasurements( + "user-1", + [STEPS_READING], + { deferRollup: true }, + ); + + // The 24 h overlap re-fetches this row hourly; an unchanged live row must + // not be rewritten (the unconditional syncVersion bump churned the DB and + // every iOS delta pull). + expect(updateMock).not.toHaveBeenCalled(); + expect(createManyMock).not.toHaveBeenCalled(); + expect(imported).toBe(0); + }); + + it("still writes when any payload field differs on a live row", async () => { + findManyMock.mockResolvedValue([ + { + id: "row-1", + type: "ACTIVITY_STEPS", + externalId: "stats:steps:2026-07-08", + value: 7999, // stale daily total — Google re-rolled the day upward + unit: "steps", + measuredAt: new Date("2026-07-08T00:00:00.000Z"), + sleepStage: null, + deletedAt: null, + }, + ]); + + const { imported } = await upsertGoogleHealthMeasurements( + "user-1", + [STEPS_READING], + { deferRollup: true }, + ); + + expect(updateMock).toHaveBeenCalledTimes(1); + expect(imported).toBe(1); + }); + + it("an identical TOMBSTONED row ALWAYS updates — the write is the resurrection", async () => { + findManyMock.mockResolvedValue([ + { + id: "row-1", + type: "ACTIVITY_STEPS", + externalId: "stats:steps:2026-07-08", + value: 8123, + unit: "steps", + measuredAt: new Date("2026-07-08T00:00:00.000Z"), + sleepStage: null, + deletedAt: new Date("2026-07-09T00:00:00.000Z"), + }, + ]); + + const { imported } = await upsertGoogleHealthMeasurements( + "user-1", + [STEPS_READING], + { deferRollup: true }, + ); + + expect(updateMock).toHaveBeenCalledTimes(1); + const upd = (updateMock.mock.calls[0] as unknown[])[0] as { + data: Record; + }; + expect(upd.data.deletedAt).toBeNull(); + expect(imported).toBe(1); + }); +}); diff --git a/src/lib/google-health/client.ts b/src/lib/google-health/client.ts index 31980d328..66d55658f 100644 --- a/src/lib/google-health/client.ts +++ b/src/lib/google-health/client.ts @@ -500,7 +500,8 @@ export const GOOGLE_HEALTH_DATA_TYPES = { timeField: "date", }, // `respiratory-rate` does not exist in the catalogue; the daily summary is - // `daily-respiratory-rate` (`dailyRespiratoryRateBpm`). + // `daily-respiratory-rate`, value leaf `dailyRespiratoryRate.breathsPerMinute` + // (the once-assumed `dailyRespiratoryRateBpm` leaf never existed). respiratoryRate: { path: "daily-respiratory-rate", filter: "daily_respiratory_rate", diff --git a/src/lib/google-health/mapping.md b/src/lib/google-health/mapping.md index 9b9470b4d..1e8615101 100644 --- a/src/lib/google-health/mapping.md +++ b/src/lib/google-health/mapping.md @@ -172,9 +172,19 @@ RFC-3339), `sleep.type` (`CLASSIC | STAGES`), `sleep.stages[]` `sleep.summary` (`minutesAsleep`/`minutesAwake`, int64 strings). HealthLog stores one `SLEEP_DURATION` row per stage SEGMENT, `measuredAt = that segment's END instant`, harmonised onto the shared `SleepStage` enum the night-total + -hypnogram readers consume. externalId = `:sleep_:` -(session `interval.endTime` ISO instant + indexed segment), so a re-scored night -overwrites in place. Stage map (`SLEEP_STAGE_TYPE` enum, lowercased): +hypnogram readers consume. externalId = `:sleep:` +(v1.28.18): the anchor prefers the DataPoint's invariant resource `name`, +falling back to the session `interval.endTime` (then `startTime`) parsed to a +UTC ISO instant, and the per-segment key is the segment's own START instant — +NOT a positional index and NOT the stage label, so a re-scored night overwrites +in place and a mere re-classification (LIGHT→DEEP on the same block) UPDATES +the row rather than orphaning it. After mapping, the sync runs a +replace-by-window cleanup (`replaceStaleGoogleHealthSleep`): any LIVE +`GOOGLE_HEALTH` `SLEEP_DURATION` row inside the session's +`[windowStart, windowEnd]` that this fetch did NOT re-produce is soft-deleted, +so segments an earlier scoring left behind (old volatile keys, dropped blocks) +never double-count the night total. Stage map (`SLEEP_STAGE_TYPE` enum, +lowercased): `LIGHT → CORE` ("light" ↔ Apple "core" shallow-NREM band), `DEEP → DEEP`, `REM → REM`, `AWAKE`/`RESTLESS → AWAKE`, classic `ASLEEP → ASLEEP`. `SLEEP_STAGE_TYPE_UNSPECIFIED` / unknown labels are skipped, not mis-bucketed. @@ -203,9 +213,19 @@ source ladder. `(userId, type, source: "GOOGLE_HEALTH", externalId)` unique. Spot/daily metric rows: `externalId = :`. Daily cumulative activity rows: `externalId = stats::` (Apple-Health overwrite shape). -Workouts: `(userId, source: "GOOGLE_HEALTH", externalId)` on the `Workout` table. -A re-fetch of the same window (daily summaries re-roll after the fact, so the -incremental overlap is 24 h) overwrites in place. +Sleep rows: `externalId = :sleep:` (see the sleep +bundle above). Workouts: `(userId, source: "GOOGLE_HEALTH", externalId)` on the +`Workout` table. A re-fetch of the same window (daily summaries re-roll after +the fact, so the incremental overlap is 24 h) overwrites in place. + +The measurements unique index is **FULL** — it covers soft-deleted rows too, so +exactly one row can ever exist per key, live or tombstoned. The upsert probe +therefore matches tombstoned rows as well, and a re-import **RESURRECTS** them +(the update branch clears `deletedAt`; v1.28.24) — Google is the source of +truth for its own rows, and treating a tombstone as absent would plan an insert +that `skipDuplicates` drops silently, wedging the key forever. An unchanged +LIVE row is skipped entirely (no write, no `syncVersion` bump) so the 24 h +overlap re-fetch doesn't churn the DB or the iOS delta pull. ## Error surfacing diff --git a/src/lib/google-health/sync-activity.ts b/src/lib/google-health/sync-activity.ts index f75388d07..b8e770557 100644 --- a/src/lib/google-health/sync-activity.ts +++ b/src/lib/google-health/sync-activity.ts @@ -128,17 +128,27 @@ export async function syncUserActivity( continue; } + // The mapper runs INSIDE a per-type catch too (mirrors sync-metrics): a + // single malformed point whose `resource.map(point)` throws must not + // escape the ROLLUP_RESOURCES loop and abort every sibling type ordered + // after it. Route a map throw through the same ledger as a fetch failure — + // record it, fail the cycle, and move on to the next type. const readings: GoogleHealthMeasurementUpsert[] = []; - for (const point of points) { - for (const m of resource.map(point)) { - readings.push({ - type: m.type, - value: m.value, - unit: m.unit, - measuredAt: m.measuredAt, - externalId: externalIdFor(m), - }); + try { + for (const point of points) { + for (const m of resource.map(point)) { + readings.push({ + type: m.type, + value: m.value, + unit: m.unit, + measuredAt: m.measuredAt, + externalId: externalIdFor(m), + }); + } } + } catch (err) { + imported += await handleCollectionFetchError(resource.verb, userId, err); + continue; } if (points.length === 0) { rollupEmptyResponse.push(resource.verb); diff --git a/src/lib/google-health/sync-sleep.ts b/src/lib/google-health/sync-sleep.ts index 5e7cf39ce..69104a4ed 100644 --- a/src/lib/google-health/sync-sleep.ts +++ b/src/lib/google-health/sync-sleep.ts @@ -29,13 +29,14 @@ import { import { getValidToken, handleCollectionFetchError, + noteHardFailure, replaceStaleGoogleHealthSleep, upsertGoogleHealthMeasurements, type GoogleHealthMeasurementUpsert, type GoogleHealthResourceSyncOptions, type GoogleHealthSleepReplaceWindow, } from "./sync"; -import { annotate } from "@/lib/logging/context"; +import { annotate, getEvent } from "@/lib/logging/context"; import { resolveUserTimezone } from "@/lib/tz/resolver"; export async function syncUserSleep( @@ -69,7 +70,20 @@ export async function syncUserSleep( const readings: GoogleHealthMeasurementUpsert[] = []; const replaceWindows: GoogleHealthSleepReplaceWindow[] = []; for (const point of points) { - const session = mapSleepSessionDetailed(point, tz); + // Per-session throw-guard (mapper parity with sync-metrics): a single + // malformed session must not abort the sibling sessions in the same page. + // The ledger entry still fails the cycle so the watermark holds and the + // bad point is re-fetched rather than silently lost past the overlap. + let session: ReturnType; + try { + session = mapSleepSessionDetailed(point, tz); + } catch (err) { + getEvent()?.addWarning( + `google-health: sleep session map failed for ${userId}: ${err}`, + ); + noteHardFailure("mapSleepSession"); + continue; + } if (session.rows.length === 0) continue; for (const m of session.rows) { readings.push({ diff --git a/src/lib/google-health/sync-workout.ts b/src/lib/google-health/sync-workout.ts index b49aee117..d4e6f3035 100644 --- a/src/lib/google-health/sync-workout.ts +++ b/src/lib/google-health/sync-workout.ts @@ -23,6 +23,7 @@ import { import { getValidToken, handleCollectionFetchError, + noteHardFailure, type GoogleHealthResourceSyncOptions, } from "./sync"; import { prisma } from "@/lib/db"; @@ -102,6 +103,10 @@ export async function syncUserWorkout( imported++; } catch (err) { getEvent()?.addWarning(`google-health: failed to upsert workout: ${err}`); + // A fetched-but-unwritten session is gone once it leaves the overlap + // window — fail the cycle's verdict so the watermark holds and the next + // tick re-fetches it. No rethrow: sibling sessions keep upserting. + noteHardFailure("workout:upsert"); } } diff --git a/src/lib/google-health/sync.ts b/src/lib/google-health/sync.ts index 0b77a52d6..675479a27 100644 --- a/src/lib/google-health/sync.ts +++ b/src/lib/google-health/sync.ts @@ -103,6 +103,12 @@ export async function getValidToken( message: "Google Health credentials missing — token refresh skipped", errorCode: "credentials_missing", }); + // A dead token means every resource "succeeds" with 0 rows — without a + // ledger entry the cycle would read as clean, stamp `markSynced`, and + // `recordSyncSuccess` would flip a parked connection back to connected + // (an hourly invalid-refresh hammer with an advancing watermark). Fail + // the cycle's verdict so the park holds. + noteHardFailure(GOOGLE_HEALTH_TOKEN_HARD_FAIL); return null; } @@ -154,6 +160,11 @@ export async function getValidToken( ); } await recordGoogleHealthSyncFailure(userId, err); + // Same verdict rule as the credentials-missing branch above: a refresh + // failure must fail the cycle, or the all-zeros run stamps success, + // un-parks `error_reauth`, and the hourly cohort retries the dead + // refresh token forever while `lastSyncedAt` advances past real data. + noteHardFailure(GOOGLE_HEALTH_TOKEN_HARD_FAIL); return null; } } @@ -204,6 +215,41 @@ interface HardFailTracker { } const hardFailStorage = new AsyncLocalStorage(); +/** + * Ledger label for a dead-token failure (`getValidToken` returning null on a + * missing-credentials or failed-refresh path). Exported so the one-shot jobs + * that run a single resource inside their own ledger scope can tell "the token + * is dead" (park, don't retry) apart from "a fetch/write hard-failed" (retry). + */ +export const GOOGLE_HEALTH_TOKEN_HARD_FAIL = "token"; + +/** + * Record a hard failure on the ambient per-cycle ledger, if one is in scope. + * No-op outside a ledger scope (e.g. a direct call from a REPL script). Used by + * `getValidToken`'s dead-token paths, the measurement write catches, and the + * workout upsert catch — anything that must fail the cycle's verdict without + * aborting sibling work. + */ +export function noteHardFailure(label: string): void { + const tracker = hardFailStorage.getStore(); + if (tracker) tracker.failures.push(label); +} + +/** + * Run `fn` inside a fresh hard-fail ledger scope and surface the collected + * failures. `syncUserGoogleHealth` owns its own scope; this export exists for + * the one-shot jobs (sleep repair) that drive a single resource sync directly + * and must still see a dead token or a hard fetch/write failure before + * stamping their completion marker. + */ +export async function runWithGoogleHealthHardFailLedger( + fn: () => Promise, +): Promise<{ result: T; failures: string[] }> { + const tracker: HardFailTracker = { failures: [] }; + const result = await hardFailStorage.run(tracker, fn); + return { result, failures: tracker.failures }; +} + /** * Per-cycle accumulator of the `(type, day)` keys every resource's writes * touched. Only populated on a `fullSync` backfill (when the inline per-day @@ -418,7 +464,15 @@ export async function upsertGoogleHealthMeasurements( // insert that `skipDuplicates` silently drops against the tombstone's key, // permanently wedging the key (see TOMBSTONES RESURRECT above). const externalIds = readings.map((r) => r.externalId); - let rowByKey = new Map(); + interface ProbedRow { + id: string; + value: number; + unit: string; + measuredAt: Date; + sleepStage: GoogleHealthMeasurementUpsert["sleepStage"]; + deletedAt: Date | null; + } + let rowByKey = new Map(); try { const existing = await prisma.measurement.findMany({ where: { @@ -426,12 +480,35 @@ export async function upsertGoogleHealthMeasurements( source: "GOOGLE_HEALTH", externalId: { in: externalIds }, }, - select: { id: true, type: true, externalId: true }, + // The payload fields ride along so the update branch can skip a no-op + // overwrite: the 24 h overlap re-fetches every recent row hourly, and an + // unconditional write would bump `syncVersion` (iOS delta churn) on rows + // whose values did not change. + select: { + id: true, + type: true, + externalId: true, + value: true, + unit: true, + measuredAt: true, + sleepStage: true, + deletedAt: true, + }, }); rowByKey = new Map( existing .filter((e) => e.externalId !== null) - .map((e) => [`${e.type} ${e.externalId}`, { id: e.id }]), + .map((e) => [ + `${e.type} ${e.externalId}`, + { + id: e.id, + value: e.value, + unit: e.unit, + measuredAt: e.measuredAt, + sleepStage: e.sleepStage, + deletedAt: e.deletedAt, + }, + ]), ); } catch (err) { // A probe failure must not strand the whole batch; fall back to treating @@ -457,9 +534,20 @@ export async function upsertGoogleHealthMeasurements( for (const r of readings) { const type = r.type as MeasurementType; const key = `${type} ${r.externalId}`; - const live = rowByKey.get(key); - if (live) { - toUpdate.push({ id: live.id, r }); + const existingRow = rowByKey.get(key); + if (existingRow) { + // Skip the no-op overwrite: a LIVE row whose payload matches the + // re-fetched reading exactly gains nothing from a write, and the + // unconditional `syncVersion` bump churned the DB + every iOS delta pull + // hourly for the whole 24 h overlap window. A TOMBSTONED row always + // updates — the write is what resurrects it (`deletedAt: null`). + const unchanged = + existingRow.deletedAt === null && + existingRow.value === r.value && + existingRow.unit === r.unit && + existingRow.measuredAt.getTime() === r.measuredAt.getTime() && + (existingRow.sleepStage ?? null) === (r.sleepStage ?? null); + if (!unchanged) toUpdate.push({ id: existingRow.id, r }); } else if (!plannedCreateKeys.has(key)) { plannedCreateKeys.add(key); toCreate.push({ @@ -518,6 +606,11 @@ export async function upsertGoogleHealthMeasurements( getEvent()?.addWarning( `google-health: failed to create measurements: ${err}`, ); + // Fetched-but-unwritten rows are lost for good once they age out of the + // 24 h overlap — fail the cycle's verdict so the watermark holds and the + // next tick re-fetches them. No rethrow: sibling chunks and resources + // must keep writing. + noteHardFailure("measurements:create"); } } @@ -549,6 +642,9 @@ export async function upsertGoogleHealthMeasurements( getEvent()?.addWarning( `google-health: failed to update measurement: ${err}`, ); + // Same rule as the create catch: hold the watermark so the overwrite is + // retried next tick rather than silently dropped. + noteHardFailure("measurements:update"); } } @@ -622,16 +718,24 @@ export async function markSynced(userId: string): Promise { * read+stamp would let the first resource move `lastSyncedAt` to now() so later * resources only see the last overlap window — silently dropping the gap after * an outage longer than the overlap. + * + * VERDICT: returns `{ imported, failed }`. `failed` is exactly the condition + * that suppresses `markSynced` — any hard failure (fetch, write, dead token) + * or an all-soft-skipped no-op, plus the parked / missing-connection early + * returns. The one-shot jobs (backfill) gate their completion markers on it; + * stamping a marker after a partial run would freeze the failure in place. */ export async function syncUserGoogleHealth( userId: string, opts: { fullSync?: boolean } = {}, -): Promise { +): Promise<{ imported: number; failed: boolean }> { if (await isReauthRequired(userId, GOOGLE_HEALTH_INTEGRATION_KEY)) { getEvent()?.addWarning( `google-health sync skipped for ${userId}: parked at error_reauth`, ); - return 0; + // Parked is NOT a clean run — a caller gating a completion marker on the + // verdict must not stamp over a no-op. + return { imported: 0, failed: true }; } // Snapshot the incremental watermark ONCE for the whole cycle. Every resource @@ -642,7 +746,7 @@ export async function syncUserGoogleHealth( where: { userId }, select: { lastSyncedAt: true }, }); - if (!connection) return 0; + if (!connection) return { imported: 0, failed: true }; const start = incrementalStart(connection.lastSyncedAt, { fullSync: opts.fullSync, }); @@ -736,8 +840,9 @@ export async function syncUserGoogleHealth( // 401. Don't stamp success when the whole cycle was soft-skipped and nothing // imported; leave the status as-is so the "looks-healthy" window closes. const allSoftSkipped = tracker.count >= resources.length && total === 0; + const failed = anyFailed || allSoftSkipped; - if (!anyFailed && !allSoftSkipped) { + if (!failed) { // Stamp the watermark ONCE for the whole cycle, only on a non-degenerate // run. Every resource already saw the snapshot `start`, so stamping now() // here can't shrink a later resource's window. @@ -746,9 +851,9 @@ export async function syncUserGoogleHealth( } annotate({ - action: { name: "googleHealth.sync", details: { imported: total } }, + action: { name: "googleHealth.sync", details: { imported: total, failed } }, }); - return total; + return { imported: total, failed }; } /** @@ -817,7 +922,9 @@ export async function runGoogleHealthPollCohort( onUserSynced?: (userId: string, imported: number) => void; } = {}, ): Promise<{ usersSynced: number; measurementsImported: number }> { - const sync = opts.sync ?? ((userId: string) => syncUserGoogleHealth(userId)); + const sync = + opts.sync ?? + ((userId: string) => syncUserGoogleHealth(userId).then((r) => r.imported)); const limit = pLimit(opts.concurrency ?? GOOGLE_HEALTH_POLL_CONCURRENCY); let usersSynced = 0; diff --git a/src/lib/insights/__tests__/graded-series-rollups.test.ts b/src/lib/insights/__tests__/graded-series-rollups.test.ts index ced555190..a6af77a54 100644 --- a/src/lib/insights/__tests__/graded-series-rollups.test.ts +++ b/src/lib/insights/__tests__/graded-series-rollups.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; const findMany = vi.fn(); +const queryRaw = vi.fn(); vi.mock("@/lib/db", () => ({ prisma: { measurement: { @@ -9,6 +10,9 @@ vi.mock("@/lib/db", () => ({ // loadUserSourcePriority lazy-loads the source-priority blob; null // (default findUnique) falls back to the default ladders. user: { findUnique: vi.fn() }, + // v1.28.25 — dense types (BLOOD_GLUCOSE / PULSE) day-bucket the + // cold-tier fallback in SQL. + $queryRaw: (...args: unknown[]) => queryRaw(...args), }, })); @@ -52,13 +56,13 @@ afterEach(() => { describe("buildGradedSeriesWithRollups — rollup-miss fallback", () => { const now = new Date("2026-05-31T12:00:00Z"); - it("folds monthly/yearly from the full history when the tier has no coverage", async () => { + it("folds monthly/yearly from the bounded fallback when the tier has no coverage", async () => { // The bounded recent read (~90 d) returns only recent points; the - // full-history read returns 2+ years so the fold has monthly/yearly. + // fallback read returns 2+ years so the fold has monthly/yearly. findMany // 1st call — bounded recent read (gte: since) .mockResolvedValueOnce(dailyRows(80, now)) - // 2nd call — full-history fallback read (no gte) + // 2nd call — bounded fallback read (gte: 1095-day horizon) .mockResolvedValueOnce(dailyRows(800, now)); // Tier miss: no MONTH / YEAR coverage at all. @@ -68,12 +72,51 @@ describe("buildGradedSeriesWithRollups — rollup-miss fallback", () => { // The bug: before the fix monthly/yearly were derived from the // bounded ~90-day read and so were always empty even when years of - // raw history existed. After the fix they fold from the full history. + // raw history existed. After the fix they fold from the fallback. expect(series.monthly.length).toBeGreaterThan(0); // 800 days of history puts at least one year into the yearly tail. expect(series.yearly.length).toBeGreaterThan(0); - // The full-history fallback read must have been issued. + // The fallback read must have been issued... expect(findMany).toHaveBeenCalledTimes(2); + // ...and bounded (v1.28.25): it must carry a `measuredAt.gte` floor + // ~1095 days back, never an unwindowed full-history walk. + const fallbackArgs = findMany.mock.calls[1][0] as { + where: { measuredAt?: { gte?: Date } }; + }; + const gte = fallbackArgs.where.measuredAt?.gte; + expect(gte).toBeInstanceOf(Date); + expect(gte!.getTime()).toBe(now.getTime() - 1095 * dayMs); + }); + + it("day-buckets the dense-type fallback in SQL instead of a raw walk (v1.28.25)", async () => { + // BLOOD_GLUCOSE on a cold tier (heavy import before the rollup + // backfill warms) used to findMany the entire CGM history. The + // fallback now runs a single day-bucket aggregate; only the bounded + // recent read touches findMany. + findMany.mockResolvedValueOnce(dailyRows(80, now)); // bounded recent + const bucketRows: Array<{ bucket_start: Date; mean: number }> = []; + for (let i = 0; i < 800; i++) { + bucketRows.push({ + bucket_start: new Date(now.getTime() - i * dayMs), + mean: 100, + }); + } + queryRaw.mockResolvedValueOnce(bucketRows.reverse()); + + readBestGranularityRollups.mockResolvedValue(null); + + const series = await buildGradedSeriesWithRollups( + "u1", + "BLOOD_GLUCOSE", + now, + ); + + expect(series.monthly.length).toBeGreaterThan(0); + expect(series.yearly.length).toBeGreaterThan(0); + // Raw findMany ran once (the bounded recent read) — the fallback + // itself went through the SQL aggregate. + expect(findMany).toHaveBeenCalledTimes(1); + expect(queryRaw).toHaveBeenCalledTimes(1); }); it("reads monthly/yearly from the tier when it has coverage and skips the full read", async () => { diff --git a/src/lib/insights/__tests__/invalidate-status-insights.test.ts b/src/lib/insights/__tests__/invalidate-status-insights.test.ts index c616c8f9b..fed621785 100644 --- a/src/lib/insights/__tests__/invalidate-status-insights.test.ts +++ b/src/lib/insights/__tests__/invalidate-status-insights.test.ts @@ -19,6 +19,7 @@ const auditFindMany = vi.fn(); const enqueueStatusGeneration = vi.fn(); const userFindUnique = vi.fn(); const measurementFindMany = vi.fn(); +const measurementGroupBy = vi.fn(); vi.mock("@/lib/db", () => ({ prisma: { @@ -27,7 +28,12 @@ vi.mock("@/lib/db", () => ({ findMany: (...a: unknown[]) => auditFindMany(...a), }, user: { findUnique: (...a: unknown[]) => userFindUnique(...a) }, - measurement: { findMany: (...a: unknown[]) => measurementFindMany(...a) }, + measurement: { + findMany: (...a: unknown[]) => measurementFindMany(...a), + // v1.28.25 — type discovery runs as a server-side GROUP BY, not a + // client-deduped `distinct` findMany. + groupBy: (...a: unknown[]) => measurementGroupBy(...a), + }, }, })); @@ -49,6 +55,7 @@ beforeEach(() => { enqueueStatusGeneration.mockResolvedValue(undefined); userFindUnique.mockResolvedValue({ locale: "de" }); measurementFindMany.mockResolvedValue([]); + measurementGroupBy.mockResolvedValue([]); }); /** Distinct scopes the invalidator enqueued a regenerate for. */ @@ -280,7 +287,7 @@ describe("invalidateStatusInsightsForTypes", () => { // ones for free — the hash baseline rows are never deleted. describe("enqueueStatusRefillForUser", () => { it("enqueues the seven specialised scopes plus the user's data-bearing generic scopes", async () => { - measurementFindMany.mockResolvedValue([ + measurementGroupBy.mockResolvedValue([ { type: "WEIGHT" }, // specialised — no generic registry entry { type: "BLOOD_GLUCOSE" }, // generic card { type: "ACTIVE_ENERGY_BURNED" }, // generic card under a remapped id @@ -328,7 +335,7 @@ describe("enqueueStatusRefillForUser", () => { }); it("still refills the specialised scopes when the generic-scope discovery read fails", async () => { - measurementFindMany.mockRejectedValue(new Error("pool exhausted")); + measurementGroupBy.mockRejectedValue(new Error("pool exhausted")); const count = await enqueueStatusRefillForUser("u1", "de"); expect(count).toBe(7); expect(enqueuedScopes()).toEqual([ diff --git a/src/lib/insights/__tests__/pulse-status-timeout.test.ts b/src/lib/insights/__tests__/pulse-status-timeout.test.ts index 298656eb8..508cff3b2 100644 --- a/src/lib/insights/__tests__/pulse-status-timeout.test.ts +++ b/src/lib/insights/__tests__/pulse-status-timeout.test.ts @@ -25,6 +25,9 @@ vi.mock("@/lib/db", () => ({ auditLog: { findFirst: vi.fn(), create: vi.fn() }, measurement: { findMany: vi.fn() }, measurementRollup: { findMany: vi.fn() }, + // v1.28.25 — the graded-series cold-tier fallback day-buckets dense + // types (PULSE) via a raw aggregate instead of a full findMany walk. + $queryRaw: vi.fn(async () => []), moodEntry: { findMany: vi.fn() }, }, })); @@ -46,6 +49,7 @@ import { generatePulseStatusForUser } from "../pulse-status"; beforeEach(() => { vi.resetAllMocks(); + vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue([] as never); }); diff --git a/src/lib/insights/__tests__/pulse-status.test.ts b/src/lib/insights/__tests__/pulse-status.test.ts index 64c64b01c..4c570d8c7 100644 --- a/src/lib/insights/__tests__/pulse-status.test.ts +++ b/src/lib/insights/__tests__/pulse-status.test.ts @@ -6,6 +6,9 @@ vi.mock("@/lib/db", () => ({ auditLog: { findFirst: vi.fn(), create: vi.fn() }, measurement: { findMany: vi.fn() }, measurementRollup: { findMany: vi.fn() }, + // v1.28.25 — the graded-series cold-tier fallback day-buckets dense + // types (PULSE) via a raw aggregate instead of a full findMany walk. + $queryRaw: vi.fn(async () => []), moodEntry: { findMany: vi.fn() }, }, })); @@ -47,6 +50,7 @@ function stubCompletion( beforeEach(() => { vi.resetAllMocks(); + vi.mocked(prisma.$queryRaw).mockResolvedValue([] as never); // Cold rollup tier: the pulse graded series folds monthly/yearly from // the full-history `measurement.findMany` fallback on a tier miss. vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue([] as never); @@ -69,6 +73,14 @@ describe("generatePulseStatusForUser — graded payload", () => { } as never); vi.mocked(prisma.auditLog.findFirst).mockResolvedValue(null); vi.mocked(prisma.measurement.findMany).mockResolvedValue(records as never); + // v1.28.25 — PULSE is a dense type, so the cold-tier fallback reads + // a SQL day-bucket aggregate instead of the raw findMany walk. Feed + // the same 1000 days as day buckets. + vi.mocked(prisma.$queryRaw).mockResolvedValue( + records + .map((r) => ({ bucket_start: r.measuredAt, mean: r.value })) + .reverse() as never, + ); vi.mocked(prisma.moodEntry.findMany).mockResolvedValue([] as never); vi.mocked(prisma.auditLog.create).mockResolvedValue({ createdAt: new Date(), diff --git a/src/lib/insights/comprehensive-generate.ts b/src/lib/insights/comprehensive-generate.ts index cef38114d..884eaf65d 100644 --- a/src/lib/insights/comprehensive-generate.ts +++ b/src/lib/insights/comprehensive-generate.ts @@ -608,10 +608,13 @@ export async function enqueueStatusRefillForUser( ): Promise { const scopes = new Set(PER_STATUS_SCOPES); try { - const rows = await prisma.measurement.findMany({ + // `groupBy` compiles to a server-side `GROUP BY` — Prisma's + // `distinct` dedups in the client AFTER pulling every live row, which + // on a dense multi-year account walks a six-figure row set to answer + // "which types exist?" on the request path. + const rows = await prisma.measurement.groupBy({ + by: ["type"], where: { userId, deletedAt: null }, - distinct: ["type"], - select: { type: true }, }); for (const row of rows) { const metricId = metricIdForMeasurementType(row.type); diff --git a/src/lib/insights/graded-series.ts b/src/lib/insights/graded-series.ts index b259d3b37..0053d6d5d 100644 --- a/src/lib/insights/graded-series.ts +++ b/src/lib/insights/graded-series.ts @@ -72,6 +72,29 @@ const WEEKLY_WINDOW_MS = (RECENT_DAYS + WEEKLY_WEEKS * 7) * MS_PER_DAY; // yearly buckets. const MONTHLY_WINDOW_MS = WEEKLY_WINDOW_MS + MONTHLY_MONTHS * 30 * MS_PER_DAY; +/** + * v1.28.25 — bound on the cold-tier fallback read. The coverage-miss + * fallback used to `findMany` the FULL raw history with no window or + * take; on a CGM / per-sample-pulse account that had just imported years + * of data (exactly the state where the rollup tier is still cold) it + * pulled six-figure row sets into JS on a request path. 1095 days + * matches the yearly rollup router window below, so the cold path now + * covers the same horizon the warm path serves. + */ +const COLD_FALLBACK_WINDOW_DAYS = 1095; + +/** + * Sample-dense types whose cold-tier fallback is day-bucketed in SQL + * (one AVG row per day) instead of read raw — same dense set the series + * route caps (CGM glucose ~288 rows/day, PULSE per-sample / hourly). + * The fold's output shape is unchanged; on the cold path only, a dense + * type's monthly/yearly min/max/n derive from day means rather than + * individual samples — the warm rollup tier restores sample-exact + * figures as soon as the backfill lands. + */ +const DENSE_FALLBACK_TYPES: ReadonlySet = + new Set(["BLOOD_GLUCOSE", "PULSE"]); + export interface RecentDayBucket { /** Berlin YYYY-MM-DD. */ date: string; @@ -326,12 +349,13 @@ function rollupYearly(rows: RollupBucketRow[]): YearlyBucket[] { * caught up on — the bounded ~90-day read can NOT supply the monthly / * yearly slices (every row in it folds into recent / weekly), so naively * reusing it would ship empty monthly / yearly arrays even when years of - * raw history exist. On that miss this falls back to a full-history - * in-memory fold so the coarse slices are populated from the raw rows - * rather than left falsely empty. The full read only happens when the - * tier is cold, which is exactly the case the write-amplified tier was - * meant to avoid — so the hot path stays bounded and the cold path stays - * correct. + * raw history exist. On that miss this falls back to an in-memory fold + * over a bounded window (1095 days — the yearly router's own horizon; + * sample-dense types are additionally day-bucketed in SQL) so the coarse + * slices are populated from the raw rows rather than left falsely empty. + * The fallback read only happens when the tier is cold, which is exactly + * the case the write-amplified tier was meant to avoid — so the hot path + * stays bounded and the cold path stays correct. */ export async function buildGradedSeriesWithRollups( userId: string, @@ -396,19 +420,57 @@ export async function buildGradedSeriesWithRollups( } // Tier coverage miss for one or both coarse slices: the bounded - // ~90-day read can't supply them, so fold the FULL history in memory - // and take whichever slices the tier left empty. The full read is the - // exception (cold-tier accounts), never the warm-tier norm. + // ~90-day read can't supply them, so fold a BOUNDED history window in + // memory and take whichever slices the tier left empty. The fallback + // read is the exception (cold-tier accounts), never the warm-tier + // norm. v1.28.25 — the read is bounded to the same 1095-day horizon + // the yearly rollup router serves (it was previously unwindowed), and + // the sample-dense types are day-bucketed in SQL so a heavy import + // ahead of the rollup backfill can't drag six-figure row sets into JS. if (!monthlyCovered || !yearlyCovered) { - const allRows = await prisma.measurement.findMany({ - where: { userId, type, deletedAt: null }, - orderBy: { measuredAt: "asc" }, - select: { measuredAt: true, value: true }, - }); - const fullGraded = buildGradedSeriesFromPoints( - allRows.map((r) => ({ measuredAt: r.measuredAt, value: r.value })), - now, + const fallbackSince = new Date( + now.getTime() - COLD_FALLBACK_WINDOW_DAYS * MS_PER_DAY, ); + let fallbackPoints: Point[]; + if (DENSE_FALLBACK_TYPES.has(type)) { + // Parameter-bound day-bucket aggregate, mirroring the rollup tier's + // `date_trunc(...) GROUP BY` (measurement-rollups.ts) — at most + // ~1095 rows regardless of sample density. + const bucketRows = await prisma.$queryRaw< + Array<{ bucket_start: Date; mean: number }> + >` + SELECT + date_trunc('day', m."measured_at") AS bucket_start, + AVG(m."value")::double precision AS mean + FROM measurements m + WHERE m."user_id" = ${userId} + AND m."type" = ${type}::"measurement_type" + AND m."measured_at" >= ${fallbackSince} + AND m."deleted_at" IS NULL + GROUP BY date_trunc('day', m."measured_at") + ORDER BY bucket_start ASC + `; + fallbackPoints = bucketRows.map((r) => ({ + measuredAt: r.bucket_start, + value: r.mean, + })); + } else { + const rows = await prisma.measurement.findMany({ + where: { + userId, + type, + deletedAt: null, + measuredAt: { gte: fallbackSince }, + }, + orderBy: { measuredAt: "asc" }, + select: { measuredAt: true, value: true }, + }); + fallbackPoints = rows.map((r) => ({ + measuredAt: r.measuredAt, + value: r.value, + })); + } + const fullGraded = buildGradedSeriesFromPoints(fallbackPoints, now); if (!monthlyCovered) monthly = fullGraded.monthly; if (!yearlyCovered) yearly = fullGraded.yearly; } diff --git a/src/lib/jobs/__tests__/google-health-backfill.test.ts b/src/lib/jobs/__tests__/google-health-backfill.test.ts new file mode 100644 index 000000000..59c23b21f --- /dev/null +++ b/src/lib/jobs/__tests__/google-health-backfill.test.ts @@ -0,0 +1,84 @@ +/** + * Pins the verdict gate of `runGoogleHealthBackfillForUser`: the + * `backfillCompletedAt` marker may only be stamped after a CLEAN full-history + * run. `syncUserGoogleHealth` swallows per-resource errors into its verdict — + * before the gate, ANY run (partial hard failure, parked no-op) stamped the + * marker and the pg-boss `retryLimit: 3` was dead code. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { prismaMock, syncUserGoogleHealthMock, isReauthRequiredMock } = + vi.hoisted(() => ({ + prismaMock: { + googleHealthConnection: { findMany: vi.fn(), update: vi.fn() }, + }, + syncUserGoogleHealthMock: vi.fn(), + isReauthRequiredMock: vi.fn(async () => false), + })); + +vi.mock("@/lib/db", () => ({ prisma: prismaMock })); + +vi.mock("@/lib/jobs/boss-instance", () => ({ + getGlobalBoss: () => ({ send: vi.fn() }), +})); + +vi.mock("@/lib/logging/context", () => ({ + annotate: () => {}, + getEvent: () => null, +})); + +vi.mock("@/lib/integrations/status", () => ({ + isReauthRequired: isReauthRequiredMock, +})); + +vi.mock("@/lib/google-health/sync", () => ({ + GOOGLE_HEALTH_INTEGRATION_KEY: "google-health", + syncUserGoogleHealth: (...a: unknown[]) => syncUserGoogleHealthMock(...a), +})); + +import { runGoogleHealthBackfillForUser } from "../google-health-backfill"; + +beforeEach(() => { + vi.clearAllMocks(); + isReauthRequiredMock.mockResolvedValue(false); + prismaMock.googleHealthConnection.update.mockResolvedValue({}); +}); + +describe("runGoogleHealthBackfillForUser — verdict-gated marker", () => { + it("stamps backfillCompletedAt after a CLEAN full-history run", async () => { + syncUserGoogleHealthMock.mockResolvedValue({ imported: 42, failed: false }); + + const { imported } = await runGoogleHealthBackfillForUser("u1"); + + expect(imported).toBe(42); + expect(syncUserGoogleHealthMock).toHaveBeenCalledWith("u1", { + fullSync: true, + }); + const updateArg = prismaMock.googleHealthConnection.update.mock + .calls[0]![0] as { + where: Record; + data: Record; + }; + expect(updateArg.where).toEqual({ userId: "u1" }); + expect(updateArg.data.backfillCompletedAt).toBeInstanceOf(Date); + }); + + it("a failed verdict THROWS without stamping — pg-boss retries become real", async () => { + syncUserGoogleHealthMock.mockResolvedValue({ imported: 7, failed: true }); + + await expect(runGoogleHealthBackfillForUser("u1")).rejects.toThrow( + /incomplete/, + ); + expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled(); + }); + + it("a connection parked at error_reauth returns WITHOUT running the sync or stamping", async () => { + isReauthRequiredMock.mockResolvedValue(true); + + await expect(runGoogleHealthBackfillForUser("u1")).resolves.toEqual({ + imported: 0, + }); + expect(syncUserGoogleHealthMock).not.toHaveBeenCalled(); + expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/jobs/__tests__/google-health-sleep-repair.test.ts b/src/lib/jobs/__tests__/google-health-sleep-repair.test.ts index e045ad70c..ade6fc956 100644 --- a/src/lib/jobs/__tests__/google-health-sleep-repair.test.ts +++ b/src/lib/jobs/__tests__/google-health-sleep-repair.test.ts @@ -10,13 +10,15 @@ */ import { beforeEach, describe, expect, it, vi } from "vitest"; -const { prismaMock, bossSend, syncUserSleep } = vi.hoisted(() => ({ - prismaMock: { - googleHealthConnection: { findMany: vi.fn(), update: vi.fn() }, - }, - bossSend: vi.fn(), - syncUserSleep: vi.fn(), -})); +const { prismaMock, bossSend, syncUserSleep, isReauthRequiredMock } = + vi.hoisted(() => ({ + prismaMock: { + googleHealthConnection: { findMany: vi.fn(), update: vi.fn() }, + }, + bossSend: vi.fn(), + syncUserSleep: vi.fn(), + isReauthRequiredMock: vi.fn(async () => false), + })); vi.mock("@/lib/db", () => ({ prisma: prismaMock })); @@ -33,6 +35,33 @@ vi.mock("@/lib/logging/context", () => ({ getEvent: () => null, })); +vi.mock("@/lib/integrations/status", () => ({ + isReauthRequired: isReauthRequiredMock, + recordSyncFailure: vi.fn(async () => {}), + recordSyncSuccess: vi.fn(async () => {}), +})); + +// The repair module threads the REAL hard-fail ledger from +// `@/lib/google-health/sync`; its heavy transitive deps are mocked the same +// way the google-health failsoft suite does. +vi.mock("@/lib/crypto", () => ({ encrypt: vi.fn(), decrypt: vi.fn() })); +vi.mock("@/lib/rollups/measurement-rollups", () => ({ + collapseToTypeDayKeys: vi.fn(() => []), + recomputeBucketsForMeasurement: vi.fn(async () => {}), + recomputeUserRollups: vi.fn(async () => {}), +})); +vi.mock("@/lib/insights/comprehensive-generate", () => ({ + invalidateStatusInsightsForTypes: vi.fn(async () => {}), +})); +vi.mock("@/lib/google-health/credentials", () => ({ + getUserGoogleHealthCredentials: vi.fn(async () => null), +})); +vi.mock("@/lib/google-health/client", () => ({ refreshAccessToken: vi.fn() })); + +import { + GOOGLE_HEALTH_TOKEN_HARD_FAIL, + noteHardFailure, +} from "@/lib/google-health/sync"; import { enqueueBootTimeGoogleHealthSleepRepair, runGoogleHealthSleepRepairForUser, @@ -40,6 +69,7 @@ import { beforeEach(() => { vi.clearAllMocks(); + isReauthRequiredMock.mockResolvedValue(false); }); describe("enqueueBootTimeGoogleHealthSleepRepair — discovery", () => { @@ -139,4 +169,41 @@ describe("runGoogleHealthSleepRepairForUser", () => { ); expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled(); }); + + it("a swallowed hard failure (ledger entry) throws and does NOT stamp — pg-boss retries", async () => { + // `syncUserSleep` swallows fetch/write hard failures into the ambient + // ledger and still resolves a count; the repair must read the ledger, not + // the count, before stamping. + syncUserSleep.mockImplementation(async () => { + noteHardFailure("fetchSleep"); + return 5; + }); + + await expect(runGoogleHealthSleepRepairForUser("u1")).rejects.toThrow( + /incomplete/, + ); + expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled(); + }); + + it("a dead token returns WITHOUT stamping and WITHOUT throwing (boot discovery re-enqueues)", async () => { + syncUserSleep.mockImplementation(async () => { + noteHardFailure(GOOGLE_HEALTH_TOKEN_HARD_FAIL); + return 0; + }); + + await expect(runGoogleHealthSleepRepairForUser("u1")).resolves.toEqual({ + imported: 0, + }); + expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled(); + }); + + it("a connection parked at error_reauth returns WITHOUT running the sync or stamping", async () => { + isReauthRequiredMock.mockResolvedValue(true); + + await expect(runGoogleHealthSleepRepairForUser("u1")).resolves.toEqual({ + imported: 0, + }); + expect(syncUserSleep).not.toHaveBeenCalled(); + expect(prismaMock.googleHealthConnection.update).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/jobs/dense-intraday-retention.ts b/src/lib/jobs/dense-intraday-retention.ts index 32cead3b4..61c1d0202 100644 --- a/src/lib/jobs/dense-intraday-retention.ts +++ b/src/lib/jobs/dense-intraday-retention.ts @@ -171,7 +171,12 @@ export async function enqueueBootTimeDenseIntradayRetention(): Promise<{ Date.now() - DENSE_INTRADAY_RETENTION_DAYS * 24 * 60 * 60 * 1000, ); - const users = await prisma.measurement.findMany({ + // `groupBy` compiles to a server-side `GROUP BY` — Prisma's + // `distinct` dedups in the client AFTER pulling every matching row, + // an instance-wide scan of every out-of-window dense-tier row just + // to list user ids. + const users = await prisma.measurement.groupBy({ + by: ["userId"], where: { source: "APPLE_HEALTH", type: { in: types }, @@ -179,8 +184,6 @@ export async function enqueueBootTimeDenseIntradayRetention(): Promise<{ NOT: { externalId: { startsWith: "stats:" } }, measuredAt: { lt: windowStart }, }, - select: { userId: true }, - distinct: ["userId"], }); if (users.length === 0) { diff --git a/src/lib/jobs/google-health-backfill.ts b/src/lib/jobs/google-health-backfill.ts index 5a09c741d..2fda8dba1 100644 --- a/src/lib/jobs/google-health-backfill.ts +++ b/src/lib/jobs/google-health-backfill.ts @@ -14,7 +14,11 @@ import { prisma } from "@/lib/db"; import { annotate } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; -import { syncUserGoogleHealth } from "@/lib/google-health/sync"; +import { isReauthRequired } from "@/lib/integrations/status"; +import { + GOOGLE_HEALTH_INTEGRATION_KEY, + syncUserGoogleHealth, +} from "@/lib/google-health/sync"; export const GOOGLE_HEALTH_BACKFILL_QUEUE = "google-health-backfill"; @@ -32,14 +36,44 @@ export interface GoogleHealthBackfillPayload { /** * Per-user backfill handler. Runs a full-history sync for one account and - * stamps `backfillCompletedAt` so the discovery query drops it. Idempotent: the - * per-resource upserts are key-stable, so a re-run (e.g. a reboot mid-walk) - * overwrites rather than duplicating. Mirrors `runFitbitBackfillForUser`. + * stamps `backfillCompletedAt` ONLY on a clean run, so the discovery query + * drops it. Idempotent: the per-resource upserts are key-stable, so a re-run + * (e.g. a reboot mid-walk) overwrites rather than duplicating. Mirrors + * `runFitbitBackfillForUser`. + * + * Verdict-gated: a partial hard failure (one collection 400ing, a write + * failing) THROWS so pg-boss retries the job — stamping the marker over a + * partial walk would freeze the gap in forever. A connection parked at + * `error_reauth` is NOT clean either, but throwing against a dead grant would + * just burn the retry budget on the same no-op: return WITHOUT stamping — the + * boot discovery re-enqueues the account on the next boot, and the stamp lands + * once the user reconnects and a clean walk completes. */ export async function runGoogleHealthBackfillForUser( userId: string, ): Promise<{ imported: number }> { - const imported = await syncUserGoogleHealth(userId, { fullSync: true }); + if (await isReauthRequired(userId, GOOGLE_HEALTH_INTEGRATION_KEY)) { + annotate({ + action: { + name: "google_health.backfill.skipped_reauth", + details: { imported: 0 }, + }, + }); + return { imported: 0 }; + } + + const { imported, failed } = await syncUserGoogleHealth(userId, { + fullSync: true, + }); + + if (failed) { + // Surface through the pg-boss retry path (retryLimit 3, backoff). If the + // failure parked the connection at error_reauth meanwhile, the next + // attempt takes the reauth return above instead of failing again. + throw new Error( + `google-health backfill incomplete for user ${userId} — marker not stamped`, + ); + } await prisma.googleHealthConnection.update({ where: { userId }, diff --git a/src/lib/jobs/google-health-sleep-repair.ts b/src/lib/jobs/google-health-sleep-repair.ts index 5008516cb..1d989ff72 100644 --- a/src/lib/jobs/google-health-sleep-repair.ts +++ b/src/lib/jobs/google-health-sleep-repair.ts @@ -26,6 +26,12 @@ import { prisma } from "@/lib/db"; import { annotate } from "@/lib/logging/context"; import { getGlobalBoss } from "@/lib/jobs/boss-instance"; +import { isReauthRequired } from "@/lib/integrations/status"; +import { + GOOGLE_HEALTH_INTEGRATION_KEY, + GOOGLE_HEALTH_TOKEN_HARD_FAIL, + runWithGoogleHealthHardFailLedger, +} from "@/lib/google-health/sync"; import { syncUserSleep } from "@/lib/google-health/sync-sleep"; export const GOOGLE_HEALTH_SLEEP_REPAIR_QUEUE = "google-health-sleep-repair"; @@ -50,11 +56,51 @@ export interface GoogleHealthSleepRepairPayload { * Idempotent: the per-segment upserts are key-stable and the replace-by-window * clears stale copies, so a re-run (e.g. a reboot mid-walk) converges rather * than duplicating. Does NOT move `lastSyncedAt`. + * + * Verdict-gated: `syncUserSleep` swallows fetch/write hard failures into the + * ambient hard-fail ledger (returning a count either way), so the repair runs + * it inside its own ledger scope — the same mechanism `syncUserGoogleHealth` + * uses for its cycle verdict — and stamps `sleepRepairedAt` ONLY on a clean + * run. A hard fetch/write failure THROWS so pg-boss retries; a dead token + * (parked connection, missing credentials, failed refresh) is NOT clean but + * does not throw — retrying a dead grant burns the retry budget on the same + * no-op. It returns WITHOUT stamping instead: the boot discovery re-enqueues + * the account on the next boot, and the stamp lands after a reconnect. */ export async function runGoogleHealthSleepRepairForUser( userId: string, ): Promise<{ imported: number }> { - const imported = await syncUserSleep(userId, { deferRollup: false }); + if (await isReauthRequired(userId, GOOGLE_HEALTH_INTEGRATION_KEY)) { + annotate({ + action: { + name: "googleHealth.sleepRepair.skipped_reauth", + details: { imported: 0 }, + }, + }); + return { imported: 0 }; + } + + const { result: imported, failures } = + await runWithGoogleHealthHardFailLedger(() => + syncUserSleep(userId, { deferRollup: false }), + ); + + if (failures.length > 0) { + if (failures.every((f) => f === GOOGLE_HEALTH_TOKEN_HARD_FAIL)) { + // Dead token: nothing was fetched, so nothing to retry until the user + // reconnects. No stamp — the next boot's discovery picks it back up. + annotate({ + action: { + name: "googleHealth.sleepRepair.skipped_token", + details: { imported }, + }, + }); + return { imported }; + } + throw new Error( + `google-health sleep repair incomplete for user ${userId} (${failures.join(", ")}) — marker not stamped`, + ); + } await prisma.googleHealthConnection.update({ where: { userId }, diff --git a/src/lib/jobs/insight-pregenerate.ts b/src/lib/jobs/insight-pregenerate.ts index 080707a2c..9648014d2 100644 --- a/src/lib/jobs/insight-pregenerate.ts +++ b/src/lib/jobs/insight-pregenerate.ts @@ -388,17 +388,21 @@ async function warmGenericMetricCaches( // rows for. Best-effort — a read failure (or a worker prisma surface // without `measurement`) yields zero metrics rather than aborting the // user's whole pre-generation loop. - let rows: Array<{ type: string }>; + let typesWithData: ReadonlySet; try { - rows = await prisma.measurement.findMany({ + // `groupBy` compiles to a server-side `GROUP BY` — Prisma's + // `distinct` dedups in the client AFTER pulling every live row, which + // on a dense multi-year account walks a six-figure row set nightly to + // answer "which types exist?". + const rows = await prisma.measurement.groupBy({ + by: ["type"], where: { userId, deletedAt: null }, - distinct: ["type"], - select: { type: true }, + orderBy: { type: "asc" }, }); + typesWithData = new Set(rows.map((r) => r.type)); } catch { return 0; } - const typesWithData = new Set(rows.map((r) => r.type)); const metricsWithData = METRIC_STATUS_IDS.filter( (id: MetricStatusMetricId) => { diff --git a/src/lib/jobs/mean-consolidation.ts b/src/lib/jobs/mean-consolidation.ts index c337b801e..a001dc2b9 100644 --- a/src/lib/jobs/mean-consolidation.ts +++ b/src/lib/jobs/mean-consolidation.ts @@ -106,15 +106,19 @@ export async function enqueueBootTimeMeanConsolidation( return { enqueued: 0, skipped: 0, error: null }; } - const users = await prisma.measurement.findMany({ + // `groupBy` compiles to a server-side `GROUP BY` — Prisma's + // `distinct` dedups in the client AFTER pulling every matching row, + // an instance-wide unbounded scan just to list user ids. No date + // bound on purpose: a fresh import of OLD per-sample rows must still + // surface its user here, however old the rows are. + const users = await prisma.measurement.groupBy({ + by: ["userId"], where: { source: "APPLE_HEALTH", type: { in: types }, deletedAt: null, NOT: { externalId: { startsWith: "stats:" } }, }, - select: { userId: true }, - distinct: ["userId"], }); if (users.length === 0) { diff --git a/src/lib/nightscout/__tests__/sync.test.ts b/src/lib/nightscout/__tests__/sync.test.ts index 463b9ab26..f9f46043a 100644 --- a/src/lib/nightscout/__tests__/sync.test.ts +++ b/src/lib/nightscout/__tests__/sync.test.ts @@ -116,6 +116,10 @@ describe("syncUserNightscout", () => { const arg = upsertMock.mock.calls[0]![0]; // An immutable sample: update must not carry a new value/measuredAt. expect(arg.update).not.toHaveProperty("value"); + expect(arg.update).not.toHaveProperty("measuredAt"); + // But it DOES carry the resurrection — Nightscout owns its rows, so a + // re-synced reading clears a tombstone (no-op on a live row). + expect(arg.update).toEqual({ deletedAt: null }); }); it("records sync success after a clean pass", async () => { diff --git a/src/lib/nightscout/sync.ts b/src/lib/nightscout/sync.ts index d3b9153b1..0a7ef96f7 100644 --- a/src/lib/nightscout/sync.ts +++ b/src/lib/nightscout/sync.ts @@ -167,7 +167,11 @@ export async function upsertNightscoutEntries( }, // First-write-wins: a CGM sample is immutable, so a re-sync of the // same reading must not rewrite the stored value or measuredAt. - update: {}, + // `deletedAt: null` is a no-op on a live row, a deliberate + // RESURRECTION on a tombstoned one — Nightscout is the source of + // truth for its own rows, so a re-synced reading brings the row + // back (mirrors Google / Fitbit). + update: { deletedAt: null }, }); touched.push({ type, measuredAt: mapped.measuredAt }); imported++; diff --git a/src/lib/oura/__tests__/client.test.ts b/src/lib/oura/__tests__/client.test.ts index 661ede829..6a44695ae 100644 --- a/src/lib/oura/__tests__/client.test.ts +++ b/src/lib/oura/__tests__/client.test.ts @@ -229,10 +229,46 @@ describe("mapSleep", () => { const onset = new Date("2026-06-09T23:00:00.000Z").getTime(); expect(mapped[0].measuredAt.getTime()).toBe(onset + 10 * 60_000); expect(mapped[1].measuredAt.getTime()).toBe(onset + 25 * 60_000); - // Distinct, record-scoped, segment-indexed externalIds (no collapse). + // Distinct, record-scoped externalIds keyed on each run's own START + // instant (no collapse). NOT a positional run index: a revised hypnogram + // re-segments the night, so an index shifted every following run onto the + // wrong row and left phantom tail rows the night-total double-counted. const ids = mapped.map((m) => m.externalId); expect(new Set(ids).size).toBe(ids.length); - expect(ids[0]).toBe("sleep:rec-T:seg:0"); + expect(ids[0]).toBe("sleep:rec-T:seg:2026-06-09T23:00:00.000Z"); + expect(ids[1]).toBe("sleep:rec-T:seg:2026-06-09T23:10:00.000Z"); + }); + + it("keeps IDENTICAL externalIds for unchanged runs across a revised hypnogram", () => { + // The same night re-scored: Oura re-classifies the SECOND run's digits, + // splitting one run into two — every later run's index would have + // shifted (+1) under the retired run-indexed key, minting fresh ids. The + // start-keyed ids of the UNCHANGED first and last runs must survive; the + // re-segmented middle mints new ids and the record-scoped sweep clears + // what the revision orphaned. + const base = { + id: "rec-T", + day: "2026-06-10", + bedtime_start: "2026-06-09T23:00:00.000Z", + bedtime_end: "2026-06-10T07:00:00.000Z", + }; + const idsOf = (hyp: string) => + mapSleep({ ...base, sleep_phase_5_min: hyp }) + .filter((m) => m.type === "SLEEP_DURATION") + .map((m) => m.externalId); + + const first = idsOf("1122234"); + const revised = idsOf("1121234"); // run "222" split into "2","1","2" + + // First run (digits 0-1) and the tail runs that keep their start offsets + // are identical across the revision. + expect(revised[0]).toBe(first[0]); + expect(revised.at(-1)).toBe(first.at(-1)); + // The re-segmented middle mints ids keyed on the new run starts — all + // still under the record's `sleep:rec-T:seg:` sweep prefix. + expect(revised.every((id) => id!.startsWith("sleep:rec-T:seg:"))).toBe( + true, + ); }); it("ignores an empty / malformed hypnogram and uses stage totals", () => { diff --git a/src/lib/oura/__tests__/sync.test.ts b/src/lib/oura/__tests__/sync.test.ts index 2d4f52a2e..a10533017 100644 --- a/src/lib/oura/__tests__/sync.test.ts +++ b/src/lib/oura/__tests__/sync.test.ts @@ -14,6 +14,7 @@ const { fetchResilienceMock, refreshMock, upsertMock, + updateManyMock, recordSuccessMock, recordFailureMock, recomputeMock, @@ -32,6 +33,7 @@ const { fetchResilienceMock: vi.fn(), refreshMock: vi.fn(), upsertMock: vi.fn(), + updateManyMock: vi.fn(), recordSuccessMock: vi.fn(), recordFailureMock: vi.fn(), recomputeMock: vi.fn(), @@ -45,7 +47,7 @@ vi.mock("../credentials", () => ({ })); vi.mock("@/lib/db", () => ({ - prisma: { measurement: { upsert: upsertMock } }, + prisma: { measurement: { upsert: upsertMock, updateMany: updateManyMock } }, })); vi.mock("@/lib/integrations/status", async (importOriginal) => ({ @@ -107,6 +109,7 @@ beforeEach(() => { fetchResilienceMock.mockReset().mockResolvedValue([]); refreshMock.mockReset(); upsertMock.mockReset().mockResolvedValue({}); + updateManyMock.mockReset().mockResolvedValue({ count: 0 }); recordSuccessMock.mockReset().mockResolvedValue(undefined); recordFailureMock.mockReset().mockResolvedValue(undefined); recomputeMock.mockReset().mockResolvedValue(undefined); @@ -231,6 +234,69 @@ describe("syncUserOura", () => { expect(new Set(externalIds).size).toBe(externalIds.length); }); + it("sweeps stale sleep rows per fetched record: live-only, record-prefixed, notIn the fresh set, soft-delete (v1.28.25)", async () => { + getConnMock.mockResolvedValue(CONN); + fetchSleepMock.mockResolvedValue([ + { + id: "rec-T", + day: "2026-06-10", + bedtime_start: "2026-06-09T23:00:00.000Z", + bedtime_end: "2026-06-10T07:00:00.000Z", + sleep_phase_5_min: "1122234", + }, + ]); + + await syncUserOura("u1"); + + expect(updateManyMock).toHaveBeenCalledTimes(1); + const arg = updateManyMock.mock.calls[0]![0] as { + where: { + userId: string; + source: string; + type: string; + deletedAt: null; + externalId: { startsWith: string; notIn: string[] }; + }; + data: Record; + }; + expect(arg.where.userId).toBe("u1"); + expect(arg.where.source).toBe("OURA"); + expect(arg.where.type).toBe("SLEEP_DURATION"); + expect(arg.where.deletedAt).toBeNull(); + // Bounded to THIS record. The prefix covers the whole record slice (runs + // AND the stage-total fallback shape) so a night that flips between the + // two shapes cannot leave the other shape's rows double-counting. + expect(arg.where.externalId.startsWith).toBe("sleep:rec-T:"); + // Every fresh SLEEP_DURATION id of the record is protected. + expect(arg.where.externalId.notIn).toEqual([ + "sleep:rec-T:seg:2026-06-09T23:00:00.000Z", + "sleep:rec-T:seg:2026-06-09T23:10:00.000Z", + "sleep:rec-T:seg:2026-06-09T23:25:00.000Z", + "sleep:rec-T:seg:2026-06-09T23:30:00.000Z", + ]); + // Soft delete only. + expect(arg.data).toEqual({ deletedAt: expect.any(Date) }); + + // A legacy run-indexed row for this record falls inside the sweep. + const legacyId = "sleep:rec-T:seg:0"; + expect(legacyId.startsWith(arg.where.externalId.startsWith)).toBe(true); + expect(arg.where.externalId.notIn).not.toContain(legacyId); + + // Replace-then-write: the sweep runs before the first upsert. + expect(updateManyMock.mock.invocationCallOrder[0]!).toBeLessThan( + upsertMock.mock.invocationCallOrder[0]!, + ); + }); + + it("never sweeps when the sleep collection returns no records", async () => { + getConnMock.mockResolvedValue(CONN); + fetchReadinessMock.mockResolvedValue([ + { id: "1", day: "2026-06-10", score: 70 }, + ]); + await syncUserOura("u1"); + expect(updateManyMock).not.toHaveBeenCalled(); + }); + it("writes the Sleep Score and SpO2 from the new collections", async () => { getConnMock.mockResolvedValue(CONN); fetchDailySleepMock.mockResolvedValue([ diff --git a/src/lib/oura/client.ts b/src/lib/oura/client.ts index e492c51bd..2081bfbc5 100644 --- a/src/lib/oura/client.ts +++ b/src/lib/oura/client.ts @@ -548,7 +548,6 @@ function mapSleepTimeline(s: OuraSleep): MappedMeasurement[] | null { if (Number.isNaN(onset.getTime())) return null; const out: MappedMeasurement[] = []; - let segIndex = 0; let i = 0; while (i < hyp.length) { const digit = hyp[i]; @@ -559,7 +558,15 @@ function mapSleepTimeline(s: OuraSleep): MappedMeasurement[] | null { if (stage) { const segStartMs = onset.getTime() + i * HYPNOGRAM_INTERVAL_MS; const segEndMs = segStartMs + run * HYPNOGRAM_INTERVAL_MS; - const fieldTag = `seg:${segIndex}`; + // Keyed by the run's own START instant, NOT a positional run index. A + // revised hypnogram re-segments the night (runs merge/split), so an + // index shifted every following run onto the wrong row — wrong + // boundaries updated in place plus phantom tail rows the night-total + // double-counted. A run's start instant is a stable coordinate: an + // unchanged run keeps its id (re-classification overwrites in place), + // and whatever the revision orphaned is swept by the record-scoped + // cleanup in the sync (which also clears legacy `seg:` rows). + const fieldTag = `seg:${new Date(segStartMs).toISOString()}`; out.push({ type: "SLEEP_DURATION", value: round2(run * HYPNOGRAM_INTERVAL_MIN), @@ -567,11 +574,10 @@ function mapSleepTimeline(s: OuraSleep): MappedMeasurement[] | null { measuredAt: new Date(segEndMs), fieldTag, sleepStage: stage, - // Record-scoped + segment-indexed so a nap's segments never collide - // with the main sleep's and a re-fetch overwrites in place. + // Record-scoped so a nap's segments never collide with the main + // sleep's and a re-fetch overwrites in place. externalId: `sleep:${s.id}:${fieldTag}`, }); - segIndex += 1; } i += run; } diff --git a/src/lib/oura/sync.ts b/src/lib/oura/sync.ts index 80ddc2834..928c3f07b 100644 --- a/src/lib/oura/sync.ts +++ b/src/lib/oura/sync.ts @@ -47,6 +47,10 @@ import { recomputeBucketsForMeasurement, } from "@/lib/rollups/measurement-rollups"; import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate"; +import { + sweepStaleSleepSegments, + type SleepSegmentSweep, +} from "@/lib/sleep/sweep-stale-segments"; import { fetchCardiovascularAge, fetchDailyActivity, @@ -133,6 +137,13 @@ export interface OuraCollectionFailure { export interface OuraFetchResult { readings: OuraMeasurementUpsert[]; failures: OuraCollectionFailure[]; + /** + * One sweep entry per fetched sleep record (`sleep::` prefix + + * that record's fresh SLEEP_DURATION externalIds). Drives the record-scoped + * cleanup of rows a revised hypnogram orphaned — including every legacy + * run-indexed `seg:` row — before the fresh set upserts. + */ + sleepSweeps: SleepSegmentSweep[]; } /** @@ -162,6 +173,12 @@ async function fetchAll( const start = new Date(now.getTime() - lookbackDays * 24 * 60 * 60 * 1000); const query = { startDate: ymd(start), endDate: ymd(now) }; + // Filled by the sleep collection's closure as it maps records; carried on + // the result so the caller can run the record-scoped sweep before the + // upsert. A collection that throws pushes nothing (the fetch is its first + // await), so a 401-retry cannot double-enter a record. + const sleepSweeps: SleepSegmentSweep[] = []; + const collections: OuraCollectionSpec[] = [ { name: "readiness", @@ -172,10 +189,27 @@ async function fetchAll( }, { name: "sleep", - collect: async (t) => - (await fetchSleep(t, query)).flatMap((s) => - toUpsert(mapSleep(s), "sleep"), - ), + collect: async (t) => { + const records = await fetchSleep(t, query); + const ups: OuraMeasurementUpsert[] = []; + for (const s of records) { + const rows = toUpsert(mapSleep(s), "sleep"); + ups.push(...rows); + // Record-scoped sweep entry: every SLEEP_DURATION row of this + // record keys under `sleep::` (hypnogram runs AND the + // stage-total fallback), so the prefix bounds the cleanup to this + // one record. Clears whatever a revised hypnogram orphaned — + // shifted run boundaries, legacy run-indexed `seg:` rows, and a + // stale stage-total set once the hypnogram appears. + sleepSweeps.push({ + prefix: `sleep:${s.id}:`, + keepIds: rows + .filter((r) => r.type === "SLEEP_DURATION") + .map((r) => r.externalId), + }); + } + return ups; + }, }, { name: "activity", @@ -256,14 +290,15 @@ async function fetchAll( const first = await attempt(accessToken, collections); if (!first.auth401) { - return { readings: first.readings, failures: first.failures }; + return { readings: first.readings, failures: first.failures, sleepSweeps }; } // Reactive refresh: the access token expired (a 401 on the shared token). // Refresh once (rotating both tokens) and retry ONLY the collections that // failed. A failed refresh throws so the caller parks reauth_required. const creds = await getOuraClientCredentials(userId); - if (!creds) return { readings: first.readings, failures: first.failures }; + if (!creds) + return { readings: first.readings, failures: first.failures, sleepSweeps }; const rotated = await refreshAccessToken(refreshToken, creds); // Compare-and-swap persist: on a lost race against a concurrent sync this @@ -276,12 +311,13 @@ async function fetchAll( refreshTokenCiphertext, ); if (!usableToken) - return { readings: first.readings, failures: first.failures }; + return { readings: first.readings, failures: first.failures, sleepSweeps }; const retry = await attempt(usableToken, first.failed); return { readings: [...first.readings, ...retry.readings], failures: retry.failures, + sleepSweeps, }; } @@ -322,6 +358,13 @@ export async function syncUserOura( throw err; } + // Clear whatever an earlier scoring left under the re-fetched sleep records + // before the fresh set upserts (mirrors Google Health's replace-by-window + // order). Best-effort inside the helper — a sweep failure never fails the + // sync; the record stays inside the lookback window, so the next tick + // retries it. + await sweepStaleSleepSegments(userId, "OURA", result.sleepSweeps); + // Import everything the healthy collections returned regardless of whether a // sibling collection failed — one bad collection must not blank the source. const imported = await upsertOuraMeasurements(userId, result.readings); @@ -393,6 +436,10 @@ export async function upsertOuraMeasurements( unit: r.unit, measuredAt: r.measuredAt, sleepStage: r.sleepStage ?? null, + // No-op on a live row, a deliberate RESURRECTION on a tombstoned + // one — Oura is the source of truth for its own rows, so a + // re-fetched reading brings the row back (mirrors Google / Fitbit). + deletedAt: null, syncVersion: { increment: 1 }, }, }); diff --git a/src/lib/polar/__tests__/client.test.ts b/src/lib/polar/__tests__/client.test.ts index 9798679d9..6f8fbfe70 100644 --- a/src/lib/polar/__tests__/client.test.ts +++ b/src/lib/polar/__tests__/client.test.ts @@ -250,10 +250,13 @@ describe("mapSleep", () => { ); expect(instants.size).toBe(3); - // Reconstructed flag + indexed externalId keep the segment rows distinct. + // Reconstructed flag + stage-tagged externalId keep the segment rows + // distinct. No positional index: an index renumbered on a re-score when a + // stage's duration flipped 0↔positive, minting fresh ids that inserted a + // duplicate night. expect(core.reconstructed).toBe(true); - expect(core.externalId).toBe("sleep:2026-06-10:seg:sleep_core:0"); - expect(rem.externalId).toBe("sleep:2026-06-10:seg:sleep_rem:2"); + expect(core.externalId).toBe("sleep:2026-06-10:seg:sleep_core"); + expect(rem.externalId).toBe("sleep:2026-06-10:seg:sleep_rem"); // IN_BED envelope + score stamp at the sleep END instant. const endMs = Date.parse("2026-06-10T07:00:00+02:00"); @@ -287,15 +290,45 @@ describe("mapSleep", () => { expect(awake).toBeDefined(); expect(awake.value).toBe(15); expect(awake.reconstructed).toBe(true); - expect(awake.externalId).toBe("sleep:2026-06-10:seg:sleep_awake:0"); - // AWAKE ends at +15m; CORE then starts there and is index 1. + expect(awake.externalId).toBe("sleep:2026-06-10:seg:sleep_awake"); + // AWAKE ends at +15m; CORE then starts there. expect(awake.measuredAt.getTime()).toBe(onset + 15 * 60_000); const core = mapped.find((m) => m.sleepStage === "CORE")!; - expect(core.externalId).toBe("sleep:2026-06-10:seg:sleep_core:1"); + expect(core.externalId).toBe("sleep:2026-06-10:seg:sleep_core"); expect(core.measuredAt.getTime()).toBe(onset + (15 + 60) * 60_000); }); + it("keeps IDENTICAL externalIds across a re-score that adds an interruption block", () => { + // The same night scored twice: first with no interruption time, then with + // a positive AWAKE block. Under the retired positional index the AWAKE + // 0↔positive flip renumbered CORE/DEEP/REM (0,1,2 → 1,2,3), minting + // fresh externalIds the upsert then INSERTED next to the old rows — the + // night double-counted. The stage-tagged ids must be identical. + const night = { + date: "2026-06-10", + sleep_start_time: "2026-06-09T23:00:00+02:00", + sleep_end_time: "2026-06-10T07:00:00+02:00", + light_sleep: 3600, + deep_sleep: 1800, + rem_sleep: 5400, + }; + const idsByStage = (rows: ReturnType) => + new Map( + rows + .filter((m) => m.reconstructed) + .map((m) => [m.sleepStage, m.externalId]), + ); + const first = idsByStage(mapSleep(night)); + const rescored = idsByStage( + mapSleep({ ...night, total_interruption_duration: 900 }), + ); + for (const stage of ["CORE", "DEEP", "REM"] as const) { + expect(rescored.get(stage)).toBe(first.get(stage)); + } + expect(rescored.get("AWAKE")).toBe("sleep:2026-06-10:seg:sleep_awake"); + }); + it("omits AWAKE when no interruption time is reported", () => { const mapped = mapSleep({ date: "2026-06-10", @@ -306,9 +339,9 @@ describe("mapSleep", () => { rem_sleep: 5400, }); expect(mapped.find((m) => m.sleepStage === "AWAKE")).toBeUndefined(); - // CORE stays index 0 when AWAKE is absent. + // CORE keeps the same stage-tagged id whether AWAKE is present or not. expect(mapped.find((m) => m.sleepStage === "CORE")!.externalId).toBe( - "sleep:2026-06-10:seg:sleep_core:0", + "sleep:2026-06-10:seg:sleep_core", ); }); diff --git a/src/lib/polar/__tests__/sync.test.ts b/src/lib/polar/__tests__/sync.test.ts index 2e0ecdd94..cf3e9c0a3 100644 --- a/src/lib/polar/__tests__/sync.test.ts +++ b/src/lib/polar/__tests__/sync.test.ts @@ -8,6 +8,7 @@ const { fetchCardioLoadsMock, fetchSpo2Mock, upsertMock, + updateManyMock, recordSuccessMock, recordFailureMock, recomputeMock, @@ -20,6 +21,7 @@ const { fetchCardioLoadsMock: vi.fn(), fetchSpo2Mock: vi.fn(), upsertMock: vi.fn(), + updateManyMock: vi.fn(), recordSuccessMock: vi.fn(), recordFailureMock: vi.fn(), recomputeMock: vi.fn(), @@ -31,7 +33,7 @@ vi.mock("../credentials", () => ({ })); vi.mock("@/lib/db", () => ({ - prisma: { measurement: { upsert: upsertMock } }, + prisma: { measurement: { upsert: upsertMock, updateMany: updateManyMock } }, })); vi.mock("@/lib/integrations/status", async (importOriginal) => ({ @@ -75,6 +77,7 @@ beforeEach(() => { fetchCardioLoadsMock.mockReset(); fetchSpo2Mock.mockReset(); upsertMock.mockReset().mockResolvedValue({}); + updateManyMock.mockReset().mockResolvedValue({ count: 0 }); recordSuccessMock.mockReset().mockResolvedValue(undefined); recordFailureMock.mockReset().mockResolvedValue(undefined); recomputeMock.mockReset().mockResolvedValue(undefined); @@ -131,7 +134,7 @@ describe("syncUserPolar", () => { expect(arg.create.value).toBe(123.45); }); - it("keeps reconstructed sleep segments distinct under their indexed externalId", async () => { + it("keeps reconstructed sleep segments distinct under their stage-tagged externalId", async () => { getConnMock.mockResolvedValue(CONN); fetchSleepsMock.mockResolvedValue([ { @@ -147,12 +150,77 @@ describe("syncUserPolar", () => { const externalIds = upsertMock.mock.calls.map( (c) => c[0].where.userId_type_source_externalId.externalId, ); - expect(externalIds).toContain("sleep:2026-06-10:seg:sleep_core:0"); - expect(externalIds).toContain("sleep:2026-06-10:seg:sleep_rem:2"); + expect(externalIds).toContain("sleep:2026-06-10:seg:sleep_core"); + expect(externalIds).toContain("sleep:2026-06-10:seg:sleep_rem"); // The several segment rows stay distinct (no collision). expect(new Set(externalIds).size).toBe(externalIds.length); }); + it("sweeps stale segment rows per fetched night: live-only, seg-prefixed, notIn the fresh set, soft-delete (v1.28.25)", async () => { + getConnMock.mockResolvedValue(CONN); + fetchSleepsMock.mockResolvedValue([ + { + date: "2026-06-10", + sleep_start_time: "2026-06-09T23:00:00+02:00", + sleep_end_time: "2026-06-10T07:00:00+02:00", + light_sleep: 3600, + deep_sleep: 1800, + rem_sleep: 5400, + }, + ]); + + await syncUserPolar("u1"); + + expect(updateManyMock).toHaveBeenCalledTimes(1); + const arg = updateManyMock.mock.calls[0]![0] as { + where: { + userId: string; + source: string; + type: string; + deletedAt: null; + externalId: { startsWith: string; notIn: string[] }; + }; + data: Record; + }; + expect(arg.where.userId).toBe("u1"); + expect(arg.where.source).toBe("POLAR"); + expect(arg.where.type).toBe("SLEEP_DURATION"); + expect(arg.where.deletedAt).toBeNull(); + // Bounded to THIS night's reconstructed segments. The prefix stays on + // `:seg:` (not the whole `sleep::` slice) because the IN_BED + // envelope keys on its measuredAt's UTC date, which can drift a calendar + // day from `date` — a broader bound could cross nights. + expect(arg.where.externalId.startsWith).toBe("sleep:2026-06-10:seg:"); + // Every fresh SLEEP_DURATION id is protected (segments + IN_BED). + expect(arg.where.externalId.notIn.sort()).toEqual([ + "sleep:2026-06-10:seg:sleep_core", + "sleep:2026-06-10:seg:sleep_deep", + "sleep:2026-06-10:seg:sleep_rem", + "sleep:2026-06-10:sleep_in_bed", + ]); + // Soft delete only. + expect(arg.data).toEqual({ deletedAt: expect.any(Date) }); + + // A legacy indexed row for this night falls inside the sweep. + const legacyId = "sleep:2026-06-10:seg:sleep_core:0"; + expect(legacyId.startsWith(arg.where.externalId.startsWith)).toBe(true); + expect(arg.where.externalId.notIn).not.toContain(legacyId); + + // Replace-then-write: the sweep runs before the first sleep upsert. + expect(updateManyMock.mock.invocationCallOrder[0]!).toBeLessThan( + upsertMock.mock.invocationCallOrder[0]!, + ); + }); + + it("never sweeps when the fetch returns no sleep records", async () => { + getConnMock.mockResolvedValue(CONN); + fetchRechargesMock.mockResolvedValue([ + { date: "2026-06-10", nightly_recharge_status: 3 }, + ]); + await syncUserPolar("u1"); + expect(updateManyMock).not.toHaveBeenCalled(); + }); + it("is idempotent — a re-sync upserts on the same externalId", async () => { getConnMock.mockResolvedValue(CONN); fetchRechargesMock.mockResolvedValue([ diff --git a/src/lib/polar/client.ts b/src/lib/polar/client.ts index 2b0ce7977..21f5c3fdf 100644 --- a/src/lib/polar/client.ts +++ b/src/lib/polar/client.ts @@ -467,7 +467,7 @@ export function mapNightlyRecharge( * emitting one timed row per segment ending at the running cursor, and stamp the * `IN_BED` envelope + the score row at `sleep_end_time`. The ORDER is synthetic, * so every reconstructed segment is flagged `reconstructed: true` and keyed by - * an indexed externalId so the several rows of one night stay distinct. + * a stage-tagged externalId so the several rows of one night stay distinct. * * If the night carries no usable start/end (older records, missing fields) we * fall back to a midnight-UTC anchor and emit untimed stage rows — degraded but @@ -517,10 +517,12 @@ export function mapSleep(s: PolarSleep): MappedMeasurement[] { measuredAt: new Date(endMs), fieldTag: "sleep_in_bed", }, - // Indexed externalId keeps the several segment rows of one night - // distinct under userId_type_source_externalId. - externalIdFor: (fieldTag, index) => - `sleep:${s.date}:seg:${fieldTag}:${index}`, + // Stage-tagged externalId keeps the several segment rows of one night + // distinct under userId_type_source_externalId (one segment per stage, + // no positional index — an index renumbered on a re-score when a + // stage's duration flipped 0↔positive, minting fresh ids that + // double-counted the night). + externalIdFor: (fieldTag) => `sleep:${s.date}:seg:${fieldTag}`, }), ); diff --git a/src/lib/polar/sync.ts b/src/lib/polar/sync.ts index 8552386e3..f89ff5753 100644 --- a/src/lib/polar/sync.ts +++ b/src/lib/polar/sync.ts @@ -48,6 +48,10 @@ import { mapSpo2, type MappedMeasurement, } from "./client"; +import { + sweepStaleSleepSegments, + type SleepSegmentSweep, +} from "@/lib/sleep/sweep-stale-segments"; import { getPolarConnection } from "./credentials"; import { PolarApiError, classifyPolarError } from "./response-classifier"; @@ -98,6 +102,7 @@ export async function syncUserPolar(userId: string): Promise { if (!conn) return 0; const readings: PolarMeasurementUpsert[] = []; + const sleepSweeps: SleepSegmentSweep[] = []; try { const [recharges, sleeps, activities, cardioLoads, spo2Tests] = await Promise.all([ @@ -111,7 +116,21 @@ export async function syncUserPolar(userId: string): Promise { readings.push(...toUpsert(mapNightlyRecharge(r), "recharge")); } for (const s of sleeps) { - readings.push(...toUpsert(mapSleep(s), "sleep")); + const rows = toUpsert(mapSleep(s), "sleep"); + readings.push(...rows); + // Night-scoped sweep entry: the reconstructed segments of this date all + // key under `sleep::seg:` (mapper-supplied). Any live row under + // that prefix this fetch did NOT re-produce is a re-score orphan or a + // legacy `:seg::` indexed row — tombstoned before the upsert. + // The prefix deliberately stays on `:seg:` — the IN_BED envelope keys + // on its measuredAt's UTC date, which can drift a calendar day from + // `s.date`, so a broader `sleep::` bound could cross nights. + sleepSweeps.push({ + prefix: `sleep:${s.date}:seg:`, + keepIds: rows + .filter((r) => r.type === "SLEEP_DURATION") + .map((r) => r.externalId), + }); } for (const a of activities) { readings.push(...toUpsert(mapActivity(a), "activity")); @@ -136,6 +155,11 @@ export async function syncUserPolar(userId: string): Promise { throw err; } + // Clear whatever an earlier scoring left under the re-fetched nights before + // the fresh set upserts (mirrors Google Health's replace-by-window order). + // Best-effort inside the helper — a sweep failure never fails the sync. + await sweepStaleSleepSegments(userId, "POLAR", sleepSweeps); + const imported = await upsertPolarMeasurements(userId, readings); await recordSyncSuccess(userId, "polar"); return imported; @@ -183,6 +207,10 @@ export async function upsertPolarMeasurements( unit: r.unit, measuredAt: r.measuredAt, sleepStage: r.sleepStage ?? null, + // No-op on a live row, a deliberate RESURRECTION on a tombstoned + // one — Polar is the source of truth for its own rows, so a + // re-fetched reading brings the row back (mirrors Google / Fitbit). + deletedAt: null, syncVersion: { increment: 1 }, }, }); diff --git a/src/lib/sleep/__tests__/reconstruct-timeline.test.ts b/src/lib/sleep/__tests__/reconstruct-timeline.test.ts index be1170302..27f66e771 100644 --- a/src/lib/sleep/__tests__/reconstruct-timeline.test.ts +++ b/src/lib/sleep/__tests__/reconstruct-timeline.test.ts @@ -14,7 +14,7 @@ describe("reconstructContiguousSleepTimeline", () => { { durationMs: 30 * 60_000, stage: "DEEP", fieldTag: "sleep_deep" }, { durationMs: 90 * 60_000, stage: "REM", fieldTag: "sleep_rem" }, ], - externalIdFor: (tag, i) => `night:seg:${tag}:${i}`, + externalIdFor: (tag) => `night:seg:${tag}`, }); const core = rows.find((r) => r.sleepStage === "CORE")!; const deep = rows.find((r) => r.sleepStage === "DEEP")!; @@ -27,18 +27,18 @@ describe("reconstructContiguousSleepTimeline", () => { expect(new Set(rows.map((r) => r.measuredAt.getTime())).size).toBe(3); }); - it("flags every laid segment reconstructed and keys an indexed externalId", () => { + it("flags every laid segment reconstructed and keys a stage-tagged externalId", () => { const rows = reconstructContiguousSleepTimeline({ startMs: onset, stages: [ { durationMs: 15 * 60_000, stage: "AWAKE", fieldTag: "sleep_awake" }, { durationMs: 60 * 60_000, stage: "CORE", fieldTag: "sleep_core" }, ], - externalIdFor: (tag, i) => `night:seg:${tag}:${i}`, + externalIdFor: (tag) => `night:seg:${tag}`, }); expect(rows.every((r) => r.reconstructed === true)).toBe(true); - expect(rows[0].externalId).toBe("night:seg:sleep_awake:0"); - expect(rows[1].externalId).toBe("night:seg:sleep_core:1"); + expect(rows[0].externalId).toBe("night:seg:sleep_awake"); + expect(rows[1].externalId).toBe("night:seg:sleep_core"); }); it("skips non-positive, null, undefined and non-finite durations", () => { @@ -51,14 +51,48 @@ describe("reconstructContiguousSleepTimeline", () => { { durationMs: Number.NaN, stage: "REM", fieldTag: "sleep_rem" }, { durationMs: 30 * 60_000, stage: "CORE", fieldTag: "sleep_core" }, ], - externalIdFor: (tag, i) => `night:seg:${tag}:${i}`, + externalIdFor: (tag) => `night:seg:${tag}`, }); expect(rows).toHaveLength(1); - // The only laid stage keeps index 0 (skipped stages do not consume indices). - expect(rows[0].externalId).toBe("night:seg:sleep_core:0"); + // The laid stage keys on its tag alone — skipped stages cannot influence + // the id of any other stage (no positional index to renumber). + expect(rows[0].externalId).toBe("night:seg:sleep_core"); expect(rows[0].measuredAt.getTime()).toBe(onset + 30 * 60_000); }); + it("keeps stable externalIds when a re-score flips a stage 0↔positive", () => { + // First scoring: no awake time. Re-score: WHOOP/Polar now report a + // positive AWAKE block. Under the retired positional index the re-score + // renumbered CORE/DEEP (0→1, 1→2), minting fresh externalIds the upsert + // then INSERTED next to the old rows — the night double-counted. The + // stage-tagged key must be identical across both scorings. + const stages = (awakeMs: number) => + [ + { durationMs: awakeMs, stage: "AWAKE", fieldTag: "sleep_awake" }, + { durationMs: 60 * 60_000, stage: "CORE", fieldTag: "sleep_core" }, + { durationMs: 30 * 60_000, stage: "DEEP", fieldTag: "sleep_deep" }, + ] as const; + const externalIdFor = (tag: string) => `night:seg:${tag}`; + + const first = reconstructContiguousSleepTimeline({ + startMs: onset, + stages: stages(0), + externalIdFor, + }); + const rescored = reconstructContiguousSleepTimeline({ + startMs: onset, + stages: stages(15 * 60_000), + externalIdFor, + }); + + const idsOf = (rows: typeof first, stage: string) => + rows.find((r) => r.sleepStage === stage)?.externalId; + // CORE and DEEP keep their ids — the awake flip cannot renumber them. + expect(idsOf(rescored, "CORE")).toBe(idsOf(first, "CORE")); + expect(idsOf(rescored, "DEEP")).toBe(idsOf(first, "DEEP")); + expect(idsOf(rescored, "AWAKE")).toBe("night:seg:sleep_awake"); + }); + it("appends a single IN_BED envelope at the given END instant", () => { const rows = reconstructContiguousSleepTimeline({ startMs: onset, @@ -70,7 +104,7 @@ describe("reconstructContiguousSleepTimeline", () => { measuredAt: new Date(end), fieldTag: "sleep_in_bed", }, - externalIdFor: (tag, i) => `night:seg:${tag}:${i}`, + externalIdFor: (tag) => `night:seg:${tag}`, }); const inBed = rows.find((r) => r.sleepStage === "IN_BED")!; expect(inBed.measuredAt.getTime()).toBe(end); @@ -91,7 +125,7 @@ describe("reconstructContiguousSleepTimeline", () => { measuredAt: new Date(end), fieldTag: "sleep_in_bed", }, - externalIdFor: (tag, i) => `night:seg:${tag}:${i}`, + externalIdFor: (tag) => `night:seg:${tag}`, }); expect(rows.find((r) => r.sleepStage === "IN_BED")).toBeUndefined(); }); diff --git a/src/lib/sleep/__tests__/sweep-stale-segments.test.ts b/src/lib/sleep/__tests__/sweep-stale-segments.test.ts new file mode 100644 index 000000000..9abfd20bd --- /dev/null +++ b/src/lib/sleep/__tests__/sweep-stale-segments.test.ts @@ -0,0 +1,109 @@ +/** + * Pins the safety contract of `sweepStaleSleepSegments` — the session-scoped + * cleanup that keeps a re-scored wearable night from double-counting (the + * prefix-bound sibling of Google Health's `replaceStaleGoogleHealthSleep`, + * whose test this mirrors). The query MUST be tightly bounded: only LIVE rows, + * only `SLEEP_DURATION`, only ONE source, only under ONE session's externalId + * prefix, and NEVER a row in the fresh keep-set. An entry with no fresh ids + * (or no prefix) is skipped entirely — an unbounded delete would be data loss. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { updateManyMock } = vi.hoisted(() => ({ + updateManyMock: vi.fn(async () => ({ count: 0 })), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { measurement: { updateMany: updateManyMock } }, +})); + +import { sweepStaleSleepSegments } from "../sweep-stale-segments"; + +beforeEach(() => { + updateManyMock.mockClear(); + updateManyMock.mockResolvedValue({ count: 2 }); +}); + +describe("sweepStaleSleepSegments — bounded session-scoped sweep", () => { + it("soft-deletes only live SLEEP_DURATION rows under the session prefix, excluding the fresh set", async () => { + const keepIds = ["night-1:seg:sleep_core", "night-1:seg:sleep_deep"]; + + const removed = await sweepStaleSleepSegments("user-1", "WHOOP", [ + { prefix: "night-1:", keepIds }, + ]); + + expect(removed).toBe(2); + expect(updateManyMock).toHaveBeenCalledTimes(1); + const arg = (updateManyMock.mock.calls[0]! as unknown[])[0] as { + where: Record; + data: Record; + }; + expect(arg.where).toEqual({ + userId: "user-1", + source: "WHOOP", + type: "SLEEP_DURATION", + deletedAt: null, + externalId: { startsWith: "night-1:", notIn: keepIds }, + }); + // A soft delete — the row is tombstoned, never hard-removed. + expect(arg.data).toEqual({ deletedAt: expect.any(Date) }); + }); + + it("sweeps a legacy indexed row: inside the prefix, outside the keep-set", async () => { + // The legacy volatile formats (`night-1:seg:sleep_core:1`, Withings' + // running index, Oura's `seg:`) all live under the session prefix and + // can never appear in a fresh keep-set — so the where-clause the sweep + // issues matches them by construction. Pin that with a literal check + // against the emitted filter. + await sweepStaleSleepSegments("user-1", "WHOOP", [ + { prefix: "night-1:", keepIds: ["night-1:seg:sleep_core"] }, + ]); + const where = ( + (updateManyMock.mock.calls[0]! as unknown[])[0] as { + where: { externalId: { startsWith: string; notIn: string[] } }; + } + ).where; + const legacyId = "night-1:seg:sleep_core:1"; + expect(legacyId.startsWith(where.externalId.startsWith)).toBe(true); + expect(where.externalId.notIn).not.toContain(legacyId); + // …while the fresh row is protected. + expect(where.externalId.notIn).toContain("night-1:seg:sleep_core"); + }); + + it("skips an entry with no fresh ids or no prefix (never an unbounded delete)", async () => { + await sweepStaleSleepSegments("user-1", "OURA", [ + { prefix: "sleep:rec-1:", keepIds: [] }, + { prefix: "", keepIds: ["sleep:rec-2:seg:x"] }, + ]); + expect(updateManyMock).not.toHaveBeenCalled(); + }); + + it("is bounded to the sessions of THIS fetch — one updateMany per entry, each under its own prefix", async () => { + await sweepStaleSleepSegments("user-1", "POLAR", [ + { prefix: "sleep:2026-06-10:seg:", keepIds: ["a"] }, + { prefix: "sleep:2026-06-11:seg:", keepIds: ["b"] }, + ]); + expect(updateManyMock).toHaveBeenCalledTimes(2); + const prefixes = updateManyMock.mock.calls.map( + (c) => + ( + (c as unknown[])[0] as { + where: { externalId: { startsWith: string } }; + } + ).where.externalId.startsWith, + ); + expect(prefixes).toEqual([ + "sleep:2026-06-10:seg:", + "sleep:2026-06-11:seg:", + ]); + }); + + it("never throws when the cleanup query fails (best-effort)", async () => { + updateManyMock.mockRejectedValueOnce(new Error("db down")); + await expect( + sweepStaleSleepSegments("user-1", "WITHINGS", [ + { prefix: "withings:sleep:u:1:", keepIds: ["withings:sleep:u:1:5"] }, + ]), + ).resolves.toBe(0); + }); +}); diff --git a/src/lib/sleep/reconstruct-timeline.ts b/src/lib/sleep/reconstruct-timeline.ts index cdb5129b5..23acd3fb7 100644 --- a/src/lib/sleep/reconstruct-timeline.ts +++ b/src/lib/sleep/reconstruct-timeline.ts @@ -63,11 +63,16 @@ export interface ReconstructTimelineOptions { /** Optional IN_BED envelope row. */ inBed?: ReconstructedInBed; /** - * Builds the indexed externalId for a laid segment so the several rows of one - * night stay distinct under `userId_type_source_externalId`. Called with the - * segment's fieldTag and its running index. + * Builds the externalId for a laid segment so the several rows of one night + * stay distinct under `userId_type_source_externalId`. Called with the + * segment's fieldTag ONLY — the builder emits at most one segment per stage, + * so the tag alone is unique within a night. A positional index was + * deliberately dropped from the key: it skipped zero-duration stages, so a + * 0↔positive flip on a vendor re-score renumbered every following stage, + * minted fresh externalIds, and the upsert double-counted the night instead + * of overwriting (the Google Health v1.28.18 bug class). */ - externalIdFor: (fieldTag: string, index: number) => string; + externalIdFor: (fieldTag: string) => string; } const MS_TO_MIN = 1 / 60_000; @@ -79,8 +84,9 @@ function round2(n: number): number { /** * Lay the asleep/awake stages contiguously from `startMs` in the given order, * emitting one timed `SLEEP_DURATION` row per stage (`measuredAt` = that - * segment's END), each flagged `reconstructed: true` and keyed by an indexed - * externalId. Appends a single IN_BED envelope row when `inBed` is provided. + * segment's END), each flagged `reconstructed: true` and keyed by a + * stage-tagged externalId. Appends a single IN_BED envelope row when `inBed` + * is provided. */ export function reconstructContiguousSleepTimeline( opts: ReconstructTimelineOptions, @@ -89,7 +95,6 @@ export function reconstructContiguousSleepTimeline( const out: ReconstructedSleepRow[] = []; let cursor = startMs; - let segIndex = 0; for (const { durationMs, stage, fieldTag } of stages) { if ( typeof durationMs !== "number" || @@ -105,12 +110,11 @@ export function reconstructContiguousSleepTimeline( unit: "minutes", measuredAt: new Date(segEnd), fieldTag, - externalId: externalIdFor(fieldTag, segIndex), + externalId: externalIdFor(fieldTag), sleepStage: stage, reconstructed: true, }); cursor = segEnd; - segIndex += 1; } if ( diff --git a/src/lib/sleep/sweep-stale-segments.ts b/src/lib/sleep/sweep-stale-segments.ts new file mode 100644 index 000000000..e1503d692 --- /dev/null +++ b/src/lib/sleep/sweep-stale-segments.ts @@ -0,0 +1,83 @@ +/** + * Session-scoped sweep of stale sleep-segment rows. + * + * Wearables re-score a night after the fact (WHOOP re-scores, Withings + * re-aggregates, Oura revises the hypnogram, Polar re-settles the stage + * totals). Before the stable-externalId fix, a re-scored night minted FRESH + * segment externalIds (positional indexes renumbered, boundaries shifted), so + * the upsert inserted a second set of rows next to the first and the + * night-total silently double-counted. The externalIds are stable now, but two + * classes of orphans remain: rows a re-score genuinely dropped (a segment that + * no longer exists in the fresh scoring) and every legacy row still keyed on + * the old volatile format. This sweep clears both, so the id change self-heals + * without a migration — the same posture as Google Health's + * `replaceStaleGoogleHealthSleep`, expressed as an externalId-prefix bound + * instead of a time-window bound because these integrations key every segment + * row under a per-session externalId prefix. + * + * Safety contract (pinned by `__tests__/sweep-stale-segments.test.ts`): + * - Bounded to the sessions of THIS fetch — each entry sweeps only rows whose + * `externalId` starts with that one session's prefix. Never a blanket + * delete. + * - LIVE rows only (`deletedAt: null`), `SLEEP_DURATION` only, one source. + * - The fresh id set (`keepIds`) is never touched; an entry with no fresh ids + * is skipped entirely (a mapper hiccup must not wipe a night). + * - Soft-delete (`deletedAt = now`), never a hard remove. + * - Best-effort: a failed sweep warns and moves on — it never fails the + * user's sync. The session stays inside the integration's re-fetch window, + * so the next tick retries the sweep. + */ +import type { MeasurementSource } from "@/generated/prisma/client"; + +import { prisma } from "@/lib/db"; +import { getEvent } from "@/lib/logging/context"; + +/** One just-fetched sleep session: its externalId prefix + the fresh ids. */ +export interface SleepSegmentSweep { + /** + * The session-scoped externalId prefix (e.g. `:` for WHOOP, + * `withings:sleep:::` for Withings). Every live + * `SLEEP_DURATION` row under it that is not in `keepIds` is considered a + * re-score orphan or a legacy-format duplicate. + */ + prefix: string; + /** The fresh externalIds this fetch produced for the session — never swept. */ + keepIds: string[]; +} + +/** + * Soft-delete every live `SLEEP_DURATION` row of `source` whose externalId + * falls under one of the given session prefixes but was not re-produced by + * this fetch. Returns the number of rows tombstoned. + */ +export async function sweepStaleSleepSegments( + userId: string, + source: MeasurementSource, + sessions: SleepSegmentSweep[], +): Promise { + let removed = 0; + for (const s of sessions) { + // An empty prefix would unbound the sweep to the whole source; an empty + // keep-set means this fetch produced nothing for the session (unscored / + // mapper hiccup) — in both cases deleting would be data loss, so skip. + if (!s.prefix || s.keepIds.length === 0) continue; + try { + const res = await prisma.measurement.updateMany({ + where: { + userId, + source, + type: "SLEEP_DURATION", + deletedAt: null, + externalId: { startsWith: s.prefix, notIn: s.keepIds }, + }, + data: { deletedAt: new Date() }, + }); + removed += res.count; + } catch (err) { + getEvent()?.addWarning( + `${source.toLowerCase()}: stale sleep-segment sweep failed for prefix ${s.prefix}: ${err}`, + ); + } + } + return removed; +} diff --git a/src/lib/targets/build-response.ts b/src/lib/targets/build-response.ts index c00ffc3ee..8cab830da 100644 --- a/src/lib/targets/build-response.ts +++ b/src/lib/targets/build-response.ts @@ -245,29 +245,42 @@ export async function buildTargetsResponse(user: AuthedUser) { // Absolute latest measurement per type within a one-year floor. // // Prisma's `distinct` does not compile to Postgres `DISTINCT ON` — the - // driver dedups after pulling the rows, so an unbounded scan on a - // 347 k-row tenant pulls every measurement back to Node only to drop - // all but seven (audit Critical Finding #2, v1.4.40). The 365-day - // floor caps the planner-side walk to the existing - // `(user_id, type, measured_at)` index range; the trade-off is that a - // user who hasn't measured a metric in over a year sees the tile's - // `current` value as null instead of pulling a stale year-old reading - // forward — which is the correct behaviour for the consistency strip - // (the strip already gates on "fewer than 3 readings in 30 days" → - // insufficientData). - limit(() => - prisma.measurement.findMany({ - where: { - userId, - type: { in: types }, - measuredAt: { gte: oneYearAgo }, - deletedAt: null, - }, - orderBy: { measuredAt: "desc" }, - distinct: ["type"], - select: { type: true, value: true }, - }), - ), + // driver dedups after pulling the rows, so even with the 365-day floor + // a dense-type tenant (per-sample PULSE, hourly HR buckets) shipped a + // year of raw rows back to Node only to drop all but seven (audit + // Critical Finding #2, v1.4.40 flagged the unbounded variant; the + // bounded one stayed O(rows-in-year)). Real `DISTINCT ON` resolves it + // index-side: one row per type, ordered walk over the existing + // `(user_id, type, measured_at)` index. The 365-day floor keeps its + // documented product semantics — a metric not measured in over a year + // reads `current: null` instead of pulling a stale reading forward. + limit(() => { + // Whitelist-splice: `types` is the closed compile-time enum list + // above; assert each member against the enum-literal shape before + // splicing (same pattern as the rollup aggregates in + // `measurement-rollups.ts`). user id + date bind as $1 / $2. + for (const t of types) { + if (!/^[A-Z0-9_]+$/.test(t)) { + throw new Error(`invalid measurement type: ${t}`); + } + } + const typeList = types.map((t) => `'${t}'::"measurement_type"`).join(","); + return prisma.$queryRawUnsafe< + Array<{ type: MeasurementType; value: number }> + >( + `SELECT DISTINCT ON (m."type") + m."type"::text AS type, + m."value" AS value + FROM measurements m + WHERE m."user_id" = $1 + AND m."type" IN (${typeList}) + AND m."measured_at" >= $2 + AND m."deleted_at" IS NULL + ORDER BY m."type" ASC, m."measured_at" DESC`, + userId, + oneYearAgo, + ); + }), // Sleep stage rows for the per-night reconstruction (section 4). limit(() => prisma.measurement.findMany({ diff --git a/src/lib/whoop/__tests__/client.test.ts b/src/lib/whoop/__tests__/client.test.ts index 78edaa9f7..b5e4cab9c 100644 --- a/src/lib/whoop/__tests__/client.test.ts +++ b/src/lib/whoop/__tests__/client.test.ts @@ -375,14 +375,52 @@ describe("mapSleep", () => { expect(r.reconstructed).toBeUndefined(); }); - it("gives each reconstructed segment a distinct indexed externalId", () => { + it("gives each reconstructed segment a distinct stage-tagged externalId", () => { const ids = mapSleep(base) .filter((r) => r.reconstructed) .map((r) => r.externalId); - expect(ids.every((id) => id?.startsWith("sleep-uuid:seg:"))).toBe(true); + // One segment per stage — the tag alone keys the row. No positional index: + // an index renumbered on a re-score whenever a stage's duration flipped + // 0↔positive, minting fresh externalIds that double-counted the night. + expect(ids).toEqual([ + "sleep-uuid:seg:sleep_awake", + "sleep-uuid:seg:sleep_core", + "sleep-uuid:seg:sleep_deep", + "sleep-uuid:seg:sleep_rem", + ]); expect(new Set(ids).size).toBe(ids.length); }); + it("keeps IDENTICAL externalIds across a re-score that flips a stage 0↔positive", () => { + // First scoring: no awake time (the leading AWAKE segment is skipped). + // Re-score of the SAME record: WHOOP now reports a positive awake block. + // Under the retired indexed key the re-score renumbered CORE/DEEP/REM, + // so the upsert inserted a second set instead of overwriting. + const first = mapSleep({ + ...base, + score: { + ...base.score!, + stage_summary: { + ...base.score!.stage_summary, + total_awake_time_milli: 0, + }, + }, + }); + const rescored = mapSleep(base); + + const idsByStage = (rows: typeof first) => + new Map( + rows + .filter((r) => r.reconstructed) + .map((r) => [r.sleepStage, r.externalId]), + ); + const a = idsByStage(first); + const b = idsByStage(rescored); + for (const stage of ["CORE", "DEEP", "REM"] as const) { + expect(b.get(stage)).toBe(a.get(stage)); + } + }); + it("sums SLEEP_NEED components ms→min", () => { const need = mapSleep(base).find((r) => r.type === "SLEEP_NEED"); expect(need?.value).toBe(490); // 450 + 30 + 10 + 0 diff --git a/src/lib/whoop/__tests__/sync-sleep-sweep.test.ts b/src/lib/whoop/__tests__/sync-sleep-sweep.test.ts new file mode 100644 index 000000000..b821b2747 --- /dev/null +++ b/src/lib/whoop/__tests__/sync-sleep-sweep.test.ts @@ -0,0 +1,143 @@ +/** + * Pins the record-scoped stale-segment sweep in the WHOOP sleep sync + * (v1.28.25). A WHOOP re-score used to renumber the reconstructed segments' + * indexed externalIds and insert a duplicate night; the ids are stage-tagged + * (stable) now, and the sync sweeps whatever an earlier scoring left under a + * re-fetched record's prefix — including every legacy `:seg::` row. + * The sweep must be bounded to the records of THIS fetch, live-only, + * SLEEP_DURATION-only, keep-protected, and soft-delete. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { updateManyMock, upsertMeasurementsMock } = vi.hoisted(() => ({ + updateManyMock: vi.fn(async () => ({ count: 0 })), + upsertMeasurementsMock: vi.fn(async () => 1), +})); + +vi.mock("@/lib/db", () => ({ + prisma: { + measurement: { updateMany: updateManyMock }, + whoopConnection: { + findUnique: vi.fn(async () => ({ + lastSyncedAt: null, + resourceCursors: null, + })), + }, + }, +})); + +vi.mock("../sync", async (importOriginal) => ({ + ...(await importOriginal()), + getValidToken: vi.fn(async () => ({ accessToken: "tok" })), + markResourceSynced: vi.fn(async () => {}), + upsertWhoopMeasurements: upsertMeasurementsMock, +})); + +const { fetchSleepsMock } = vi.hoisted(() => ({ fetchSleepsMock: vi.fn() })); +vi.mock("../client", async (importOriginal) => ({ + ...(await importOriginal()), + fetchSleeps: fetchSleepsMock, + fetchSleepById: fetchSleepsMock, +})); + +import type { WhoopSleep } from "../client"; +import { syncUserSleep } from "../sync-sleep"; + +const NIGHT: WhoopSleep = { + id: "sleep-uuid", + user_id: 42, + created_at: "2026-06-01T05:00:00.000Z", + updated_at: "2026-06-01T07:00:00.000Z", + start: "2026-05-31T23:00:00.000Z", + end: "2026-06-01T07:00:00.000Z", + nap: false, + score_state: "SCORED", + score: { + stage_summary: { + total_in_bed_time_milli: 28_800_000, + total_awake_time_milli: 1_800_000, + total_light_sleep_time_milli: 14_400_000, + total_slow_wave_sleep_time_milli: 5_400_000, + total_rem_sleep_time_milli: 7_200_000, + }, + sleep_needed: { + baseline_milli: 27_000_000, + need_from_sleep_debt_milli: 0, + need_from_recent_strain_milli: 0, + need_from_recent_nap_milli: 0, + }, + respiratory_rate: 15.2, + sleep_performance_percentage: 88, + sleep_efficiency_percentage: 93.5, + sleep_consistency_percentage: 71, + }, +}; + +beforeEach(() => { + updateManyMock.mockClear().mockResolvedValue({ count: 0 }); + upsertMeasurementsMock.mockClear().mockResolvedValue(1); + fetchSleepsMock.mockReset(); +}); + +describe("syncUserSleep — record-scoped stale-segment sweep", () => { + it("sweeps live WHOOP SLEEP_DURATION rows under the record prefix, keeping the fresh set", async () => { + fetchSleepsMock.mockResolvedValue([NIGHT]); + + await syncUserSleep("user-1"); + + expect(updateManyMock).toHaveBeenCalledTimes(1); + const arg = (updateManyMock.mock.calls[0]! as unknown[])[0] as { + where: { + userId: string; + source: string; + type: string; + deletedAt: null; + externalId: { startsWith: string; notIn: string[] }; + }; + data: Record; + }; + expect(arg.where.userId).toBe("user-1"); + expect(arg.where.source).toBe("WHOOP"); + expect(arg.where.type).toBe("SLEEP_DURATION"); + expect(arg.where.deletedAt).toBeNull(); + expect(arg.where.externalId.startsWith).toBe("sleep-uuid:"); + // The fresh set: the four stage-tagged segments + the IN_BED envelope. + expect(arg.where.externalId.notIn.sort()).toEqual([ + "sleep-uuid:seg:sleep_awake", + "sleep-uuid:seg:sleep_core", + "sleep-uuid:seg:sleep_deep", + "sleep-uuid:seg:sleep_rem", + "sleep-uuid:sleep_in_bed", + ]); + // Soft delete only. + expect(arg.data).toEqual({ deletedAt: expect.any(Date) }); + + // A legacy indexed row for this record falls inside the sweep. + const legacyId = "sleep-uuid:seg:sleep_core:1"; + expect(legacyId.startsWith(arg.where.externalId.startsWith)).toBe(true); + expect(arg.where.externalId.notIn).not.toContain(legacyId); + }); + + it("sweeps BEFORE the fresh set upserts (replace-then-write, google parity)", async () => { + fetchSleepsMock.mockResolvedValue([NIGHT]); + await syncUserSleep("user-1"); + const sweepOrder = updateManyMock.mock.invocationCallOrder[0]!; + const upsertOrder = upsertMeasurementsMock.mock.invocationCallOrder[0]!; + expect(sweepOrder).toBeLessThan(upsertOrder); + }); + + it("skips the sweep for an unscored record (no fresh ids — deleting would wipe the night)", async () => { + fetchSleepsMock.mockResolvedValue([ + { ...NIGHT, score_state: "PENDING_SCORE", score: undefined }, + ]); + await syncUserSleep("user-1"); + expect(updateManyMock).not.toHaveBeenCalled(); + }); + + it("never fails the sync when the sweep query throws", async () => { + fetchSleepsMock.mockResolvedValue([NIGHT]); + updateManyMock.mockRejectedValueOnce(new Error("db down")); + await expect(syncUserSleep("user-1")).resolves.toBe(1); + expect(upsertMeasurementsMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/whoop/client.ts b/src/lib/whoop/client.ts index e1722e92c..e3d7704b8 100644 --- a/src/lib/whoop/client.ts +++ b/src/lib/whoop/client.ts @@ -646,10 +646,12 @@ export function mapSleep(s: WhoopSleep): MappedMeasurement[] { measuredAt, fieldTag: "sleep_in_bed", }, - // The sync layer keys reconstructed rows by this externalId verbatim; the - // index keeps the several rows of one night distinct under - // userId_type_source_externalId. - externalIdFor: (tag, index) => `${s.id}:seg:${tag}:${index}`, + // The sync layer keys reconstructed rows by this externalId verbatim; + // the stage tag keeps the several rows of one night distinct under + // userId_type_source_externalId (one segment per stage, no positional + // index — an index renumbered on a re-score when a stage's duration + // flipped 0↔positive, minting fresh ids that double-counted the night). + externalIdFor: (tag) => `${s.id}:seg:${tag}`, }), ); diff --git a/src/lib/whoop/sync-sleep.ts b/src/lib/whoop/sync-sleep.ts index a97173178..84828749b 100644 --- a/src/lib/whoop/sync-sleep.ts +++ b/src/lib/whoop/sync-sleep.ts @@ -9,10 +9,23 @@ * timestamps), so `mapSleep` RECONSTRUCTS an ordered, contiguous per-segment * timeline (CORE/DEEP/REM/AWAKE laid back-to-back from sleep onset) and flags * those rows `reconstructed`; each reconstructed segment carries its own - * indexed externalId `:seg::`. IN_BED stays a single - * envelope row keyed `:sleep_in_bed`. The non-segment scores keep - * the `:` shape. + * stage-tagged externalId `:seg:` — stable across a WHOOP + * re-score (the retired positional index renumbered whenever a stage's + * duration flipped 0↔positive). IN_BED stays a single envelope row keyed + * `:sleep_in_bed`. The non-segment scores keep the + * `:` shape. + * + * Before the upsert, `sweepStaleSleepSegments` tombstones any live + * SLEEP_DURATION row under a re-fetched record's `:` prefix that + * this fetch did not re-produce — clearing re-score orphans AND every legacy + * `:seg::` row, so the id change self-heals without a + * migration (mirrors Google Health's replace-by-window cleanup). */ +import { + sweepStaleSleepSegments, + type SleepSegmentSweep, +} from "@/lib/sleep/sweep-stale-segments"; + import { fetchSleeps, fetchSleepById, mapSleep } from "./client"; import { getValidToken, @@ -52,22 +65,34 @@ export async function syncUserSleep( } const readings: WhoopMeasurementUpsert[] = []; + const sweeps: SleepSegmentSweep[] = []; for (const s of records) { + const keepIds: string[] = []; for (const m of mapSleep(s)) { + // Reconstructed sleep segments carry their own stage-tagged externalId + // so the several rows of one night stay distinct; everything else keeps + // the `:` shape. + const externalId = m.externalId ?? `${s.id}:${m.fieldTag}`; + if (m.type === "SLEEP_DURATION") keepIds.push(externalId); readings.push({ type: m.type, value: m.value, unit: m.unit, measuredAt: m.measuredAt, - // Reconstructed sleep segments carry their own indexed externalId so - // the several rows of one stage stay distinct; everything else keeps - // the `:` shape. - externalId: m.externalId ?? `${s.id}:${m.fieldTag}`, + externalId, sleepStage: m.sleepStage ?? null, }); } + // Record-scoped sweep entry — only rows under THIS record's prefix are + // candidates, and only when the fetch re-produced the record (an unscored + // record maps to nothing and is skipped by the sweep's keep-guard). + sweeps.push({ prefix: `${s.id}:`, keepIds }); } + // Clear whatever an earlier scoring left under the re-fetched records before + // the fresh set upserts (mirrors Google Health's replace-by-window order). + await sweepStaleSleepSegments(userId, "WHOOP", sweeps); + const imported = await upsertWhoopMeasurements(userId, readings); await markResourceSynced(userId, "sleep"); return imported; @@ -95,18 +120,27 @@ export async function syncWhoopSleepById( } const readings: WhoopMeasurementUpsert[] = []; + const keepIds: string[] = []; for (const m of mapSleep(record)) { + // Reconstructed sleep segments carry their own stage-tagged externalId + // (see syncUserSleep). + const externalId = m.externalId ?? `${record.id}:${m.fieldTag}`; + if (m.type === "SLEEP_DURATION") keepIds.push(externalId); readings.push({ type: m.type, value: m.value, unit: m.unit, measuredAt: m.measuredAt, - // Reconstructed sleep segments carry their own indexed externalId - // (see syncUserSleep). - externalId: m.externalId ?? `${record.id}:${m.fieldTag}`, + externalId, sleepStage: m.sleepStage ?? null, }); } + // A webhook refresh IS a re-score in most cases — sweep the record's stale + // rows before the fresh set upserts (see syncUserSleep). + await sweepStaleSleepSegments(userId, "WHOOP", [ + { prefix: `${record.id}:`, keepIds }, + ]); + return upsertWhoopMeasurements(userId, readings); } diff --git a/src/lib/whoop/sync.ts b/src/lib/whoop/sync.ts index 5015819ea..348cd2fe3 100644 --- a/src/lib/whoop/sync.ts +++ b/src/lib/whoop/sync.ts @@ -363,6 +363,10 @@ export async function upsertWhoopMeasurements( unit: r.unit, measuredAt: r.measuredAt, sleepStage: r.sleepStage ?? null, + // No-op on a live row, a deliberate RESURRECTION on a tombstoned + // one — WHOOP is the source of truth for its own rows, so a + // re-fetched reading brings the row back (mirrors Google / Fitbit). + deletedAt: null, // Surface the server-side mutation to the iOS LWW reconciler. syncVersion: { increment: 1 }, }, diff --git a/src/lib/withings/__tests__/sync-activity.test.ts b/src/lib/withings/__tests__/sync-activity.test.ts index 6a926c349..c34b0d512 100644 --- a/src/lib/withings/__tests__/sync-activity.test.ts +++ b/src/lib/withings/__tests__/sync-activity.test.ts @@ -212,18 +212,26 @@ describe("syncUserActivity — field mapping + idempotency", () => { // overwrite (the value moved). Keyed by the same four columns the // batched read matches on. vi.mocked(prisma.measurement.findMany).mockResolvedValue([ - { id: "row-steps", type: "ACTIVITY_STEPS", measuredAt, value: 1 }, + { + id: "row-steps", + type: "ACTIVITY_STEPS", + measuredAt, + value: 1, + deletedAt: null, + }, { id: "row-distance", type: "WALKING_RUNNING_DISTANCE", measuredAt, value: 1, + deletedAt: null, }, { id: "row-calories", type: "ACTIVE_ENERGY_BURNED", measuredAt, value: 1, + deletedAt: null, }, ] as never); vi.mocked(prisma.measurement.update).mockResolvedValue({} as never); @@ -241,7 +249,13 @@ describe("syncUserActivity — field mapping + idempotency", () => { installFetchMock([{ date: "2026-05-12", steps: 8420 }]); const measuredAt = new Date("2026-05-12T12:00:00.000Z"); vi.mocked(prisma.measurement.findMany).mockResolvedValue([ - { id: "row-steps", type: "ACTIVITY_STEPS", measuredAt, value: 8420 }, + { + id: "row-steps", + type: "ACTIVITY_STEPS", + measuredAt, + value: 8420, + deletedAt: null, + }, ] as never); const imported = await syncUserActivity("user-1"); @@ -254,6 +268,34 @@ describe("syncUserActivity — field mapping + idempotency", () => { expect(prisma.$transaction).not.toHaveBeenCalled(); }); + it("resurrects a tombstoned slot even when the value is unchanged (deletedAt: null)", async () => { + installFetchMock([{ date: "2026-05-12", steps: 8420 }]); + const measuredAt = new Date("2026-05-12T12:00:00.000Z"); + // Same value, but the row is tombstoned: the write is what resurrects + // it — Withings owns the row, so a re-fetched day brings it back. + vi.mocked(prisma.measurement.findMany).mockResolvedValue([ + { + id: "row-steps", + type: "ACTIVITY_STEPS", + measuredAt, + value: 8420, + deletedAt: new Date("2026-05-13T00:00:00.000Z"), + }, + ] as never); + vi.mocked(prisma.measurement.update).mockResolvedValue({} as never); + + const imported = await syncUserActivity("user-1"); + + expect(imported).toBe(1); + expect(prisma.measurement.createMany).not.toHaveBeenCalled(); + expect(prisma.measurement.update).toHaveBeenCalledTimes(1); + const arg = vi.mocked(prisma.measurement.update).mock.calls[0][0] as { + data: { value: number; deletedAt: Date | null }; + }; + expect(arg.data.value).toBe(8420); + expect(arg.data.deletedAt).toBeNull(); + }); + it("anchors measuredAt at noon UTC so the instant lands inside the local day for every supported tz", async () => { installFetchMock([{ date: "2026-05-12", steps: 1000 }]); vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); diff --git a/src/lib/withings/__tests__/sync-sleep.test.ts b/src/lib/withings/__tests__/sync-sleep.test.ts index 91bad4170..7cf59156d 100644 --- a/src/lib/withings/__tests__/sync-sleep.test.ts +++ b/src/lib/withings/__tests__/sync-sleep.test.ts @@ -15,6 +15,7 @@ vi.mock("@/lib/db", () => ({ create: vi.fn(), update: vi.fn(), upsert: vi.fn(), + updateMany: vi.fn(), }, withingsConnection: { findUnique: vi.fn() }, }, @@ -314,11 +315,14 @@ describe("syncUserSleep — segment writes + idempotency", () => { value: 60, measuredAt: new Date(1715003600 * 1000), sleepStage: "DEEP", + // No-op on a live row, resurrection on a tombstoned one — the + // source-owned resurrect rule. + deletedAt: null, }, }); }); - it("stamps every row with an externalId tied to the segment", async () => { + it("stamps every row with an externalId keyed on session id + segment START", async () => { installFetchMock([ { startdate: 1715000000, enddate: 1715003600, state: 2, id: 42 }, ]); @@ -329,7 +333,102 @@ describe("syncUserSleep — segment writes + idempotency", () => { const arg = vi.mocked(prisma.measurement.create).mock.calls[0][0] as { data: { externalId: string }; }; - expect(arg.data.externalId).toBe("withings:sleep:user-1:42:0"); + // Session id + the segment's own startdate — both stable within a session. + // The retired running index counted across the whole rolling 30-day fetch + // window, so the window slide renumbered every segment on each sync and a + // re-aggregated night inserted a duplicate set. + expect(arg.data.externalId).toBe("withings:sleep:user-1:42:1715000000"); + }); + + it("mints IDENTICAL externalIds when the fetch window slides (re-fetch of the same night)", async () => { + // The same two segments of session 42, fetched twice: the second fetch's + // window has slid so the series now carries an EXTRA leading segment from + // a different session — under the retired running index this shifted + // every index of session 42 (0,1 → 1,2), minting fresh ids that inserted + // a duplicate night. The start-keyed ids must be identical across fetches. + const night = [ + { startdate: 1715000000, enddate: 1715003600, state: 1, id: 42 }, + { startdate: 1715003600, enddate: 1715005400, state: 2, id: 42 }, + ]; + vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null); + vi.mocked(prisma.measurement.create).mockResolvedValue({} as never); + + installFetchMock(night); + await syncUserSleep("user-1"); + const firstIds = vi + .mocked(prisma.measurement.create) + .mock.calls.map((c) => (c[0].data as { externalId: string }).externalId); + + vi.mocked(prisma.measurement.create).mockClear(); + installFetchMock([ + { startdate: 1714990000, enddate: 1714993600, state: 1, id: 7 }, + ...night, + ]); + await syncUserSleep("user-1"); + const secondIds = vi + .mocked(prisma.measurement.create) + .mock.calls.map((c) => (c[0].data as { externalId: string }).externalId) + .filter((id) => id.includes(":42:")); + + expect(secondIds).toEqual(firstIds); + }); + + it("sweeps stale rows per fetched session: live-only, session-prefixed, notIn the fresh set, soft-delete (v1.28.25)", async () => { + installFetchMock([ + { startdate: 1715000000, enddate: 1715003600, state: 1, id: 42 }, + { startdate: 1715003600, enddate: 1715005400, state: 2, id: 42 }, + ]); + vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null); + vi.mocked(prisma.measurement.create).mockResolvedValue({} as never); + vi.mocked(prisma.measurement.updateMany).mockResolvedValue({ + count: 0, + } as never); + + await syncUserSleep("user-1"); + + expect(prisma.measurement.updateMany).toHaveBeenCalledTimes(1); + const arg = vi.mocked(prisma.measurement.updateMany).mock.calls[0][0] as { + where: Record; + data: Record; + }; + expect(arg.where).toEqual({ + userId: "user-1", + source: "WITHINGS", + type: "SLEEP_DURATION", + deletedAt: null, + externalId: { + startsWith: "withings:sleep:user-1:42:", + notIn: [ + "withings:sleep:user-1:42:1715000000", + "withings:sleep:user-1:42:1715003600", + ], + }, + }); + expect(arg.data).toEqual({ deletedAt: expect.any(Date) }); + + // A legacy running-index row for this session falls inside the sweep: + // under the session prefix, never in the fresh start-keyed set. + const where = arg.where as { + externalId: { startsWith: string; notIn: string[] }; + }; + const legacyId = "withings:sleep:user-1:42:0"; + expect(legacyId.startsWith(where.externalId.startsWith)).toBe(true); + expect(where.externalId.notIn).not.toContain(legacyId); + }); + + it("never sweeps sessions absent from this fetch (no id → no sweep entry)", async () => { + // Segments without a session id cannot be bounded to one night — their + // shared `no-id` prefix would span every id-less night in history — so + // they must not produce a sweep. + installFetchMock([ + { startdate: 1715000000, enddate: 1715003600, state: 2 }, + ]); + vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null); + vi.mocked(prisma.measurement.create).mockResolvedValue({} as never); + + const imported = await syncUserSleep("user-1"); + expect(imported).toBe(1); + expect(prisma.measurement.updateMany).not.toHaveBeenCalled(); }); it("calls recordSyncSuccess after a clean round-trip", async () => { diff --git a/src/lib/withings/sync-activity.ts b/src/lib/withings/sync-activity.ts index b3c00e2f2..ca21ad6ca 100644 --- a/src/lib/withings/sync-activity.ts +++ b/src/lib/withings/sync-activity.ts @@ -372,13 +372,23 @@ export async function syncUserActivity( type: { in: distinctTypes }, measuredAt: { in: distinctMeasuredAt }, }, - select: { id: true, type: true, measuredAt: true, value: true }, + select: { + id: true, + type: true, + measuredAt: true, + value: true, + deletedAt: true, + }, }); - const existingBySlot = new Map(); + const existingBySlot = new Map< + string, + { id: string; value: number; deletedAt: Date | null } + >(); for (const row of existingRows) { existingBySlot.set(slotKey(row.type, row.measuredAt), { id: row.id, value: row.value, + deletedAt: row.deletedAt, }); } @@ -388,8 +398,11 @@ export async function syncUserActivity( const existing = existingBySlot.get(slotKey(p.type, p.measuredAt)); if (existing) { // Only write when the value actually moved — a re-sync of an - // unchanged day is then a pure no-op on the write path. - if (existing.value !== p.value) { + // unchanged LIVE day is then a pure no-op on the write path. A + // TOMBSTONED row always writes: the update is what resurrects it + // (`deletedAt: null`) — Withings is the source of truth for its + // own rows (mirrors Google / Fitbit). + if (existing.value !== p.value || existing.deletedAt !== null) { toUpdate.push({ id: existing.id, value: p.value }); } } else { @@ -425,7 +438,9 @@ export async function syncUserActivity( toUpdate.map((u) => prisma.measurement.update({ where: { id: u.id }, - data: { value: u.value }, + // `deletedAt: null` — no-op on a live row, deliberate + // RESURRECTION on a tombstoned one (see the skip above). + data: { value: u.value, deletedAt: null }, }), ), ); diff --git a/src/lib/withings/sync-sleep.ts b/src/lib/withings/sync-sleep.ts index 31940c073..395dac608 100644 --- a/src/lib/withings/sync-sleep.ts +++ b/src/lib/withings/sync-sleep.ts @@ -19,11 +19,14 @@ * reserved for HealthKit ingest (pre-iOS-16 `asleepUnspecified` and * iOS-16+ `inBed` respectively) and are NOT emitted from this path. * - * Idempotency: every segment writes through the v1.4.25 W17b/c - * composite (Migration 0055 — `(userId, type, measuredAt, source, - * sleepStage)` with NULLS NOT DISTINCT). The five-tuple guarantees a - * re-sync upserts the same row rather than inserting a duplicate, even - * when Withings re-emits the night with adjusted segment boundaries. + * Idempotency: every segment keys on the stable externalId + * `withings:sleep:::` — the session id + * plus the segment's own start instant, both stable across re-syncs of an + * unchanged night. When Withings re-aggregates a night with adjusted + * boundaries, the shifted segments mint new ids and the session-scoped sweep + * (`sweepStaleSleepSegments`) tombstones whatever the re-aggregation + * orphaned — including every legacy row from the retired running-index + * format, so that fix self-heals without a migration. * * Date semantics: each segment's `measuredAt` is the segment's * `enddate` (unix seconds → `Date`). Every reader treats `measuredAt` @@ -46,6 +49,11 @@ import { } from "@/lib/rollups/measurement-rollups"; import { invalidateStatusInsightsForTypes } from "@/lib/insights/comprehensive-generate"; +import { + sweepStaleSleepSegments, + type SleepSegmentSweep, +} from "@/lib/sleep/sweep-stale-segments"; + import { hasActivityScope } from "./client"; import { WithingsApiError, @@ -389,17 +397,20 @@ export async function syncUserSleep( } let imported = 0; - let segmentIndex = 0; // v1.4.39.1 — track every (type, measuredAt) we touched so the // persistent rollup tier can be re-folded at the end. See sync.ts for // the full rationale. const touched: Array<{ type: MeasurementType; measuredAt: Date }> = []; + // Fresh segment externalIds per session id, driving the session-scoped + // sweep below. Segments without a session id can't be bounded to one night + // and are excluded (their `no-id` prefix would span every id-less night in + // history, not just this fetch). + const freshBySession = new Map(); for (const segment of segments) { const stage = mapWithingsSleepState(segment.state); if (!stage) { // State 4 (manual / synthetic marker) is ignored — see file // header. - segmentIndex++; continue; } // `measuredAt` is the segment END (`enddate`). Every reader treats @@ -413,15 +424,29 @@ export async function syncUserSleep( // floating-point noise. const minutes = Math.round(durationSec / 60); - const externalId = `withings:sleep:${userId}:${segment.id ?? "no-id"}:${segmentIndex}`; + // Keyed by the segment's own START instant — a coordinate that is stable + // within a session — NOT a positional index. The old running index counted + // across the whole 30-day fetch window, so the sliding window renumbered + // every segment on each sync: the probe below never matched, the update + // branch was dead code, and a re-aggregated night inserted a second set of + // rows next to the first (the same double-count bug class Google Health + // fixed with its stable session anchor). Legacy indexed rows are swept by + // the session-scoped cleanup below, so the format change self-heals. + const externalId = `withings:sleep:${userId}:${segment.id ?? "no-id"}:${segment.startdate}`; + if (typeof segment.id === "number") { + const fresh = freshBySession.get(segment.id) ?? []; + fresh.push(externalId); + freshBySession.set(segment.id, fresh); + } try { - // Key the re-sync lookup on the STABLE `externalId` (segment id + - // index), not `measuredAt`. Withings re-aggregates a night between - // syncs, so a segment's `enddate` — and therefore its END-stamped - // `measuredAt` — shifts; keying on `measuredAt` would miss the prior - // row and then collide with the unique `externalId`, leaving the night - // stuck at the stale value. Update both `value` and `measuredAt`. + // Key the re-sync lookup on the STABLE `externalId` (session id + + // segment start), not `measuredAt`. Withings re-aggregates a night + // between syncs, so a segment's `enddate` — and therefore its + // END-stamped `measuredAt` — shifts; keying on `measuredAt` would miss + // the prior row and then collide with the unique `externalId`, leaving + // the night stuck at the stale value. Update both `value` and + // `measuredAt`. const existing = await prisma.measurement.findFirst({ where: { userId, @@ -434,7 +459,16 @@ export async function syncUserSleep( if (existing) { await prisma.measurement.update({ where: { id: existing.id }, - data: { value: minutes, measuredAt, sleepStage: stage }, + // `deletedAt: null` is a no-op on a live row, a deliberate + // RESURRECTION on a tombstoned one — Withings is the source of + // truth for its own rows, so a re-fetched segment brings the row + // back (mirrors Google / Fitbit). + data: { + value: minutes, + measuredAt, + sleepStage: stage, + deletedAt: null, + }, }); } else { await prisma.measurement.create({ @@ -457,9 +491,23 @@ export async function syncUserSleep( `Failed to upsert sleep segment (${stage}, ${measuredAt.toISOString()}): ${err}`, ); } - segmentIndex++; } + // Session-scoped sweep (mirrors Google Health's replace-by-window): for each + // session this fetch re-produced, soft-delete any live WITHINGS + // SLEEP_DURATION row under the session's externalId prefix that is NOT in + // the fresh set. Collapses re-aggregation orphans AND every legacy + // running-index row for re-fetched sessions. Sessions absent from this fetch + // are never touched; a sweep failure never fails the sync (the session stays + // inside the rolling 30-day window, so the hourly cron retries it). + const sweeps: SleepSegmentSweep[] = [...freshBySession].map( + ([sessionId, keepIds]) => ({ + prefix: `withings:sleep:${userId}:${sessionId}:`, + keepIds, + }), + ); + await sweepStaleSleepSegments(userId, "WITHINGS", sweeps); + // v1.18.10 P0 — nightly sleep vitals (avg HR / respiratory rate / SDNN HRV // / avg SpO2 / sleep score) from the per-night summary. A summary-fetch // failure must not lose the stage segments already written, so it runs in @@ -502,6 +550,10 @@ export async function syncUserSleep( update: { value: vital.value, measuredAt, + // No-op on a live row, a deliberate RESURRECTION on a + // tombstoned one — Withings is the source of truth for its + // own rows (mirrors Google / Fitbit). + deletedAt: null, syncVersion: { increment: 1 }, }, }); diff --git a/src/lib/withings/sync.ts b/src/lib/withings/sync.ts index c887398f6..144743154 100644 --- a/src/lib/withings/sync.ts +++ b/src/lib/withings/sync.ts @@ -237,7 +237,11 @@ export async function syncUserMeasurements( if (existing) { await prisma.measurement.update({ where: { id: existing.id }, - data: { value: m.value }, + // `deletedAt: null` is a no-op on a live row, a deliberate + // RESURRECTION on a tombstoned one — Withings is the source of + // truth for its own rows, so a re-fetched measure brings the row + // back (mirrors Google / Fitbit). + data: { value: m.value, deletedAt: null }, }); } else { await prisma.measurement.create({