diff --git a/prisma/migrations/0421_withings_sleep_stage_reads/migration.sql b/prisma/migrations/0421_withings_sleep_stage_reads/migration.sql new file mode 100644 index 000000000..58ed41ab3 --- /dev/null +++ b/prisma/migrations/0421_withings_sleep_stage_reads/migration.sql @@ -0,0 +1,17 @@ +-- Per-night stage-read bookkeeping for the Withings sleep sync. +-- +-- The sync reads each night's stage timeline on its own. A night whose read +-- failed was lost with every night read before it, and a night that never +-- carries stages (a tracker without them) was read again on every run for as +-- long as it stayed in the 30-day window. The column records, per night, how +-- many reads came back without a stage and whether the last one failed, so a +-- failed night is read first next time and an unstaged one is given up after +-- a few tries. +-- +-- Nullable, no default, no backfill: a connection without it reads every +-- night as before and starts keeping the record on its next sync. +-- +-- Reversibility: dropping the column loses only the record; the sync then +-- reads every night as before. + +ALTER TABLE "withings_connections" ADD COLUMN "sleep_stage_reads" JSONB; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 5509b6aeb..1d79d97b1 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -4251,6 +4251,15 @@ model WithingsConnection { /// Withings card name what the plan withholds. Versioned and validated on /// read (`src/lib/withings/api-plan.ts`); reset on reconnect. apiPlanState Json? @map("api_plan_state") + /// v1.43.11 — what the sleep sync learned reading each night's stage + /// timeline: `{ v: 1, nights: { ":": { empty, failed } } }`, + /// keyed by the night's unix span. `empty` counts reads that answered with + /// no stage, so a night a tracker never stages stops being read after a + /// few tries; `failed` marks a night whose read errored, so it is read + /// first on the next run. Pruned to the nights of the summary window. + /// Validated on read (`src/lib/withings/sleep-stage-reads.ts`); reset on + /// reconnect. + sleepStageReads Json? @map("sleep_stage_reads") createdAt DateTime @default(now()) @map("created_at") updatedAt DateTime @updatedAt @map("updated_at") diff --git a/src/app/api/withings/callback/route.ts b/src/app/api/withings/callback/route.ts index faed894ae..f1c02e33c 100644 --- a/src/app/api/withings/callback/route.ts +++ b/src/app/api/withings/callback/route.ts @@ -216,6 +216,9 @@ export const GET = apiHandler(async (request: NextRequest) => { // A reconnect may come with another developer app (another plan), so // what the old grant's plan refused is asked for again. apiPlanState: Prisma.DbNull, + // A reconnect may be another account or device, so nights it gave + // up on are tried again. + sleepStageReads: Prisma.DbNull, }, create: { userId: user.id, diff --git a/src/lib/withings/__tests__/sync-sleep.test.ts b/src/lib/withings/__tests__/sync-sleep.test.ts index 96847511c..89994ad07 100644 --- a/src/lib/withings/__tests__/sync-sleep.test.ts +++ b/src/lib/withings/__tests__/sync-sleep.test.ts @@ -23,7 +23,7 @@ vi.mock("@/lib/db", () => ({ upsert: vi.fn(), updateMany: vi.fn(), }, - withingsConnection: { findUnique: vi.fn() }, + withingsConnection: { findUnique: vi.fn(), update: vi.fn() }, $executeRaw: vi.fn(), $transaction: transactionMock, }, @@ -98,6 +98,7 @@ import { mapWithingsSleepSummary, syncUserSleep, } from "../sync-sleep"; +import { SLEEP_STAGE_MAX_EMPTY_READS } from "../sleep-stage-reads"; interface FakeSegment { startdate: number; @@ -1024,6 +1025,125 @@ describe("syncUserSleep — one stage read per night (Sleep v2 Get answers 24 h) expect(stageReads(fetchMock)).toHaveLength(12); }); + /** Answer the stage read of the night starting at `startdate` with an error. */ + function failNight( + fetchMock: ReturnType, + startdate: number, + ) { + const answer = fetchMock.getMockImplementation()!; + fetchMock.mockImplementation( + async (url: string, init: { body: string }) => { + const params = new URLSearchParams(init.body); + if ( + params.get("action") === "get" && + Number(params.get("startdate")) === startdate + ) { + return { + status: 200, + json: async () => ({ status: 293, error: "Invalid params" }), + } as unknown as Awaited>; + } + return answer(url, init); + }, + ); + } + + function savedReads(): Record { + const call = vi.mocked(prisma.withingsConnection.update).mock.calls.at(-1); + const data = call?.[0].data as { + sleepStageReads: { nights: Record }; + }; + return data.sleepStageReads.nights; + } + + it("keeps the nights read around a failing one and books the leg failed", async () => { + // One failing night used to throw out of the loop, so the nights read + // before it and the summary vitals were never written. + const nights = [0, 3, 9].map((n, i) => night(n, 200 + i)); + const fetchMock = installSleepApi( + nights.flatMap((x) => x.segments), + nights.map((x) => ({ ...x.summary, data: { hr_average: 55 } })), + ); + failNight(fetchMock, nights[1]!.summary.startdate); + + await expect(syncUserSleep("user-1")).rejects.toThrow(/293/); + + const written = reconcileMock.mock.calls.map( + ([, input]) => (input as { externalId: string }).externalId, + ); + // Both stage segments of the two good nights, and every night's vitals. + expect(written.filter((id) => !id.endsWith(":hr"))).toHaveLength(4); + expect(written.filter((id) => id.endsWith(":hr"))).toHaveLength(3); + expect(recordSyncSuccess).not.toHaveBeenCalled(); + expect(recordSyncFailure).toHaveBeenCalledWith( + expect.objectContaining({ leg: "sleep" }), + ); + const failedKey = `${nights[1]!.summary.startdate}:${nights[1]!.summary.enddate}`; + expect(savedReads()[failedKey]).toEqual({ empty: 0, failed: true }); + }); + + it("reads a night whose last read failed first on the next run", async () => { + const nights = Array.from({ length: 12 }, (_, i) => night(i + 3, i + 1)); + const fetchMock = installSleepApi( + nights.flatMap((x) => x.segments), + nights.map((x) => x.summary), + ); + await syncUserSleep("user-1"); + const unread = nights.find( + (x) => + !stageReads(fetchMock).some(([from]) => from === x.summary.startdate), + )!; + expect(unread).toBeDefined(); + + fetchMock.mockClear(); + const key = `${unread.summary.startdate}:${unread.summary.enddate}`; + vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({ + scope: "user.metrics,user.activity", + sleepStageReads: { v: 1, nights: { [key]: { empty: 0, failed: true } } }, + } as never); + await syncUserSleep("user-1"); + + expect(stageReads(fetchMock)[0]).toEqual([ + unread.summary.startdate, + Math.min(unread.summary.enddate, unread.summary.startdate + DAY), + ]); + // Read cleanly this time: the record forgets it. + expect(savedReads()[key]).toBeUndefined(); + }); + + it("stops reading a night that answered without a stage three times", async () => { + const given = night(5, 31); + const trying = night(6, 32); + const key = (x: typeof given) => + `${x.summary.startdate}:${x.summary.enddate}`; + vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({ + scope: "user.metrics,user.activity", + sleepStageReads: { + v: 1, + nights: { + [key(given)]: { empty: SLEEP_STAGE_MAX_EMPTY_READS }, + [key(trying)]: { empty: SLEEP_STAGE_MAX_EMPTY_READS - 1 }, + }, + }, + } as never); + // Withings lists both nights and stages neither. + const fetchMock = installSleepApi([], [given.summary, trying.summary]); + + await syncUserSleep("user-1"); + + expect(stageReads(fetchMock)).toEqual([ + [trying.summary.startdate, trying.summary.enddate], + ]); + expect(savedReads()).toEqual({ + [key(given)]: { empty: SLEEP_STAGE_MAX_EMPTY_READS }, + [key(trying)]: { empty: SLEEP_STAGE_MAX_EMPTY_READS }, + }); + // A full sync still asks for it. + fetchMock.mockClear(); + await syncUserSleep("user-1", { fullSync: true }); + expect(stageReads(fetchMock)).toHaveLength(2); + }); + it("caps a night longer than a day at the 24 hours one read can return", async () => { const startdate = nowUnix - 30 * 3600; const enddate = nowUnix - 2 * 3600; // 28 hours in bed diff --git a/src/lib/withings/__tests__/sync-typed-classification.test.ts b/src/lib/withings/__tests__/sync-typed-classification.test.ts index cc82c1f50..6fee0110f 100644 --- a/src/lib/withings/__tests__/sync-typed-classification.test.ts +++ b/src/lib/withings/__tests__/sync-typed-classification.test.ts @@ -31,7 +31,10 @@ vi.mock("@/lib/db", () => ({ create: vi.fn(), update: vi.fn(), }, - withingsConnection: { findUnique: vi.fn() }, + withingsConnection: { findUnique: vi.fn(), update: vi.fn() }, + // A failed stage read no longer throws before the writes: the nights + // read before it, and the summary vitals, are still written. + $transaction: vi.fn(async (fn: (tx: unknown) => unknown) => fn({})), }, })); diff --git a/src/lib/withings/sleep-stage-reads.ts b/src/lib/withings/sleep-stage-reads.ts new file mode 100644 index 000000000..e5ce58221 --- /dev/null +++ b/src/lib/withings/sleep-stage-reads.ts @@ -0,0 +1,92 @@ +/** + * What the Withings sleep sync learned reading each night's stage timeline, + * kept on `WithingsConnection.sleepStageReads`. + * + * Two facts per night, keyed by its unix span (`":"`): + * + * - `empty`: reads that answered without a single stage. A tracker that + * never stages a night (a nap, a watch without stages) used to be read + * again on every run for as long as the night stayed in the 30-day + * window. After {@link SLEEP_STAGE_MAX_EMPTY_READS} such answers it is + * no longer read, except by a full sync or while Withings marks it + * changed. + * - `failed`: the night's last read errored. It is read first next run. + * + * Versioned and validated on read: anything that is not the current shape + * reads as no record at all, which is the behaviour before the record + * existed. + */ + +export const SLEEP_STAGE_MAX_EMPTY_READS = 3; + +export interface SleepStageNightRecord { + empty: number; + failed: boolean; +} + +export type SleepStageReads = Map; + +export function sleepNightKey(night: { + startdate: number; + enddate: number; +}): string { + return `${night.startdate}:${night.enddate}`; +} + +export function parseSleepStageReads(raw: unknown): SleepStageReads { + const out: SleepStageReads = new Map(); + if (raw === null || typeof raw !== "object") return out; + const record = raw as { v?: unknown; nights?: unknown }; + if (record.v !== 1) return out; + if (record.nights === null || typeof record.nights !== "object") return out; + for (const [key, value] of Object.entries(record.nights)) { + if (!/^\d+:\d+$/.test(key)) continue; + if (value === null || typeof value !== "object") continue; + const { empty, failed } = value as { empty?: unknown; failed?: unknown }; + const count = + typeof empty === "number" && Number.isInteger(empty) && empty >= 0 + ? empty + : 0; + out.set(key, { empty: count, failed: failed === true }); + } + return out; +} + +export function serialiseSleepStageReads(reads: SleepStageReads): { + v: 1; + nights: Record; +} { + const nights: Record = {}; + for (const [key, night] of reads) { + if (night.empty === 0 && !night.failed) continue; + nights[key] = night.failed + ? { empty: night.empty, failed: true } + : { empty: night.empty }; + } + return { v: 1, nights }; +} + +/** The outcome of one night's read in this run. */ +export type SleepStageReadOutcome = "staged" | "empty" | "failed"; + +/** + * The record after this run: each read night's outcome applied, and every + * night that is no longer in the summary window dropped. + */ +export function nextSleepStageReads( + previous: SleepStageReads, + outcomes: ReadonlyMap, + windowNightKeys: ReadonlySet, +): SleepStageReads { + const next: SleepStageReads = new Map(); + for (const [key, night] of previous) { + if (windowNightKeys.has(key)) next.set(key, { ...night }); + } + for (const [key, outcome] of outcomes) { + const prior = next.get(key) ?? { empty: 0, failed: false }; + if (outcome === "staged") next.delete(key); + else if (outcome === "failed") next.set(key, { ...prior, failed: true }); + else next.set(key, { empty: prior.empty + 1, failed: false }); + } + return next; +} diff --git a/src/lib/withings/sync-sleep.ts b/src/lib/withings/sync-sleep.ts index d2e6a0855..61cea896a 100644 --- a/src/lib/withings/sync-sleep.ts +++ b/src/lib/withings/sync-sleep.ts @@ -60,6 +60,15 @@ import { } from "@/lib/measurements/reconcile-external-measurement"; import { hasActivityScope } from "./client"; +import { + nextSleepStageReads, + parseSleepStageReads, + serialiseSleepStageReads, + SLEEP_STAGE_MAX_EMPTY_READS, + sleepNightKey, + type SleepStageReadOutcome, + type SleepStageReads, +} from "./sleep-stage-reads"; import { isPlanGatedError, isPlanReprobeDue, @@ -409,11 +418,13 @@ export async function syncUserSleep( let summaries: WithingsSleepSummary[] = []; let summaryFailure: unknown = null; let nightsKnown = false; + let stageReads: SleepStageReads = new Map(); try { const planRow = await prisma.withingsConnection.findUnique({ where: { userId }, - select: { apiPlanState: true }, + select: { apiPlanState: true, sleepStageReads: true }, }); + stageReads = parseSleepStageReads(planRow?.sleepStageReads); const sleepPlan = parseWithingsApiPlanState( planRow?.apiPlanState, ).sleepSummary; @@ -460,6 +471,7 @@ export async function syncUserSleep( const windows: StageWindow[] = nightsKnown ? await stageWindowsToFetch(userId, summaries, now, { all: opts.fullSync === true, + reads: stageReads, }) : [ { @@ -472,6 +484,12 @@ export async function syncUserSleep( const segments: WithingsSleepSegment[] = []; const wholeNightSessions = new Set(); const seenSegments = new Set(); + // How each night's read went, for the per-night record below. + const outcomes = new Map(); + // The first read that failed. The nights read before and after it are + // still written, and the summary vitals with them; the failure is booked + // as the leg's once they are, and the night is read first next run. + let stageFailure: unknown = null; for (const window of windows) { let series: WithingsSleepSegment[]; try { @@ -489,33 +507,21 @@ export async function syncUserSleep( annotate({ meta: { withings_sleep_segments_skipped: "api_plan" } }); break; } - // v1.4.43 — typed-classification path (see sync-activity.ts - // for the full rationale). The thrown `WithingsApiError` carries - // the classification verdict; `recordWithingsSyncFailure` routes - // it through `classifyError`, which falls back to the legacy regex - // for un-prototyped errors (pg-boss JSON round-trip). - // - // BL-P3-2 defence-in-depth — symmetric to sync-activity. A 403 on - // this scope-gated endpoint always means scope-missing or - // token-revoked; force `reauth_required` so the 3-strike alert - // still pages on the unexpected case. - const withingsStatus = - err instanceof WithingsApiError ? err.withingsStatus : undefined; - if (withingsStatus === 403) { - await recordSyncFailure({ - userId, - integration: "withings", - kind: "reauth_required", - message: err instanceof Error ? err.message : String(err), - errorCode: "403", - leg: WITHINGS_LEG_SLEEP, - }); - throw err; + stageFailure ??= err; + if (window.nightKey) outcomes.set(window.nightKey, "failed"); + // A revoked grant or a missing scope refuses every night alike, so the + // rest are not asked for. + if ( + err instanceof WithingsApiError && + (err.withingsStatus === 403 || err.classification === "reauth_required") + ) { + break; } - await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_SLEEP); - throw err; + continue; } + let staged = false; for (const segment of series) { + if (mapWithingsSleepState(segment.state)) staged = true; // Two windows can return the same segment (a night longer than its // 24-hour slice, a nap inside a night); it is written once. const key = `${segment.id ?? "no-id"}:${segment.startdate}`; @@ -526,6 +532,9 @@ export async function syncUserSleep( wholeNightSessions.add(segment.id); } } + if (window.nightKey) { + outcomes.set(window.nightKey, staged ? "staged" : "empty"); + } } let imported = 0; @@ -703,6 +712,57 @@ export async function syncUserSleep( () => {}, ); + if (nightsKnown) { + const windowNightKeys = new Set(summaries.map(sleepNightKey)); + try { + await prisma.withingsConnection.update({ + where: { userId }, + data: { + sleepStageReads: serialiseSleepStageReads( + nextSleepStageReads(stageReads, outcomes, windowNightKeys), + ), + }, + }); + } catch (err) { + // Bookkeeping only: without it the next run reads as it did before. + getEvent()?.addWarning( + `withings sleep: stage-read record not saved for ${userId}: ${err}`, + ); + } + } + + if (stageFailure !== null) { + // v1.4.43 — typed-classification path (see sync-activity.ts for the full + // rationale). The thrown `WithingsApiError` carries the classification + // verdict; `recordWithingsSyncFailure` routes it through + // `classifyError`, which falls back to the legacy regex for + // un-prototyped errors (pg-boss JSON round-trip). + // + // BL-P3-2 defence-in-depth — symmetric to sync-activity. A 403 on this + // scope-gated endpoint always means scope-missing or token-revoked; force + // `reauth_required` so the 3-strike alert still pages on the unexpected + // case. + const withingsStatus = + stageFailure instanceof WithingsApiError + ? stageFailure.withingsStatus + : undefined; + if (withingsStatus === 403) { + await recordSyncFailure({ + userId, + integration: "withings", + kind: "reauth_required", + message: + stageFailure instanceof Error + ? stageFailure.message + : String(stageFailure), + errorCode: "403", + leg: WITHINGS_LEG_SLEEP, + }); + } else { + await recordWithingsSyncFailure(userId, stageFailure, WITHINGS_LEG_SLEEP); + } + throw stageFailure; + } if (summaryFailure !== null) { await recordWithingsSyncFailure(userId, summaryFailure, WITHINGS_LEG_SLEEP); return imported; @@ -729,6 +789,8 @@ interface StageWindow { enddate: number; /** The read covers the whole night, so its session may be swept. */ wholeNight: boolean; + /** The summary night it reads (`sleepNightKey`), absent for the blind slice. */ + nightKey?: string; } /** @@ -744,7 +806,7 @@ async function stageWindowsToFetch( userId: string, summaries: readonly WithingsSleepSummary[], now: Date, - opts: { all: boolean }, + opts: { all: boolean; reads: SleepStageReads }, ): Promise { const nights = new Map< string, @@ -769,6 +831,7 @@ async function stageWindowsToFetch( startdate: n.startdate, enddate: Math.min(n.enddate, n.startdate + DAY_SECONDS), wholeNight: n.enddate - n.startdate <= DAY_SECONDS, + nightKey: sleepNightKey(n), }); if (opts.all || all.length === 0) return all.map(toWindow); @@ -794,12 +857,29 @@ async function stageWindowsToFetch( select: { measuredAt: true }, }); const endsAt = stored.map((row) => row.measuredAt.getTime() / 1000); + // A night whose last read failed goes first. One that answered without a + // stage `SLEEP_STAGE_MAX_EMPTY_READS` times is not asked again: the tracker + // does not stage it, and reading it every run for a month cost a request + // an hour for nothing. + const record = (n: { startdate: number; enddate: number }) => + opts.reads.get(sleepNightKey(n)); const missing = older .filter((n) => !endsAt.some((t) => t > n.startdate && t <= n.enddate)) + .filter((n) => (record(n)?.empty ?? 0) < SLEEP_STAGE_MAX_EMPTY_READS) .sort((x, y) => x.enddate - y.enddate); + const failed = missing.filter((n) => record(n)?.failed === true); + const rest = missing.filter((n) => record(n)?.failed !== true); const budget = Math.max(0, SLEEP_STAGE_NIGHTS_PER_RUN - recent.length); - if (missing.length <= budget) return [...recent, ...missing].map(toWindow); - const offset = Math.floor(nowUnix / 3600) % missing.length; - const rotated = [...missing.slice(offset), ...missing.slice(0, offset)]; - return [...recent, ...rotated.slice(0, budget)].map(toWindow); + if (missing.length <= budget) { + return [...recent, ...failed, ...rest].map(toWindow); + } + const restBudget = Math.max(0, budget - failed.length); + const offset = + rest.length === 0 ? 0 : Math.floor(nowUnix / 3600) % rest.length; + const rotated = [...rest.slice(offset), ...rest.slice(0, offset)]; + return [ + ...recent, + ...failed.slice(0, budget), + ...rotated.slice(0, restBudget), + ].map(toWindow); }