fix(withings): keep the sleep nights read around a failing one

One failing per-night stage read threw out of the loop, so the nights
already read and the summary vitals were never written. A failed read
now leaves the others to be written, books the leg failed afterwards
and marks the night to be read first next run. A night that answers
without a stage three times is no longer read on every run; a full sync
still reads it. The per-night record lives on the connection (migration
0421) and is reset on reconnect.

prisma/migrations/0421_withings_sleep_stage_reads/migration.sql

@@ -0,0 +1,17 @@
+-- Per-night stage-read bookkeeping for the Withings sleep sync.
+--
+-- The sync reads each night's stage timeline on its own. A night whose read
+-- failed was lost with every night read before it, and a night that never
+-- carries stages (a tracker without them) was read again on every run for as
+-- long as it stayed in the 30-day window. The column records, per night, how
+-- many reads came back without a stage and whether the last one failed, so a
+-- failed night is read first next time and an unstaged one is given up after
+-- a few tries.
+--
+-- Nullable, no default, no backfill: a connection without it reads every
+-- night as before and starts keeping the record on its next sync.
+--
+-- Reversibility: dropping the column loses only the record; the sync then
+-- reads every night as before.
+
+ALTER TABLE "withings_connections" ADD COLUMN "sleep_stage_reads" JSONB;

prisma/schema.prisma

@@ -4251,6 +4251,15 @@ model WithingsConnection {
   /// Withings card name what the plan withholds. Versioned and validated on
   /// read (`src/lib/withings/api-plan.ts`); reset on reconnect.
   apiPlanState               Json?     @map("api_plan_state")
+  /// v1.43.11 — what the sleep sync learned reading each night's stage
+  /// timeline: `{ v: 1, nights: { "<start>:<end>": { empty, failed } } }`,
+  /// keyed by the night's unix span. `empty` counts reads that answered with
+  /// no stage, so a night a tracker never stages stops being read after a
+  /// few tries; `failed` marks a night whose read errored, so it is read
+  /// first on the next run. Pruned to the nights of the summary window.
+  /// Validated on read (`src/lib/withings/sleep-stage-reads.ts`); reset on
+  /// reconnect.
+  sleepStageReads            Json?     @map("sleep_stage_reads")
   createdAt                  DateTime  @default(now()) @map("created_at")
   updatedAt                  DateTime  @updatedAt @map("updated_at")
 

src/app/api/withings/callback/route.ts

@@ -216,6 +216,9 @@ export const GET = apiHandler(async (request: NextRequest) => {
           // A reconnect may come with another developer app (another plan), so
           // what the old grant's plan refused is asked for again.
           apiPlanState: Prisma.DbNull,
+          // A reconnect may be another account or device, so nights it gave
+          // up on are tried again.
+          sleepStageReads: Prisma.DbNull,
         },
         create: {
           userId: user.id,

src/lib/withings/__tests__/sync-sleep.test.ts

@@ -23,7 +23,7 @@ vi.mock("@/lib/db", () => ({
       upsert: vi.fn(),
       updateMany: vi.fn(),
     },
-    withingsConnection: { findUnique: vi.fn() },
+    withingsConnection: { findUnique: vi.fn(), update: vi.fn() },
     $executeRaw: vi.fn(),
     $transaction: transactionMock,
   },
@@ -98,6 +98,7 @@ import {
   mapWithingsSleepSummary,
   syncUserSleep,
 } from "../sync-sleep";
+import { SLEEP_STAGE_MAX_EMPTY_READS } from "../sleep-stage-reads";
 
 interface FakeSegment {
   startdate: number;
@@ -1024,6 +1025,125 @@ describe("syncUserSleep — one stage read per night (Sleep v2 Get answers 24 h)
     expect(stageReads(fetchMock)).toHaveLength(12);
   });
 
+  /** Answer the stage read of the night starting at `startdate` with an error. */
+  function failNight(
+    fetchMock: ReturnType<typeof installSleepApi>,
+    startdate: number,
+  ) {
+    const answer = fetchMock.getMockImplementation()!;
+    fetchMock.mockImplementation(
+      async (url: string, init: { body: string }) => {
+        const params = new URLSearchParams(init.body);
+        if (
+          params.get("action") === "get" &&
+          Number(params.get("startdate")) === startdate
+        ) {
+          return {
+            status: 200,
+            json: async () => ({ status: 293, error: "Invalid params" }),
+          } as unknown as Awaited<ReturnType<typeof answer>>;
+        }
+        return answer(url, init);
+      },
+    );
+  }
+
+  function savedReads(): Record<string, { empty: number; failed?: true }> {
+    const call = vi.mocked(prisma.withingsConnection.update).mock.calls.at(-1);
+    const data = call?.[0].data as {
+      sleepStageReads: { nights: Record<string, { empty: number }> };
+    };
+    return data.sleepStageReads.nights;
+  }
+
+  it("keeps the nights read around a failing one and books the leg failed", async () => {
+    // One failing night used to throw out of the loop, so the nights read
+    // before it and the summary vitals were never written.
+    const nights = [0, 3, 9].map((n, i) => night(n, 200 + i));
+    const fetchMock = installSleepApi(
+      nights.flatMap((x) => x.segments),
+      nights.map((x) => ({ ...x.summary, data: { hr_average: 55 } })),
+    );
+    failNight(fetchMock, nights[1]!.summary.startdate);
+
+    await expect(syncUserSleep("user-1")).rejects.toThrow(/293/);
+
+    const written = reconcileMock.mock.calls.map(
+      ([, input]) => (input as { externalId: string }).externalId,
+    );
+    // Both stage segments of the two good nights, and every night's vitals.
+    expect(written.filter((id) => !id.endsWith(":hr"))).toHaveLength(4);
+    expect(written.filter((id) => id.endsWith(":hr"))).toHaveLength(3);
+    expect(recordSyncSuccess).not.toHaveBeenCalled();
+    expect(recordSyncFailure).toHaveBeenCalledWith(
+      expect.objectContaining({ leg: "sleep" }),
+    );
+    const failedKey = `${nights[1]!.summary.startdate}:${nights[1]!.summary.enddate}`;
+    expect(savedReads()[failedKey]).toEqual({ empty: 0, failed: true });
+  });
+
+  it("reads a night whose last read failed first on the next run", async () => {
+    const nights = Array.from({ length: 12 }, (_, i) => night(i + 3, i + 1));
+    const fetchMock = installSleepApi(
+      nights.flatMap((x) => x.segments),
+      nights.map((x) => x.summary),
+    );
+    await syncUserSleep("user-1");
+    const unread = nights.find(
+      (x) =>
+        !stageReads(fetchMock).some(([from]) => from === x.summary.startdate),
+    )!;
+    expect(unread).toBeDefined();
+
+    fetchMock.mockClear();
+    const key = `${unread.summary.startdate}:${unread.summary.enddate}`;
+    vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({
+      scope: "user.metrics,user.activity",
+      sleepStageReads: { v: 1, nights: { [key]: { empty: 0, failed: true } } },
+    } as never);
+    await syncUserSleep("user-1");
+
+    expect(stageReads(fetchMock)[0]).toEqual([
+      unread.summary.startdate,
+      Math.min(unread.summary.enddate, unread.summary.startdate + DAY),
+    ]);
+    // Read cleanly this time: the record forgets it.
+    expect(savedReads()[key]).toBeUndefined();
+  });
+
+  it("stops reading a night that answered without a stage three times", async () => {
+    const given = night(5, 31);
+    const trying = night(6, 32);
+    const key = (x: typeof given) =>
+      `${x.summary.startdate}:${x.summary.enddate}`;
+    vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({
+      scope: "user.metrics,user.activity",
+      sleepStageReads: {
+        v: 1,
+        nights: {
+          [key(given)]: { empty: SLEEP_STAGE_MAX_EMPTY_READS },
+          [key(trying)]: { empty: SLEEP_STAGE_MAX_EMPTY_READS - 1 },
+        },
+      },
+    } as never);
+    // Withings lists both nights and stages neither.
+    const fetchMock = installSleepApi([], [given.summary, trying.summary]);
+
+    await syncUserSleep("user-1");
+
+    expect(stageReads(fetchMock)).toEqual([
+      [trying.summary.startdate, trying.summary.enddate],
+    ]);
+    expect(savedReads()).toEqual({
+      [key(given)]: { empty: SLEEP_STAGE_MAX_EMPTY_READS },
+      [key(trying)]: { empty: SLEEP_STAGE_MAX_EMPTY_READS },
+    });
+    // A full sync still asks for it.
+    fetchMock.mockClear();
+    await syncUserSleep("user-1", { fullSync: true });
+    expect(stageReads(fetchMock)).toHaveLength(2);
+  });
+
   it("caps a night longer than a day at the 24 hours one read can return", async () => {
     const startdate = nowUnix - 30 * 3600;
     const enddate = nowUnix - 2 * 3600; // 28 hours in bed

src/lib/withings/__tests__/sync-typed-classification.test.ts

@@ -31,7 +31,10 @@ vi.mock("@/lib/db", () => ({
       create: vi.fn(),
       update: vi.fn(),
     },
-    withingsConnection: { findUnique: vi.fn() },
+    withingsConnection: { findUnique: vi.fn(), update: vi.fn() },
+    // A failed stage read no longer throws before the writes: the nights
+    // read before it, and the summary vitals, are still written.
+    $transaction: vi.fn(async (fn: (tx: unknown) => unknown) => fn({})),
   },
 }));
 

src/lib/withings/sleep-stage-reads.ts

@@ -0,0 +1,92 @@
+/**
+ * What the Withings sleep sync learned reading each night's stage timeline,
+ * kept on `WithingsConnection.sleepStageReads`.
+ *
+ * Two facts per night, keyed by its unix span (`"<start>:<end>"`):
+ *
+ *   - `empty`: reads that answered without a single stage. A tracker that
+ *     never stages a night (a nap, a watch without stages) used to be read
+ *     again on every run for as long as the night stayed in the 30-day
+ *     window. After {@link SLEEP_STAGE_MAX_EMPTY_READS} such answers it is
+ *     no longer read, except by a full sync or while Withings marks it
+ *     changed.
+ *   - `failed`: the night's last read errored. It is read first next run.
+ *
+ * Versioned and validated on read: anything that is not the current shape
+ * reads as no record at all, which is the behaviour before the record
+ * existed.
+ */
+
+export const SLEEP_STAGE_MAX_EMPTY_READS = 3;
+
+export interface SleepStageNightRecord {
+  empty: number;
+  failed: boolean;
+}
+
+export type SleepStageReads = Map<string, SleepStageNightRecord>;
+
+export function sleepNightKey(night: {
+  startdate: number;
+  enddate: number;
+}): string {
+  return `${night.startdate}:${night.enddate}`;
+}
+
+export function parseSleepStageReads(raw: unknown): SleepStageReads {
+  const out: SleepStageReads = new Map();
+  if (raw === null || typeof raw !== "object") return out;
+  const record = raw as { v?: unknown; nights?: unknown };
+  if (record.v !== 1) return out;
+  if (record.nights === null || typeof record.nights !== "object") return out;
+  for (const [key, value] of Object.entries(record.nights)) {
+    if (!/^\d+:\d+$/.test(key)) continue;
+    if (value === null || typeof value !== "object") continue;
+    const { empty, failed } = value as { empty?: unknown; failed?: unknown };
+    const count =
+      typeof empty === "number" && Number.isInteger(empty) && empty >= 0
+        ? empty
+        : 0;
+    out.set(key, { empty: count, failed: failed === true });
+  }
+  return out;
+}
+
+export function serialiseSleepStageReads(reads: SleepStageReads): {
+  v: 1;
+  nights: Record<string, { empty: number; failed?: true }>;
+} {
+  const nights: Record<string, { empty: number; failed?: true }> = {};
+  for (const [key, night] of reads) {
+    if (night.empty === 0 && !night.failed) continue;
+    nights[key] = night.failed
+      ? { empty: night.empty, failed: true }
+      : { empty: night.empty };
+  }
+  return { v: 1, nights };
+}
+
+/** The outcome of one night's read in this run. */
+export type SleepStageReadOutcome = "staged" | "empty" | "failed";
+
+/**
+ * The record after this run: each read night's outcome applied, and every
+ * night that is no longer in the summary window dropped.
+ */
+export function nextSleepStageReads(
+  previous: SleepStageReads,
+  outcomes: ReadonlyMap<string, SleepStageReadOutcome>,
+  windowNightKeys: ReadonlySet<string>,
+): SleepStageReads {
+  const next: SleepStageReads = new Map();
+  for (const [key, night] of previous) {
+    if (windowNightKeys.has(key)) next.set(key, { ...night });
+  }
+  for (const [key, outcome] of outcomes) {
+    const prior = next.get(key) ?? { empty: 0, failed: false };
+    if (outcome === "staged") next.delete(key);
+    else if (outcome === "failed") next.set(key, { ...prior, failed: true });
+    else next.set(key, { empty: prior.empty + 1, failed: false });
+  }
+  return next;
+}

src/lib/withings/sync-sleep.ts

@@ -60,6 +60,15 @@ import {
 } from "@/lib/measurements/reconcile-external-measurement";
 
 import { hasActivityScope } from "./client";
+import {
+  nextSleepStageReads,
+  parseSleepStageReads,
+  serialiseSleepStageReads,
+  SLEEP_STAGE_MAX_EMPTY_READS,
+  sleepNightKey,
+  type SleepStageReadOutcome,
+  type SleepStageReads,
+} from "./sleep-stage-reads";
 import {
   isPlanGatedError,
   isPlanReprobeDue,
@@ -409,11 +418,13 @@ export async function syncUserSleep(
   let summaries: WithingsSleepSummary[] = [];
   let summaryFailure: unknown = null;
   let nightsKnown = false;
+  let stageReads: SleepStageReads = new Map();
   try {
     const planRow = await prisma.withingsConnection.findUnique({
       where: { userId },
-      select: { apiPlanState: true },
+      select: { apiPlanState: true, sleepStageReads: true },
     });
+    stageReads = parseSleepStageReads(planRow?.sleepStageReads);
     const sleepPlan = parseWithingsApiPlanState(
       planRow?.apiPlanState,
     ).sleepSummary;
@@ -460,6 +471,7 @@ export async function syncUserSleep(
   const windows: StageWindow[] = nightsKnown
     ? await stageWindowsToFetch(userId, summaries, now, {
         all: opts.fullSync === true,
+        reads: stageReads,
       })
     : [
         {
@@ -472,6 +484,12 @@ export async function syncUserSleep(
   const segments: WithingsSleepSegment[] = [];
   const wholeNightSessions = new Set<number>();
   const seenSegments = new Set<string>();
+  // How each night's read went, for the per-night record below.
+  const outcomes = new Map<string, SleepStageReadOutcome>();
+  // The first read that failed. The nights read before and after it are
+  // still written, and the summary vitals with them; the failure is booked
+  // as the leg's once they are, and the night is read first next run.
+  let stageFailure: unknown = null;
   for (const window of windows) {
     let series: WithingsSleepSegment[];
     try {
@@ -489,33 +507,21 @@ export async function syncUserSleep(
         annotate({ meta: { withings_sleep_segments_skipped: "api_plan" } });
         break;
       }
-      // v1.4.43 — typed-classification path (see sync-activity.ts
-      // for the full rationale). The thrown `WithingsApiError` carries
-      // the classification verdict; `recordWithingsSyncFailure` routes
-      // it through `classifyError`, which falls back to the legacy regex
-      // for un-prototyped errors (pg-boss JSON round-trip).
-      //
-      // BL-P3-2 defence-in-depth — symmetric to sync-activity. A 403 on
-      // this scope-gated endpoint always means scope-missing or
-      // token-revoked; force `reauth_required` so the 3-strike alert
-      // still pages on the unexpected case.
-      const withingsStatus =
-        err instanceof WithingsApiError ? err.withingsStatus : undefined;
-      if (withingsStatus === 403) {
-        await recordSyncFailure({
-          userId,
-          integration: "withings",
-          kind: "reauth_required",
-          message: err instanceof Error ? err.message : String(err),
-          errorCode: "403",
-          leg: WITHINGS_LEG_SLEEP,
-        });
-        throw err;
+      stageFailure ??= err;
+      if (window.nightKey) outcomes.set(window.nightKey, "failed");
+      // A revoked grant or a missing scope refuses every night alike, so the
+      // rest are not asked for.
+      if (
+        err instanceof WithingsApiError &&
+        (err.withingsStatus === 403 || err.classification === "reauth_required")
+      ) {
+        break;
       }
-      await recordWithingsSyncFailure(userId, err, WITHINGS_LEG_SLEEP);
-      throw err;
+      continue;
     }
+    let staged = false;
     for (const segment of series) {
+      if (mapWithingsSleepState(segment.state)) staged = true;
       // Two windows can return the same segment (a night longer than its
       // 24-hour slice, a nap inside a night); it is written once.
       const key = `${segment.id ?? "no-id"}:${segment.startdate}`;
@@ -526,6 +532,9 @@ export async function syncUserSleep(
         wholeNightSessions.add(segment.id);
       }
     }
+    if (window.nightKey) {
+      outcomes.set(window.nightKey, staged ? "staged" : "empty");
+    }
   }
 
   let imported = 0;
@@ -703,6 +712,57 @@ export async function syncUserSleep(
     () => {},
   );
 
+  if (nightsKnown) {
+    const windowNightKeys = new Set(summaries.map(sleepNightKey));
+    try {
+      await prisma.withingsConnection.update({
+        where: { userId },
+        data: {
+          sleepStageReads: serialiseSleepStageReads(
+            nextSleepStageReads(stageReads, outcomes, windowNightKeys),
+          ),
+        },
+      });
+    } catch (err) {
+      // Bookkeeping only: without it the next run reads as it did before.
+      getEvent()?.addWarning(
+        `withings sleep: stage-read record not saved for ${userId}: ${err}`,
+      );
+    }
+  }
+
+  if (stageFailure !== null) {
+    // v1.4.43 — typed-classification path (see sync-activity.ts for the full
+    // rationale). The thrown `WithingsApiError` carries the classification
+    // verdict; `recordWithingsSyncFailure` routes it through
+    // `classifyError`, which falls back to the legacy regex for
+    // un-prototyped errors (pg-boss JSON round-trip).
+    //
+    // BL-P3-2 defence-in-depth — symmetric to sync-activity. A 403 on this
+    // scope-gated endpoint always means scope-missing or token-revoked; force
+    // `reauth_required` so the 3-strike alert still pages on the unexpected
+    // case.
+    const withingsStatus =
+      stageFailure instanceof WithingsApiError
+        ? stageFailure.withingsStatus
+        : undefined;
+    if (withingsStatus === 403) {
+      await recordSyncFailure({
+        userId,
+        integration: "withings",
+        kind: "reauth_required",
+        message:
+          stageFailure instanceof Error
+            ? stageFailure.message
+            : String(stageFailure),
+        errorCode: "403",
+        leg: WITHINGS_LEG_SLEEP,
+      });
+    } else {
+      await recordWithingsSyncFailure(userId, stageFailure, WITHINGS_LEG_SLEEP);
+    }
+    throw stageFailure;
+  }
   if (summaryFailure !== null) {
     await recordWithingsSyncFailure(userId, summaryFailure, WITHINGS_LEG_SLEEP);
     return imported;
@@ -729,6 +789,8 @@ interface StageWindow {
   enddate: number;
   /** The read covers the whole night, so its session may be swept. */
   wholeNight: boolean;
+  /** The summary night it reads (`sleepNightKey`), absent for the blind slice. */
+  nightKey?: string;
 }
 
 /**
@@ -744,7 +806,7 @@ async function stageWindowsToFetch(
   userId: string,
   summaries: readonly WithingsSleepSummary[],
   now: Date,
-  opts: { all: boolean },
+  opts: { all: boolean; reads: SleepStageReads },
 ): Promise<StageWindow[]> {
   const nights = new Map<
     string,
@@ -769,6 +831,7 @@ async function stageWindowsToFetch(
     startdate: n.startdate,
     enddate: Math.min(n.enddate, n.startdate + DAY_SECONDS),
     wholeNight: n.enddate - n.startdate <= DAY_SECONDS,
+    nightKey: sleepNightKey(n),
   });
   if (opts.all || all.length === 0) return all.map(toWindow);
 
@@ -794,12 +857,29 @@ async function stageWindowsToFetch(
     select: { measuredAt: true },
   });
   const endsAt = stored.map((row) => row.measuredAt.getTime() / 1000);
+  // A night whose last read failed goes first. One that answered without a
+  // stage `SLEEP_STAGE_MAX_EMPTY_READS` times is not asked again: the tracker
+  // does not stage it, and reading it every run for a month cost a request
+  // an hour for nothing.
+  const record = (n: { startdate: number; enddate: number }) =>
+    opts.reads.get(sleepNightKey(n));
   const missing = older
     .filter((n) => !endsAt.some((t) => t > n.startdate && t <= n.enddate))
+    .filter((n) => (record(n)?.empty ?? 0) < SLEEP_STAGE_MAX_EMPTY_READS)
     .sort((x, y) => x.enddate - y.enddate);
+  const failed = missing.filter((n) => record(n)?.failed === true);
+  const rest = missing.filter((n) => record(n)?.failed !== true);
   const budget = Math.max(0, SLEEP_STAGE_NIGHTS_PER_RUN - recent.length);
-  if (missing.length <= budget) return [...recent, ...missing].map(toWindow);
-  const offset = Math.floor(nowUnix / 3600) % missing.length;
-  const rotated = [...missing.slice(offset), ...missing.slice(0, offset)];
-  return [...recent, ...rotated.slice(0, budget)].map(toWindow);
+  if (missing.length <= budget) {
+    return [...recent, ...failed, ...rest].map(toWindow);
+  }
+  const restBudget = Math.max(0, budget - failed.length);
+  const offset =
+    rest.length === 0 ? 0 : Math.floor(nowUnix / 3600) % rest.length;
+  const rotated = [...rest.slice(offset), ...rest.slice(0, offset)];
+  return [
+    ...recent,
+    ...failed.slice(0, budget),
+    ...rotated.slice(0, restBudget),
+  ].map(toWindow);
 }