fix(withings): give up a sleep night that keeps failing, and fail the leg only on recent nights

A night Withings answered with an error on every read kept the
failed-first slot and failed the sleep leg on every run, holding the
connection in an error state and paging for as long as the night stayed
in the window. Failed reads now count per night and a night is given up
after three, like an unstaged one (a full sync still reads it). A
failed read of a night older than 48 hours, while the rest synced, is a
warning instead of a leg failure.

src/lib/withings/__tests__/sync-sleep.test.ts

@@ -98,7 +98,10 @@ import {
   mapWithingsSleepSummary,
   syncUserSleep,
 } from "../sync-sleep";
-import { SLEEP_STAGE_MAX_EMPTY_READS } from "../sleep-stage-reads";
+import {
+  SLEEP_STAGE_MAX_EMPTY_READS,
+  SLEEP_STAGE_MAX_FAILED_READS,
+} from "../sleep-stage-reads";
 
 interface FakeSegment {
   startdate: number;
@@ -1056,7 +1059,7 @@ describe("syncUserSleep — one stage read per night (Sleep v2 Get answers 24 h)
     return data.sleepStageReads.nights;
   }
 
-  it("keeps the nights read around a failing one and books the leg failed", async () => {
+  it("keeps the nights read around a failing recent one and books the leg failed", async () => {
     // One failing night used to throw out of the loop, so the nights read
     // before it and the summary vitals were never written.
     const nights = [0, 3, 9].map((n, i) => night(n, 200 + i));
@@ -1064,7 +1067,7 @@ describe("syncUserSleep — one stage read per night (Sleep v2 Get answers 24 h)
       nights.flatMap((x) => x.segments),
       nights.map((x) => ({ ...x.summary, data: { hr_average: 55 } })),
     );
-    failNight(fetchMock, nights[1]!.summary.startdate);
+    failNight(fetchMock, nights[0]!.summary.startdate);
 
     await expect(syncUserSleep("user-1")).rejects.toThrow(/293/);
 
@@ -1078,8 +1081,61 @@ describe("syncUserSleep — one stage read per night (Sleep v2 Get answers 24 h)
     expect(recordSyncFailure).toHaveBeenCalledWith(
       expect.objectContaining({ leg: "sleep" }),
     );
-    const failedKey = `${nights[1]!.summary.startdate}:${nights[1]!.summary.enddate}`;
-    expect(savedReads()[failedKey]).toEqual({ empty: 0, failed: true });
+    const failedKey = `${nights[0]!.summary.startdate}:${nights[0]!.summary.enddate}`;
+    expect(savedReads()[failedKey]).toEqual({
+      empty: 0,
+      failed: true,
+      failures: 1,
+    });
+  });
+
+  it("does not fail the leg when only an older night fails", async () => {
+    // A night days back that Withings answers with an error is that night's
+    // problem: the rest synced, and failing the leg every run would park the
+    // connection in an error state for as long as the night stays listed.
+    const nights = [0, 3, 9].map((n, i) => night(n, 300 + i));
+    const fetchMock = installSleepApi(
+      nights.flatMap((x) => x.segments),
+      nights.map((x) => x.summary),
+    );
+    failNight(fetchMock, nights[1]!.summary.startdate);
+
+    await expect(syncUserSleep("user-1")).resolves.toBe(4);
+
+    expect(recordSyncFailure).not.toHaveBeenCalled();
+    expect(recordSyncSuccess).toHaveBeenCalledWith("user-1", "withings", {
+      leg: "sleep",
+    });
+    const key = `${nights[1]!.summary.startdate}:${nights[1]!.summary.enddate}`;
+    expect(savedReads()[key]).toEqual({ empty: 0, failed: true, failures: 1 });
+  });
+
+  it("gives up a night that has failed SLEEP_STAGE_MAX_FAILED_READS times", async () => {
+    const failing = night(5, 41);
+    const key = `${failing.summary.startdate}:${failing.summary.enddate}`;
+    vi.mocked(prisma.withingsConnection.findUnique).mockResolvedValue({
+      scope: "user.metrics,user.activity",
+      sleepStageReads: {
+        v: 1,
+        nights: {
+          [key]: {
+            empty: 0,
+            failed: true,
+            failures: SLEEP_STAGE_MAX_FAILED_READS,
+          },
+        },
+      },
+    } as never);
+    const fetchMock = installSleepApi(failing.segments, [failing.summary]);
+
+    await syncUserSleep("user-1");
+
+    // Neither first in line nor read at all.
+    expect(stageReads(fetchMock)).toEqual([]);
+    // A full sync still asks for it.
+    fetchMock.mockClear();
+    await syncUserSleep("user-1", { fullSync: true });
+    expect(stageReads(fetchMock)).toHaveLength(1);
   });
 
   it("reads a night whose last read failed first on the next run", async () => {

src/lib/withings/sleep-stage-reads.ts

@@ -11,6 +11,10 @@
  *     no longer read, except by a full sync or while Withings marks it
  *     changed.
  *   - `failed`: the night's last read errored. It is read first next run.
+ *   - `failures`: reads that errored. A night Withings answers with an error
+ *     every time is given up after {@link SLEEP_STAGE_MAX_FAILED_READS} of
+ *     them, like an unstaged one: otherwise it held the failed-first slot and
+ *     failed the leg on every run for as long as it stayed in the window.
  *
  * Versioned and validated on read: anything that is not the current shape
  * reads as no record at all, which is the behaviour before the record
@@ -18,10 +22,23 @@
  */
 
 export const SLEEP_STAGE_MAX_EMPTY_READS = 3;
+export const SLEEP_STAGE_MAX_FAILED_READS = 3;
 
 export interface SleepStageNightRecord {
   empty: number;
   failed: boolean;
+  failures: number;
+}
+
+/** A night the sync no longer reads outside a full sync. */
+export function isSleepNightGivenUp(
+  night: SleepStageNightRecord | undefined,
+): boolean {
+  return (
+    night !== undefined &&
+    (night.empty >= SLEEP_STAGE_MAX_EMPTY_READS ||
+      night.failures >= SLEEP_STAGE_MAX_FAILED_READS)
+  );
 }
 
 export type SleepStageReads = Map<string, SleepStageNightRecord>;
@@ -42,26 +59,35 @@ export function parseSleepStageReads(raw: unknown): SleepStageReads {
   for (const [key, value] of Object.entries(record.nights)) {
     if (!/^\d+:\d+$/.test(key)) continue;
     if (value === null || typeof value !== "object") continue;
-    const { empty, failed } = value as { empty?: unknown; failed?: unknown };
-    const count =
-      typeof empty === "number" && Number.isInteger(empty) && empty >= 0
-        ? empty
-        : 0;
-    out.set(key, { empty: count, failed: failed === true });
+    const { empty, failed, failures } = value as {
+      empty?: unknown;
+      failed?: unknown;
+      failures?: unknown;
+    };
+    const count = (n: unknown) =>
+      typeof n === "number" && Number.isInteger(n) && n >= 0 ? n : 0;
+    out.set(key, {
+      empty: count(empty),
+      failed: failed === true,
+      failures: count(failures),
+    });
   }
   return out;
 }
 
+type SerialisedNight = { empty: number; failed?: true; failures?: number };
+
 export function serialiseSleepStageReads(reads: SleepStageReads): {
   v: 1;
-  nights: Record<string, { empty: number; failed?: true }>;
+  nights: Record<string, SerialisedNight>;
 } {
-  const nights: Record<string, { empty: number; failed?: true }> = {};
+  const nights: Record<string, SerialisedNight> = {};
   for (const [key, night] of reads) {
-    if (night.empty === 0 && !night.failed) continue;
-    nights[key] = night.failed
-      ? { empty: night.empty, failed: true }
-      : { empty: night.empty };
+    if (night.empty === 0 && !night.failed && night.failures === 0) continue;
+    const out: SerialisedNight = { empty: night.empty };
+    if (night.failed) out.failed = true;
+    if (night.failures > 0) out.failures = night.failures;
+    nights[key] = out;
   }
   return { v: 1, nights };
 }
@@ -83,10 +109,15 @@ export function nextSleepStageReads(
     if (windowNightKeys.has(key)) next.set(key, { ...night });
   }
   for (const [key, outcome] of outcomes) {
-    const prior = next.get(key) ?? { empty: 0, failed: false };
+    const prior = next.get(key) ?? { empty: 0, failed: false, failures: 0 };
     if (outcome === "staged") next.delete(key);
-    else if (outcome === "failed") next.set(key, { ...prior, failed: true });
-    else next.set(key, { empty: prior.empty + 1, failed: false });
+    else if (outcome === "failed") {
+      next.set(key, {
+        ...prior,
+        failed: true,
+        failures: prior.failures + 1,
+      });
+    } else next.set(key, { ...prior, empty: prior.empty + 1, failed: false });
   }
   return next;
 }

src/lib/withings/sync-sleep.ts

@@ -64,7 +64,7 @@ import {
   nextSleepStageReads,
   parseSleepStageReads,
   serialiseSleepStageReads,
-  SLEEP_STAGE_MAX_EMPTY_READS,
+  isSleepNightGivenUp,
   sleepNightKey,
   type SleepStageReadOutcome,
   type SleepStageReads,
@@ -478,6 +478,7 @@ export async function syncUserSleep(
           startdate: endUnix - DAY_SECONDS,
           enddate: endUnix,
           wholeNight: false,
+          recent: true,
         },
       ];
 
@@ -507,16 +508,26 @@ export async function syncUserSleep(
         annotate({ meta: { withings_sleep_segments_skipped: "api_plan" } });
         break;
       }
-      stageFailure ??= err;
       if (window.nightKey) outcomes.set(window.nightKey, "failed");
+      const reauth =
+        err instanceof WithingsApiError &&
+        (err.withingsStatus === 403 ||
+          err.classification === "reauth_required");
+      // An older night that will not read is that night's problem, not the
+      // connection's: the rest synced, so it is a warning, and the record
+      // gives the night up after a few tries. A recent night, the blind
+      // slice, or a refused grant fails the leg.
+      if (window.recent || reauth) {
+        stageFailure ??= err;
+      } else {
+        getEvent()?.addWarning(
+          `withings sleep: stage read failed for an older night of ${userId}: ${err}`,
+        );
+        annotate({ meta: { withings_sleep_older_night_failed: true } });
+      }
       // A revoked grant or a missing scope refuses every night alike, so the
       // rest are not asked for.
-      if (
-        err instanceof WithingsApiError &&
-        (err.withingsStatus === 403 || err.classification === "reauth_required")
-      ) {
-        break;
-      }
+      if (reauth) break;
       continue;
     }
     let staged = false;
@@ -791,6 +802,11 @@ interface StageWindow {
   wholeNight: boolean;
   /** The summary night it reads (`sleepNightKey`), absent for the blind slice. */
   nightKey?: string;
+  /**
+   * The night ended, or Withings changed it, in the last 48 hours (the blind
+   * slice is always recent). Only a failed recent read fails the leg.
+   */
+  recent: boolean;
 }
 
 /**
@@ -824,18 +840,20 @@ async function stageWindowsToFetch(
     });
   }
   const all = [...nights.values()].sort((x, y) => y.enddate - x.enddate);
+  const nowUnix = Math.floor(now.getTime() / 1000);
   const toWindow = (n: {
     startdate: number;
     enddate: number;
+    changed: number;
   }): StageWindow => ({
     startdate: n.startdate,
     enddate: Math.min(n.enddate, n.startdate + DAY_SECONDS),
     wholeNight: n.enddate - n.startdate <= DAY_SECONDS,
     nightKey: sleepNightKey(n),
+    recent: n.changed >= nowUnix - RECENT_NIGHT_SECONDS,
   });
   if (opts.all || all.length === 0) return all.map(toWindow);
 
-  const nowUnix = Math.floor(now.getTime() / 1000);
   // Recent: ended, or was changed by Withings, within the last 48 hours.
   const recent = all.filter((n) => n.changed >= nowUnix - RECENT_NIGHT_SECONDS);
   const older = all.filter((n) => n.changed < nowUnix - RECENT_NIGHT_SECONDS);
@@ -865,7 +883,7 @@ async function stageWindowsToFetch(
     opts.reads.get(sleepNightKey(n));
   const missing = older
     .filter((n) => !endsAt.some((t) => t > n.startdate && t <= n.enddate))
-    .filter((n) => (record(n)?.empty ?? 0) < SLEEP_STAGE_MAX_EMPTY_READS)
+    .filter((n) => !isSleepNightGivenUp(record(n)))
     .sort((x, y) => x.enddate - y.enddate);
   const failed = missing.filter((n) => record(n)?.failed === true);
   const rest = missing.filter((n) => record(n)?.failed !== true);