From d770f81278223627dfb09f0e2cd7da82203bab18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 09:34:21 +0200 Subject: [PATCH] fix(integrations): end a page walk on an empty cursor WHOOP and Oura answer the last page with an empty next_token as well as with null. The walk followed the empty string as a cursor, the client sent no token, read the first page a second time and the walk ended as a repeat, so the records came in twice and the backfill stayed incomplete for good. An empty cursor now ends the walk, for every provider that walks through it. --- .../integrations/__tests__/page-walk.test.ts | 10 ++++++++++ src/lib/integrations/page-walk.ts | 19 ++++++++++++++++--- src/lib/oura/__tests__/client.test.ts | 17 +++++++++++++++++ src/lib/whoop/__tests__/client.test.ts | 10 ++++++++++ 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/lib/integrations/__tests__/page-walk.test.ts b/src/lib/integrations/__tests__/page-walk.test.ts index 69980ff49..4db821965 100644 --- a/src/lib/integrations/__tests__/page-walk.test.ts +++ b/src/lib/integrations/__tests__/page-walk.test.ts @@ -32,6 +32,16 @@ describe("walkPages", () => { expect(walk.items).toEqual([0, 1, 2, 3]); }); + it("ends on an empty cursor instead of following it back to the first page", async () => { + const fetchPage = vi.fn(async (cursor: string | undefined) => ({ + items: [cursor ?? "first"], + next: "", + })); + const walk = await walkPages("test.empty", 10, fetchPage); + expect(walk).toEqual({ items: ["first"], pages: 1, truncated: false }); + expect(fetchPage).toHaveBeenCalledTimes(1); + }); + it("stops on a cursor it already followed, the first one included", async () => { const fetchPage = vi.fn(async () => ({ items: [1], next: 0 })); const walk = await walkPages( diff --git a/src/lib/integrations/page-walk.ts b/src/lib/integrations/page-walk.ts index 08af1771d..b8b92f451 100644 --- a/src/lib/integrations/page-walk.ts +++ b/src/lib/integrations/page-walk.ts @@ -15,7 +15,14 @@ * The backfill contract (`backfillVerdict`), the same for every provider: * - a failed collection is transient: the job throws and pg-boss retries; * - a truncated collection is not complete: the marker is not stamped, and - * boot discovery offers the account again, without a retry storm; + * boot discovery offers the account again, without a retry storm. Where + * a real history can outrun the ceiling (Strava's forty pages, about + * 4 000 activities), the backfill records how far it got and the next + * run goes on from there (`strava-backfill.ts`), or each run would read + * the same newest pages again. The other ceilings sit far above any + * history (a thousand pages for WHOOP and Withings), so a truncation + * there is a cursor that does not advance, which no resume can cure; + * the repeat check ends it on the first page it would read twice; * - a connection parked for re-authorisation is permanent until the person * reconnects: the runner returns without stamping and without a retry. * @@ -32,7 +39,13 @@ import { AsyncLocalStorage } from "node:async_hooks"; import { annotate } from "@/lib/logging/context"; -/** One page as the client read it. A null `next` ends the walk. */ +/** + * One page as the client read it. A null, absent or empty `next` ends the + * walk: WHOOP and Oura answer the last page with `next_token: ""` as well as + * with null, and an empty token followed as a cursor is no cursor at all, so + * the client asked for the first page again, read it twice and the walk + * ended as a repeat, leaving the backfill incomplete for good. + */ export interface WalkPage { items: readonly T[]; next: C | null | undefined; @@ -66,7 +79,7 @@ export async function walkPages( const result = await fetchPage(cursor, page); items.push(...result.items); const next = result.next; - if (next === null || next === undefined) { + if (next === null || next === undefined || next === "") { return { items, pages: page + 1, truncated: false }; } if (seen.has(next)) return truncated(label, items, page + 1, "repeat"); diff --git a/src/lib/oura/__tests__/client.test.ts b/src/lib/oura/__tests__/client.test.ts index b8e6b0893..e4c6148df 100644 --- a/src/lib/oura/__tests__/client.test.ts +++ b/src/lib/oura/__tests__/client.test.ts @@ -286,6 +286,23 @@ describe("fetchReadiness pagination", () => { }); expect(r).toHaveLength(2); }); + + it("ends on an empty next_token without reading the first page again", async () => { + const page = { + status: 200, + body: { + data: [{ id: "1", day: "2026-06-09", score: 70 }], + next_token: "", + }, + }; + const fetchMock = installFetchMock([page, page]); + const r = await fetchReadiness("tok", { + startDate: "2026-06-01", + endDate: "2026-06-10", + }); + expect(r).toHaveLength(1); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); }); describe("mapReadiness", () => { diff --git a/src/lib/whoop/__tests__/client.test.ts b/src/lib/whoop/__tests__/client.test.ts index b5e4cab9c..fa0db0f1b 100644 --- a/src/lib/whoop/__tests__/client.test.ts +++ b/src/lib/whoop/__tests__/client.test.ts @@ -138,6 +138,16 @@ describe("fetchCollection pagination", () => { expect(fetchMock).toHaveBeenCalledTimes(1); }); + it("ends on an empty next_token without reading the first page again", async () => { + const fetchMock = installFetchMock([ + { status: 200, body: { records: [{ id: 1 }], next_token: "" } }, + { status: 200, body: { records: [{ id: 1 }], next_token: "" } }, + ]); + const recs = await fetchRecoveries("at"); + expect(recs).toHaveLength(1); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + it("throws WhoopApiError when a page returns 500", async () => { installFetchMock([{ status: 500, body: { error: "boom" } }]); await expect(fetchRecoveries("at")).rejects.toBeInstanceOf(WhoopApiError);