fix(integrations): end a page walk on an empty cursor

WHOOP and Oura answer the last page with an empty next_token as well as
with null. The walk followed the empty string as a cursor, the client sent
no token, read the first page a second time and the walk ended as a repeat,
so the records came in twice and the backfill stayed incomplete for good.
An empty cursor now ends the walk, for every provider that walks through it.
This commit is contained in:
Marc-André Bombeck committed 2026-10-20 09:34:21 +02:00
1 parent 34eea153a8
commit d770f81278
4 files changed
+53 -3

No files matched your search

@@ -32,6 +32,16 @@ describe("walkPages", () => {
expect(walk.items).toEqual([0, 1, 2, 3]);
});
it("ends on an empty cursor instead of following it back to the first page", async () => {
const fetchPage = vi.fn(async (cursor: string | undefined) => ({
items: [cursor ?? "first"],
next: "",
}));
const walk = await walkPages<string, string>("test.empty", 10, fetchPage);
expect(walk).toEqual({ items: ["first"], pages: 1, truncated: false });
expect(fetchPage).toHaveBeenCalledTimes(1);
});
it("stops on a cursor it already followed, the first one included", async () => {
const fetchPage = vi.fn(async () => ({ items: [1], next: 0 }));
const walk = await walkPages<number, number>(
+16 -3
View File
@@ -15,7 +15,14 @@
* The backfill contract (`backfillVerdict`), the same for every provider:
* - a failed collection is transient: the job throws and pg-boss retries;
* - a truncated collection is not complete: the marker is not stamped, and
* boot discovery offers the account again, without a retry storm;
* boot discovery offers the account again, without a retry storm. Where
* a real history can outrun the ceiling (Strava's forty pages, about
* 4 000 activities), the backfill records how far it got and the next
* run goes on from there (`strava-backfill.ts`), or each run would read
* the same newest pages again. The other ceilings sit far above any
* history (a thousand pages for WHOOP and Withings), so a truncation
* there is a cursor that does not advance, which no resume can cure;
* the repeat check ends it on the first page it would read twice;
* - a connection parked for re-authorisation is permanent until the person
* reconnects: the runner returns without stamping and without a retry.
*
@@ -32,7 +39,13 @@ import { AsyncLocalStorage } from "node:async_hooks";
import { annotate } from "@/lib/logging/context";
/** One page as the client read it. A null `next` ends the walk. */
/**
* One page as the client read it. A null, absent or empty `next` ends the
* walk: WHOOP and Oura answer the last page with `next_token: ""` as well as
* with null, and an empty token followed as a cursor is no cursor at all, so
* the client asked for the first page again, read it twice and the walk
* ended as a repeat, leaving the backfill incomplete for good.
*/
export interface WalkPage<C, T> {
items: readonly T[];
next: C | null | undefined;
@@ -66,7 +79,7 @@ export async function walkPages<C, T>(
const result = await fetchPage(cursor, page);
items.push(...result.items);
const next = result.next;
if (next === null || next === undefined) {
if (next === null || next === undefined || next === "") {
return { items, pages: page + 1, truncated: false };
}
if (seen.has(next)) return truncated(label, items, page + 1, "repeat");
+17
View File
@@ -286,6 +286,23 @@ describe("fetchReadiness pagination", () => {
});
expect(r).toHaveLength(2);
});
it("ends on an empty next_token without reading the first page again", async () => {
const page = {
status: 200,
body: {
data: [{ id: "1", day: "2026-06-09", score: 70 }],
next_token: "",
},
};
const fetchMock = installFetchMock([page, page]);
const r = await fetchReadiness("tok", {
startDate: "2026-06-01",
endDate: "2026-06-10",
});
expect(r).toHaveLength(1);
expect(fetchMock).toHaveBeenCalledTimes(1);
});
});
describe("mapReadiness", () => {
+10
View File
@@ -138,6 +138,16 @@ describe("fetchCollection pagination", () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it("ends on an empty next_token without reading the first page again", async () => {
const fetchMock = installFetchMock([
{ status: 200, body: { records: [{ id: 1 }], next_token: "" } },
{ status: 200, body: { records: [{ id: 1 }], next_token: "" } },
]);
const recs = await fetchRecoveries("at");
expect(recs).toHaveLength(1);
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it("throws WhoopApiError when a page returns 500", async () => {
installFetchMock([{ status: 500, body: { error: "boom" } }]);
await expect(fetchRecoveries("at")).rejects.toBeInstanceOf(WhoopApiError);