fix(integrations): report a page walk that stops short instead of ending quietly

The WHOOP, Oura and Withings list walks and the Fitbit exercise walk each
kept a page loop that, at its ceiling or on a cursor that did not advance,
returned what it had read and said nothing, and the backfill around it
stamped the history complete. `walkPages` keeps a ceiling but reports a
walk that stops with a page pending, or on a cursor it already followed,
as truncated. A backfill reads that through one verdict: a failure
retries, a truncated collection leaves the marker unset without a retry,
a clean run stamps it. The Fitbit exercise walk may read a hundred pages
during a backfill instead of twelve. A guard keeps page-counting loops out
of the provider directories, with Google Health and Strava listed until
their next change.
This commit is contained in:
Marc-André Bombeck committed 2026-10-20 05:06:21 +02:00
1 parent 0497685db0
commit 483c84b42e
12 files changed
+655 -295

No files matched your search

@@ -0,0 +1,66 @@
import { describe, expect, it, vi } from "vitest";
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
import { backfillVerdict, walkPages, withPageWalkLedger } from "../page-walk";
describe("walkPages", () => {
it("reads until a page has no next", async () => {
const walk = await walkPages<number, string>(
"test.done",
10,
async (cursor) => ({
items: [`p${cursor ?? 0}`],
next: (cursor ?? 0) < 2 ? (cursor ?? 0) + 1 : null,
}),
);
expect(walk).toEqual({
items: ["p0", "p1", "p2"],
pages: 3,
truncated: false,
});
});
it("reports a walk that reaches its ceiling with a page pending", async () => {
const walk = await walkPages<number, number>(
"test.ceiling",
4,
async (cursor) => ({ items: [cursor ?? 0], next: (cursor ?? 0) + 1 }),
);
expect(walk.truncated).toBe(true);
expect(walk.pages).toBe(4);
expect(walk.items).toEqual([0, 1, 2, 3]);
});
it("stops on a cursor it already followed, the first one included", async () => {
const fetchPage = vi.fn(async () => ({ items: [1], next: 0 }));
const walk = await walkPages<number, number>(
"test.repeat",
50,
fetchPage,
0,
);
expect(walk.truncated).toBe(true);
expect(fetchPage).toHaveBeenCalledTimes(1);
});
});
describe("withPageWalkLedger and backfillVerdict", () => {
it("collects the truncated walks of a run", async () => {
const { result, truncated } = await withPageWalkLedger(async () => {
await walkPages("b.walk", 1, async () => ({ items: [], next: "x" }));
await walkPages("a.walk", 5, async () => ({ items: [], next: null }));
return "done";
});
expect(result).toBe("done");
expect(truncated).toEqual(["b.walk"]);
});
it("retries a failure, leaves a truncated history incomplete, stamps a clean one", () => {
expect(backfillVerdict({ failed: true, truncated: ["a"] })).toBe("retry");
expect(backfillVerdict({ failed: false, truncated: ["a"] })).toBe(
"incomplete",
);
expect(backfillVerdict({ failed: false, truncated: [] })).toBe("complete");
});
});
+112
View File
@@ -0,0 +1,112 @@
/**
* The one way a provider client walks a paged collection, and the contract a
* history backfill reads off it.
*
* Every client used to keep its own `while (pages < MAX)` loop, and each one
* ended the same way at its ceiling: it returned what it had read, said
* nothing, and the backfill around it stamped the connection complete. A
* history longer than the ceiling, or a cursor that never advanced, was cut
* off for good without a trace. `walkPages` keeps the ceiling, because a
* runaway cursor must stop somewhere, but a walk that stops with a page still
* pending, or on a cursor it has already followed, reports itself
* `truncated`: on the run's wide event, and in the ledger a backfill opens
* with `withPageWalkLedger`.
*
* The backfill contract (`backfillVerdict`), the same for every provider:
* - a failed collection is transient: the job throws and pg-boss retries;
* - a truncated collection is not complete: the marker is not stamped, and
* boot discovery offers the account again, without a retry storm;
* - a connection parked for re-authorisation is permanent until the person
* reconnects: the runner returns without stamping and without a retry.
*
* Google Health keeps its own progress-based walk (`google-health/
* backfill-bounds.ts`), whose `no_progress` end is this module's runaway case
* measured in time rather than in cursors; it moves onto this contract with
* its next change. `src/__tests__/page-walk-guard.test.ts` keeps hand-rolled
* page loops out of the provider directories.
*/
import { AsyncLocalStorage } from "node:async_hooks";
import { annotate } from "@/lib/logging/context";
/** One page as the client read it. A null `next` ends the walk. */
export interface WalkPage<C, T> {
items: readonly T[];
next: C | null | undefined;
}
/** What the walk came to. */
export interface PageWalk<T> {
items: T[];
pages: number;
/** Stopped at the ceiling or on a repeated cursor with history left. */
truncated: boolean;
}
const ledger = new AsyncLocalStorage<Set<string>>();
/**
* Walk `fetchPage` from `first` (or no cursor) until a page has no `next`,
* at most `maxPages` pages. `label` names the collection (`whoop.recovery`)
* on the wide event and in the ledger.
*/
export async function walkPages<C, T>(
label: string,
maxPages: number,
fetchPage: (cursor: C | undefined, page: number) => Promise<WalkPage<C, T>>,
first?: C,
): Promise<PageWalk<T>> {
const items: T[] = [];
const seen = new Set<C>(first === undefined ? [] : [first]);
let cursor = first;
for (let page = 0; page < maxPages; page += 1) {
const result = await fetchPage(cursor, page);
items.push(...result.items);
const next = result.next;
if (next === null || next === undefined) {
return { items, pages: page + 1, truncated: false };
}
if (seen.has(next)) return truncated(label, items, page + 1, "repeat");
seen.add(next);
cursor = next;
}
return truncated(label, items, maxPages, "ceiling");
}
function truncated<T>(
label: string,
items: T[],
pages: number,
reason: "ceiling" | "repeat",
): PageWalk<T> {
annotate({
action: { name: "integration.page_walk.truncated" },
meta: { walk: label, pages, reason },
});
ledger.getStore()?.add(label);
return { items, pages, truncated: true };
}
/**
* Run `run` with a ledger of the walks inside it that ended truncated. A
* backfill wraps its full sync in this and reads the ledger before it
* stamps its marker.
*/
export async function withPageWalkLedger<R>(
run: () => Promise<R>,
): Promise<{ result: R; truncated: string[] }> {
const walks = new Set<string>();
const result = await ledger.run(walks, run);
return { result, truncated: [...walks].sort() };
}
/** What a backfill does with its full sync's outcome. */
export type BackfillVerdict = "complete" | "retry" | "incomplete";
export function backfillVerdict(outcome: {
failed: boolean;
truncated: readonly string[];
}): BackfillVerdict {
if (outcome.failed) return "retry";
return outcome.truncated.length > 0 ? "incomplete" : "complete";
}