mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-17 23:52:24 +00:00
fix(integrations): report a page walk that stops short instead of ending quietly
The WHOOP, Oura and Withings list walks and the Fitbit exercise walk each kept a page loop that, at its ceiling or on a cursor that did not advance, returned what it had read and said nothing, and the backfill around it stamped the history complete. `walkPages` keeps a ceiling but reports a walk that stops with a page pending, or on a cursor it already followed, as truncated. A backfill reads that through one verdict: a failure retries, a truncated collection leaves the marker unset without a retry, a clean run stamps it. The Fitbit exercise walk may read a hundred pages during a backfill instead of twelve. A guard keeps page-counting loops out of the provider directories, with Google Health and Strava listed until their next change.
This commit is contained in:
1 parent
0497685db0
commit
483c84b42e
12 files changed
+655
-295
No files matched your search
@@ -0,0 +1,66 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
|
||||
|
||||
import { backfillVerdict, walkPages, withPageWalkLedger } from "../page-walk";
|
||||
|
||||
describe("walkPages", () => {
|
||||
it("reads until a page has no next", async () => {
|
||||
const walk = await walkPages<number, string>(
|
||||
"test.done",
|
||||
10,
|
||||
async (cursor) => ({
|
||||
items: [`p${cursor ?? 0}`],
|
||||
next: (cursor ?? 0) < 2 ? (cursor ?? 0) + 1 : null,
|
||||
}),
|
||||
);
|
||||
expect(walk).toEqual({
|
||||
items: ["p0", "p1", "p2"],
|
||||
pages: 3,
|
||||
truncated: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a walk that reaches its ceiling with a page pending", async () => {
|
||||
const walk = await walkPages<number, number>(
|
||||
"test.ceiling",
|
||||
4,
|
||||
async (cursor) => ({ items: [cursor ?? 0], next: (cursor ?? 0) + 1 }),
|
||||
);
|
||||
expect(walk.truncated).toBe(true);
|
||||
expect(walk.pages).toBe(4);
|
||||
expect(walk.items).toEqual([0, 1, 2, 3]);
|
||||
});
|
||||
|
||||
it("stops on a cursor it already followed, the first one included", async () => {
|
||||
const fetchPage = vi.fn(async () => ({ items: [1], next: 0 }));
|
||||
const walk = await walkPages<number, number>(
|
||||
"test.repeat",
|
||||
50,
|
||||
fetchPage,
|
||||
0,
|
||||
);
|
||||
expect(walk.truncated).toBe(true);
|
||||
expect(fetchPage).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("withPageWalkLedger and backfillVerdict", () => {
|
||||
it("collects the truncated walks of a run", async () => {
|
||||
const { result, truncated } = await withPageWalkLedger(async () => {
|
||||
await walkPages("b.walk", 1, async () => ({ items: [], next: "x" }));
|
||||
await walkPages("a.walk", 5, async () => ({ items: [], next: null }));
|
||||
return "done";
|
||||
});
|
||||
expect(result).toBe("done");
|
||||
expect(truncated).toEqual(["b.walk"]);
|
||||
});
|
||||
|
||||
it("retries a failure, leaves a truncated history incomplete, stamps a clean one", () => {
|
||||
expect(backfillVerdict({ failed: true, truncated: ["a"] })).toBe("retry");
|
||||
expect(backfillVerdict({ failed: false, truncated: ["a"] })).toBe(
|
||||
"incomplete",
|
||||
);
|
||||
expect(backfillVerdict({ failed: false, truncated: [] })).toBe("complete");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* The one way a provider client walks a paged collection, and the contract a
|
||||
* history backfill reads off it.
|
||||
*
|
||||
* Every client used to keep its own `while (pages < MAX)` loop, and each one
|
||||
* ended the same way at its ceiling: it returned what it had read, said
|
||||
* nothing, and the backfill around it stamped the connection complete. A
|
||||
* history longer than the ceiling, or a cursor that never advanced, was cut
|
||||
* off for good without a trace. `walkPages` keeps the ceiling, because a
|
||||
* runaway cursor must stop somewhere, but a walk that stops with a page still
|
||||
* pending, or on a cursor it has already followed, reports itself
|
||||
* `truncated`: on the run's wide event, and in the ledger a backfill opens
|
||||
* with `withPageWalkLedger`.
|
||||
*
|
||||
* The backfill contract (`backfillVerdict`), the same for every provider:
|
||||
* - a failed collection is transient: the job throws and pg-boss retries;
|
||||
* - a truncated collection is not complete: the marker is not stamped, and
|
||||
* boot discovery offers the account again, without a retry storm;
|
||||
* - a connection parked for re-authorisation is permanent until the person
|
||||
* reconnects: the runner returns without stamping and without a retry.
|
||||
*
|
||||
* Google Health keeps its own progress-based walk (`google-health/
|
||||
* backfill-bounds.ts`), whose `no_progress` end is this module's runaway case
|
||||
* measured in time rather than in cursors; it moves onto this contract with
|
||||
* its next change. `src/__tests__/page-walk-guard.test.ts` keeps hand-rolled
|
||||
* page loops out of the provider directories.
|
||||
*/
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
|
||||
/** One page as the client read it. A null `next` ends the walk. */
|
||||
export interface WalkPage<C, T> {
|
||||
items: readonly T[];
|
||||
next: C | null | undefined;
|
||||
}
|
||||
|
||||
/** What the walk came to. */
|
||||
export interface PageWalk<T> {
|
||||
items: T[];
|
||||
pages: number;
|
||||
/** Stopped at the ceiling or on a repeated cursor with history left. */
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
const ledger = new AsyncLocalStorage<Set<string>>();
|
||||
|
||||
/**
|
||||
* Walk `fetchPage` from `first` (or no cursor) until a page has no `next`,
|
||||
* at most `maxPages` pages. `label` names the collection (`whoop.recovery`)
|
||||
* on the wide event and in the ledger.
|
||||
*/
|
||||
export async function walkPages<C, T>(
|
||||
label: string,
|
||||
maxPages: number,
|
||||
fetchPage: (cursor: C | undefined, page: number) => Promise<WalkPage<C, T>>,
|
||||
first?: C,
|
||||
): Promise<PageWalk<T>> {
|
||||
const items: T[] = [];
|
||||
const seen = new Set<C>(first === undefined ? [] : [first]);
|
||||
let cursor = first;
|
||||
for (let page = 0; page < maxPages; page += 1) {
|
||||
const result = await fetchPage(cursor, page);
|
||||
items.push(...result.items);
|
||||
const next = result.next;
|
||||
if (next === null || next === undefined) {
|
||||
return { items, pages: page + 1, truncated: false };
|
||||
}
|
||||
if (seen.has(next)) return truncated(label, items, page + 1, "repeat");
|
||||
seen.add(next);
|
||||
cursor = next;
|
||||
}
|
||||
return truncated(label, items, maxPages, "ceiling");
|
||||
}
|
||||
|
||||
function truncated<T>(
|
||||
label: string,
|
||||
items: T[],
|
||||
pages: number,
|
||||
reason: "ceiling" | "repeat",
|
||||
): PageWalk<T> {
|
||||
annotate({
|
||||
action: { name: "integration.page_walk.truncated" },
|
||||
meta: { walk: label, pages, reason },
|
||||
});
|
||||
ledger.getStore()?.add(label);
|
||||
return { items, pages, truncated: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `run` with a ledger of the walks inside it that ended truncated. A
|
||||
* backfill wraps its full sync in this and reads the ledger before it
|
||||
* stamps its marker.
|
||||
*/
|
||||
export async function withPageWalkLedger<R>(
|
||||
run: () => Promise<R>,
|
||||
): Promise<{ result: R; truncated: string[] }> {
|
||||
const walks = new Set<string>();
|
||||
const result = await ledger.run(walks, run);
|
||||
return { result, truncated: [...walks].sort() };
|
||||
}
|
||||
|
||||
/** What a backfill does with its full sync's outcome. */
|
||||
export type BackfillVerdict = "complete" | "retry" | "incomplete";
|
||||
|
||||
export function backfillVerdict(outcome: {
|
||||
failed: boolean;
|
||||
truncated: readonly string[];
|
||||
}): BackfillVerdict {
|
||||
if (outcome.failed) return "retry";
|
||||
return outcome.truncated.length > 0 ? "incomplete" : "complete";
|
||||
}
|
||||
Reference in new issue
Block a user