feat(oura): backfill the full history in resumable slices

Every Oura collection, workouts and heart rate included, is now read
back to the start of the account's history by a backfill that runs
through the shared integration lane in five-minute slices. Each
collection walks from the day the backfill began toward the past one
window at a time (ninety days, thirty for sleep, seven for heart rate)
and the progress note on the user row records after every window where
it got to, so a slice cut short by its budget, the lane or a rate limit
resumes there. A 429 ends the slice and the next one waits for the
Retry-After Oura named. Activity and sleep are walked first to learn
where history starts; the rest stop a week before it.

Heart rate streams a page at a time and keeps only the days it touched,
so memory stays flat however long the history is. A collection the
grant does not cover is a scope note, not a failure; the first slice of
the next chain asks again, and a connect clears the completion stamp so
a newly granted scope is read without walking anything twice.

The two columns arrive empty, so every existing connection is offered
the backfill at the next boot without reconnecting. Before anything
else the backfill retires the WALKING_RUNNING_DISTANCE rows earlier
syncs wrote from the equivalent walking distance and refolds their days;
an account that kept such rows after disconnecting is offered too.
This commit is contained in:
Marc-André Bombeck committed 2026-10-21 19:53:57 +02:00
1 parent 7585994697
commit bce9ad1c25
21 files changed
+1462 -10

No files matched your search

@@ -206,12 +206,17 @@ describe("GET /api/oura/callback — issuer", () => {
* card for up to an hour.
*/
describe("GET /api/oura/callback — connect-time first sync", () => {
it("enqueues exactly one targeted oura-sync job for the connecting user", async () => {
it("enqueues one targeted oura-sync job and the history backfill for the connecting user", async () => {
const res = await GET(makeReq({ code: "c", state: "S", cookie: "S" }));
expect(res.headers.get("location")).toContain("oura=connected");
expect(bossSend).toHaveBeenCalledTimes(1);
expect(bossSend).toHaveBeenCalledTimes(2);
expect(bossSend).toHaveBeenCalledWith("oura-sync", { userId: "u1" });
expect(bossSend).toHaveBeenCalledWith(
"oura-backfill",
expect.objectContaining({ userId: "u1" }),
expect.objectContaining({ singletonKey: "oura-backfill|u1" }),
);
});
it("still connects when no boss instance is available", async () => {
+24
View File
@@ -17,6 +17,11 @@ import { ouraCollectionsMissingFromGrant } from "@/lib/oura/scope";
import { markReconnected } from "@/lib/integrations/status";
import { OURA_SYNC_QUEUE } from "@/lib/jobs/integration-poll-queues";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import { integrationBackfillSourceOptions } from "@/lib/jobs/integration-backfill-admission";
import {
OURA_BACKFILL_QUEUE,
type OuraBackfillPayload,
} from "@/lib/jobs/oura-backfill";
import { NextRequest, NextResponse } from "next/server";
/**
@@ -134,6 +139,10 @@ async function completeCallback(
ouraRefreshTokenEncrypted: encrypt(tokens.refresh_token),
ouraScopeDenied:
missing && missing.length > 0 ? missing : Prisma.DbNull,
// Open the history backfill again: a first connect has none yet,
// and a reconnect may grant what the old grant refused. What was
// already walked stays in the progress note and is not read again.
ouraBackfillCompletedAt: null,
},
});
@@ -153,6 +162,21 @@ async function completeCallback(
.catch((err) =>
getEvent()?.addWarning(`oura-sync connect enqueue failed: ${err}`),
);
const backfill: OuraBackfillPayload = {
userId,
enqueuedAt: new Date().toISOString(),
};
await boss
.send(
OURA_BACKFILL_QUEUE,
backfill,
integrationBackfillSourceOptions(`oura-backfill|${userId}`),
)
.catch((err) =>
getEvent()?.addWarning(
`oura-backfill connect enqueue failed: ${err}`,
),
);
}
const response = NextResponse.redirect(