HealthLog/tests/integration/oura-backfill.test.ts

/**
 * The Oura history backfill against the real schema.
 *
 * Oura's API is replaced at the `safeFetch` seam by a generator that builds
 * each page on request: a year and a half of daily activity, one workout, and
 * a heart-rate reading every five minutes over the same span (some 160 000
 * samples, in pages of 1 000). The backfill runs slice after slice, each one
 * cut short by its budget, until it stamps its marker. What is pinned:
 *
 *   - it reaches the end: every heart-rate sample is a PULSE row, every day a
 *     steps row, the workout a Workout row, the marker is stamped, and the
 *     legacy equivalent-distance rows are retired with their rollup day;
 *   - it resumes: no window is read twice across slices, apart from the one
 *     window a 429 interrupted, and that slice sends the next one after the
 *     Retry-After the 429 named;
 *   - it does not grow with the history: the collected heap after the first
 *     quarter of heart-rate pages stays inside a budget a collect-everything
 *     walk of the same pages passes several times over;
 *   - discovery offers an existing connection without a reconnect, and an
 *     account that kept only the legacy rows, and nothing already stamped.
 */
import v8 from "node:v8";
import vm from "node:vm";

import { beforeEach, describe, expect, it, vi } from "vitest";

import { getPrismaClient, truncateAllTables } from "./setup";

const USER = "oura-backfill-owner";
const DAY_MS = 24 * 60 * 60 * 1000;
/** The day the backfill begins, and the end of the generated history. */
const ANCHOR = "2026-10-01";
const NOW = Date.parse(`${ANCHOR}T12:00:00Z`);
/** The first day of the generated history. */
const HISTORY_FROM = "2025-04-01";
const HR_FROM = Date.parse(`${HISTORY_FROM}T00:00:00Z`);
const HR_STEP = 5 * 60_000;
const HR_PAGE = 1000;
const HR_TOTAL = Math.floor((NOW - HR_FROM) / HR_STEP);

const GROWTH_BUDGET_BYTES = 24 * 1024 * 1024;

const fake = vi.hoisted(() => ({
  requests: [] as string[],
  heartRatePages: 0,
  /** Answer the next heart-rate page with a 429 once this many have been served. */
  rateLimitAtPage: -1,
  onPage: null as (() => void) | null,
  /** Collections answered with a 401 naming their scope. */
  denied: new Set<string>(),
  sends: [] as Array<{ queue: string; payload: unknown; options: unknown }>,
}));

vi.mock("@/lib/db-compat", () => ({
  ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));

vi.mock("@/lib/jobs/boss-instance", () => ({
  getGlobalBoss: () => ({
    send: async (queue: string, payload: unknown, options: unknown) => {
      fake.sends.push({ queue, payload, options });
      return `job-${fake.sends.length}`;
    },
  }),
}));

function dayKey(ms: number): string {
  return new Date(ms).toISOString().slice(0, 10);
}

vi.mock("@/lib/safe-fetch", async (importOriginal) => {
  const actual = await importOriginal<typeof import("@/lib/safe-fetch")>();
  const json = (body: unknown, status = 200, headers = {}) =>
    new Response(JSON.stringify(body), {
      status,
      headers: { "content-type": "application/json", ...headers },
    });
  return {
    ...actual,
    // A plain function, not `vi.fn`: a mock records every result, and the
    // results here are whole pages.
    safeFetch: async (url: string) => {
      const u = new URL(url);
      const p = u.searchParams;
      const collection = u.pathname.split("/").pop()!;
      const token = p.get("next_token");
      fake.requests.push(
        `${collection}|${p.get("start_date") ?? p.get("start_datetime")}|${p.get("end_date") ?? p.get("end_datetime")}|${token ?? ""}`,
      );
      if (fake.denied.has(collection)) {
        return json(
          { detail: `Token is not authorized access ${collection} scope.` },
          401,
        );
      }
      if (collection === "heartrate") {
        if (fake.rateLimitAtPage === fake.heartRatePages) {
          fake.rateLimitAtPage = -1;
          return json({ detail: "Too many requests" }, 429, {
            "retry-after": "600",
          });
        }
        const start = Date.parse(p.get("start_datetime")!);
        const end = Date.parse(p.get("end_datetime")!);
        const first = Math.max(0, Math.ceil((start - HR_FROM) / HR_STEP));
        const last = Math.min(HR_TOTAL, Math.ceil((end - HR_FROM) / HR_STEP));
        const from = first + Number(token ?? 0) * HR_PAGE;
        const to = Math.min(last, from + HR_PAGE);
        const data = [];
        for (let i = from; i < to; i++) {
          data.push({
            bpm: 52 + (i % 40),
            source: "awake",
            timestamp: new Date(HR_FROM + i * HR_STEP).toISOString(),
          });
        }
        fake.heartRatePages++;
        fake.onPage?.();
        return json({
          data,
          next_token: to < last ? String(Number(token ?? 0) + 1) : null,
        });
      }
      if (collection === "daily_activity") {
        const from =
          p.get("start_date")! < HISTORY_FROM
            ? HISTORY_FROM
            : p.get("start_date")!;
        const to = p.get("end_date")! > ANCHOR ? ANCHOR : p.get("end_date")!;
        const data = [];
        for (
          let t = Date.parse(`${from}T00:00:00Z`);
          dayKey(t) <= to;
          t += DAY_MS
        ) {
          data.push({
            id: `act-${dayKey(t)}`,
            day: dayKey(t),
            steps: 5000,
            active_calories: 300,
            equivalent_walking_distance: 4000,
          });
        }
        return json({ data, next_token: null });
      }
      if (collection === "workout") {
        const day = "2026-02-14";
        const inRange =
          p.get("start_date")! <= day && day <= p.get("end_date")!;
        return json({
          data: inRange
            ? [
                {
                  id: "oura-workout-1",
                  activity: "cycling",
                  calories: 410,
                  distance: 15000,
                  intensity: "moderate",
                  source: "manual",
                  day,
                  start_datetime: `${day}T08:00:00+00:00`,
                  end_datetime: `${day}T09:00:00+00:00`,
                },
              ]
            : [],
          next_token: null,
        });
      }
      return json({ data: [], next_token: null });
    },
  };
});

const forceGc = ((): (() => void) => {
  v8.setFlagsFromString("--expose-gc");
  const gc = vm.runInNewContext("gc") as () => void;
  v8.setFlagsFromString("--no-expose-gc");
  return gc;
})();

function liveHeapBytes(): number {
  forceGc();
  forceGc();
  return process.memoryUsage().heapUsed;
}

beforeEach(async () => {
  const prisma = getPrismaClient();
  await truncateAllTables(prisma);
  fake.requests = [];
  fake.heartRatePages = 0;
  fake.rateLimitAtPage = -1;
  fake.sends = [];
  fake.denied = new Set();
  const { encrypt } = await import("@/lib/crypto");
  await prisma.user.create({
    data: {
      id: USER,
      username: USER,
      timezone: "UTC",
      ouraAccessTokenEncrypted: encrypt("access"),
      ouraRefreshTokenEncrypted: encrypt("refresh"),
    },
  });
}, 120_000);

/** Seed the rows an earlier sync wrote from the equivalent distance. */
async function seedLegacyDistance(userId: string): Promise<void> {
  const prisma = getPrismaClient();
  await prisma.measurement.createMany({
    data: [
      {
        userId,
        type: "WALKING_RUNNING_DISTANCE",
        source: "OURA",
        value: 4000,
        unit: "m",
        measuredAt: new Date("2026-09-01T12:00:00Z"),
        externalId: "activity:2026-09-01:distance",
      },
      {
        userId,
        type: "WALKING_RUNNING_DISTANCE",
        source: "OURA",
        value: 5200,
        unit: "m",
        measuredAt: new Date("2026-09-02T12:00:00Z"),
        externalId: "activity:2026-09-02:distance",
      },
      // Another source's distance on the same day stays.
      {
        userId,
        type: "WALKING_RUNNING_DISTANCE",
        source: "APPLE_HEALTH",
        value: 3100,
        unit: "m",
        measuredAt: new Date("2026-09-01T18:00:00Z"),
        externalId: "hk-dist-1",
      },
    ],
  });
}

describe("retireOuraEquivalentDistance", () => {
  it("soft-deletes only the OURA equivalent-distance rows and refolds the day", async () => {
    const prisma = getPrismaClient();
    await seedLegacyDistance(USER);
    const { recomputeBucketsForMeasurement } =
      await import("@/lib/rollups/measurement-rollups");
    await recomputeBucketsForMeasurement(
      USER,
      "WALKING_RUNNING_DISTANCE",
      new Date("2026-09-01T12:00:00Z"),
    );
    const ouraDay = () =>
      prisma.measurementRollup.findFirst({
        where: {
          userId: USER,
          type: "WALKING_RUNNING_DISTANCE",
          granularity: "DAY",
          source: "OURA",
          bucketStart: new Date("2026-09-01T00:00:00Z"),
        },
      });
    expect((await ouraDay())?.count).toBe(1);

    const { retireOuraEquivalentDistance } =
      await import("@/lib/oura/retire-distance");
    expect(await retireOuraEquivalentDistance(USER)).toBe(2);
    expect(await retireOuraEquivalentDistance(USER)).toBe(0);

    const live = await prisma.measurement.findMany({
      where: { userId: USER, deletedAt: null },
      select: { source: true, externalId: true },
    });
    expect(live).toEqual([{ source: "APPLE_HEALTH", externalId: "hk-dist-1" }]);
    const after = await ouraDay();
    expect(after?.count ?? 0).toBe(0);
  });
});

describe("Oura history backfill", () => {
  it("walks the whole history in resumable slices with bounded memory", async () => {
    const prisma = getPrismaClient();
    await seedLegacyDistance(USER);
    const { runOuraBackfillForUser } = await import("@/lib/jobs/oura-backfill");

    // A rate limit in the middle of the heart-rate walk.
    fake.rateLimitAtPage = 40;

    let clock = NOW;
    const now = () => (clock += 1000);
    const quarter = Math.ceil(HR_TOTAL / HR_PAGE / 4);
    let anchor: number | null = null;
    let peak = 0;
    let samples = 0;
    fake.onPage = () => {
      if (fake.heartRatePages % 5 !== 0) return;
      const live = liveHeapBytes();
      if (anchor === null) {
        if (fake.heartRatePages >= quarter) anchor = peak = live;
        return;
      }
      samples++;
      peak = Math.max(peak, live);
    };

    let slice = 0;
    let continued = true;
    let slices = 0;
    try {
      while (continued) {
        const run = await runOuraBackfillForUser(USER, () => false, slice, {
          sliceMs: 40_000,
          now,
        });
        continued = run.continued;
        slice += 1;
        slices += 1;
        expect(slices).toBeLessThan(200);
      }
    } finally {
      fake.onPage = null;
    }
    process.stderr.write(
      `[oura-backfill] slices=${slices} hrPages=${fake.heartRatePages} growth=${((peak - (anchor ?? 0)) / 1024 / 1024).toFixed(1)} MB over ${samples} samples\n`,
    );

    // It ran in many slices, and finished.
    expect(slices).toBeGreaterThan(5);
    const user = await prisma.user.findUniqueOrThrow({
      where: { id: USER },
      select: { ouraBackfillCompletedAt: true, ouraBackfillProgress: true },
    });
    expect(user.ouraBackfillCompletedAt).not.toBeNull();
    const progress = user.ouraBackfillProgress as {
      historyStart: string;
      distanceRetired: boolean;
    };
    expect(progress.historyStart).toBe(HISTORY_FROM);
    expect(progress.distanceRetired).toBe(true);

    // Every heart-rate sample, every day, the workout.
    expect(
      await prisma.measurement.count({
        where: { userId: USER, type: "PULSE", source: "OURA" },
      }),
    ).toBe(HR_TOTAL);
    const days =
      (Date.parse(`${ANCHOR}T00:00:00Z`) -
        Date.parse(`${HISTORY_FROM}T00:00:00Z`)) /
        DAY_MS +
      1;
    expect(
      await prisma.measurement.count({
        where: { userId: USER, type: "ACTIVITY_STEPS", source: "OURA" },
      }),
    ).toBe(days);
    const workouts = await prisma.workout.findMany({
      where: { userId: USER, source: "OURA" },
      select: { externalId: true, sportType: true, totalDistanceM: true },
    });
    expect(workouts).toEqual([
      {
        externalId: "oura-workout-1",
        sportType: "cycling",
        totalDistanceM: 15000,
      },
    ]);
    // No equivalent distance written, and the legacy rows retired.
    expect(
      await prisma.measurement.count({
        where: {
          userId: USER,
          source: "OURA",
          type: "WALKING_RUNNING_DISTANCE",
          deletedAt: null,
        },
      }),
    ).toBe(0);

    // Resumed, never restarted: every first page of a window was asked for
    // once, apart from the heart-rate window the 429 interrupted.
    const firstPages = fake.requests.filter((r) => r.endsWith("|"));
    const repeated = firstPages.filter((r, i) => firstPages.indexOf(r) !== i);
    expect(repeated).toHaveLength(1);
    expect(repeated[0]).toMatch(/^heartrate\|/);
    // The slice the 429 ended sent the next one after the wait it named.
    const delays = fake.sends.map(
      (s) => (s.options as { startAfter?: number }).startAfter ?? 0,
    );
    expect(delays).toContain(600);

    // Bounded: the heap after the first quarter of the stream stays flat.
    expect(samples).toBeGreaterThan(5);
    expect(peak - anchor!).toBeLessThan(GROWTH_BUDGET_BYTES);
  }, 900_000);

  it("leaves an existing connection's history to discovery, no reconnect needed", async () => {
    const prisma = getPrismaClient();
    // An account whose Oura connection is gone but whose legacy rows stay.
    await prisma.user.create({
      data: { id: "oura-legacy-only", username: "oura-legacy-only" },
    });
    await seedLegacyDistance("oura-legacy-only");
    // An account already backfilled.
    await prisma.user.create({
      data: {
        id: "oura-done",
        username: "oura-done",
        ouraAccessTokenEncrypted: "x",
        ouraRefreshTokenEncrypted: "y",
        ouraBackfillCompletedAt: new Date(),
      },
    });
    const { enqueueBootTimeOuraBackfill, runOuraBackfillForUser } =
      await import("@/lib/jobs/oura-backfill");

    const result = await enqueueBootTimeOuraBackfill(30);
    expect(result).toEqual({ enqueued: 2, skipped: 0, error: null });
    expect(
      fake.sends.map((s) => (s.payload as { userId: string }).userId).sort(),
    ).toEqual(["oura-backfill-owner", "oura-legacy-only"]);

    // The legacy-only account retires its rows and is stamped at once.
    await runOuraBackfillForUser("oura-legacy-only");
    const legacy = await prisma.user.findUniqueOrThrow({
      where: { id: "oura-legacy-only" },
      select: { ouraBackfillCompletedAt: true },
    });
    expect(legacy.ouraBackfillCompletedAt).not.toBeNull();
    expect(
      await prisma.measurement.count({
        where: { userId: "oura-legacy-only", source: "OURA", deletedAt: null },
      }),
    ).toBe(0);
  });

  it("finishes past a collection the grant refuses, and asks again after a reconnect", async () => {
    const prisma = getPrismaClient();
    const { OURA_BACKFILL_ORDER, runOuraBackfillForUser } =
      await import("@/lib/jobs/oura-backfill");
    // Everything but workouts already walked.
    await prisma.user.update({
      where: { id: USER },
      data: {
        ouraBackfillProgress: {
          anchor: ANCHOR,
          historyStart: "2026-01-01",
          distanceRetired: true,
          collections: Object.fromEntries(
            OURA_BACKFILL_ORDER.filter((n) => n !== "workout").map((n) => [
              n,
              { before: "2015-01-01", done: true },
            ]),
          ),
        },
      },
    });
    fake.denied.add("workout");

    expect(await runOuraBackfillForUser(USER)).toEqual({
      imported: 0,
      continued: false,
    });
    let user = await prisma.user.findUniqueOrThrow({
      where: { id: USER },
      select: { ouraBackfillCompletedAt: true, ouraBackfillProgress: true },
    });
    expect(user.ouraBackfillCompletedAt).not.toBeNull();
    expect(
      (user.ouraBackfillProgress as { collections: Record<string, unknown> })
        .collections.workout,
    ).toEqual({ before: ANCHOR, denied: true });
    // One request found it out; nothing was recorded as a failure.
    expect(fake.requests.filter((r) => r.startsWith("workout|"))).toHaveLength(
      1,
    );
    expect(await prisma.integrationStatus.count()).toBe(0);

    // A reconnect that grants the scope clears the stamp; the next chain
    // reads workouts and nothing else.
    fake.denied.clear();
    fake.requests = [];
    await prisma.user.update({
      where: { id: USER },
      data: { ouraBackfillCompletedAt: null },
    });
    await runOuraBackfillForUser(USER);
    user = await prisma.user.findUniqueOrThrow({
      where: { id: USER },
      select: { ouraBackfillCompletedAt: true, ouraBackfillProgress: true },
    });
    expect(user.ouraBackfillCompletedAt).not.toBeNull();
    expect(new Set(fake.requests.map((r) => r.split("|")[0]))).toEqual(
      new Set(["workout"]),
    );
    expect(
      await prisma.workout.count({ where: { userId: USER, source: "OURA" } }),
    ).toBe(1);
  });
});