From 8ce8065372e9cb09d277046671358955f51592a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Mon, 19 Oct 2026 17:26:04 +0200 Subject: [PATCH] test(integration): size the multi-year backfill run from the environment The multi-year Google Health backfill run walked three and four years of heart rate on every pull request, some 3.7 million readings in over twenty minutes, which filled most of its shard's time budget. What a run costs is its pages, so a shorter history only helps with shorter slices. HEALTHLOG_BACKFILL_TEST_DAYS now sets the size. Unset, the runs walk 120 and 160 days in 50-page slices with the faults at the same places within them and the light collections already walked, about a tenth of the pages. The heap growth bound applies to the full run only. The full three and four years run nightly and on demand in integration-long.yml. Refs #1220 --- .github/workflows/integration-long.yml | 45 +++++++ .../google-health-backfill-multi-year.test.ts | 117 ++++++++++++++---- 2 files changed, 141 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/integration-long.yml diff --git a/.github/workflows/integration-long.yml b/.github/workflows/integration-long.yml new file mode 100644 index 000000000..e249156b1 --- /dev/null +++ b/.github/workflows/integration-long.yml @@ -0,0 +1,45 @@ +name: Integration tests (long) + +# The integration tests that take their size from the environment run small +# on every pull request and at full size here, once a night. Today that is +# the Google Health multi-year backfill: 45 and 60 days of heart rate on a +# pull request, three and four years (some 3.7 million readings, over twenty +# minutes) here. See the header of +# tests/integration/google-health-backfill-multi-year.test.ts. +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + long: + runs-on: ubuntu-24.04 + timeout-minutes: 75 + name: integration long + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - run: pnpm db:generate + + - run: pnpm exec vitest run --config vitest.integration.config.mts tests/integration/google-health-backfill-multi-year.test.ts + env: + HEALTHLOG_BACKFILL_TEST_DAYS: "1095" + # Under the production heap ceiling, which the test's heap check + # is about. + NODE_OPTIONS: "--max-old-space-size=524" + # Quoted for the reason given in integration.yml. + API_TOKEN_HMAC_KEY: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" + SESSION_SECRET: "integration-tests" diff --git a/tests/integration/google-health-backfill-multi-year.test.ts b/tests/integration/google-health-backfill-multi-year.test.ts index da6915dcc..40869dace 100644 --- a/tests/integration/google-health-backfill-multi-year.test.ts +++ b/tests/integration/google-health-backfill-multi-year.test.ts @@ -22,9 +22,10 @@ * - the hourly sync runs between slices (and beside one, by waiting for * it) instead of being skipped for as long as the import runs; * - every slice writes a log event naming its requests and where it got to; - * - the heap does not grow with the history (run it under the production - * ceiling: `NODE_OPTIONS=--max-old-space-size=524`); - * - and, in a second run over four years, a Google that refuses the resume + * - over the full history, the heap does not grow with it (run it under + * the production ceiling: `NODE_OPTIONS=--max-old-space-size=524`); + * - and, in a second run over four years (a third longer than the first), + * a Google that refuses the resume * edge's upper bound (`<=`): the backfill falls back to reading without * it, notes the refusal so it is met only once, keeps its place in the * unbounded list from slice to slice so the pages it reads stay linear in @@ -34,6 +35,24 @@ * Google is replaced at the `safeFetch` seam by a generator that builds each * page on request from the filter and page token it is asked with, so the * fixture never sits in memory. + * + * How much history it walks is set by `HEALTHLOG_BACKFILL_TEST_DAYS`. What a + * run costs is its heart-rate pages (a request, a thousand row writes and a + * progress note each), so a shorter history only helps if the slices shrink + * with it. Unset, the first run walks 120 days and the second 160, at one + * reading a minute and a thousand a page, in slices of 50 pages instead of + * 500, with the faults moved to the same places within them: the same + * sequence of slices, boundaries, retries and rate limit, at about a tenth + * of the pages. Two things differ. The light collections start out walked, + * because the first slice walks them before heart rate (some hundred empty + * requests here) and a 50-page slice would spend itself on them; and 120 + * days is a floor, not a choice: the first hourly sync reads the last 30 + * days, the first slice has to have written them already, as it has in the + * full run, or the backfill's import counts no longer add up to the history, + * and the history has to outlast four slices. The full three and four years + * (some 3.7 million readings, every collection, production slices, over + * twenty minutes) run when the variable is at least `1095`, which the + * nightly `integration-long.yml` workflow does. */ import v8 from "node:v8"; import vm from "node:vm"; @@ -48,8 +67,6 @@ const USER = "gh-multi-year-owner"; const MINUTE = 60_000; /** The newest reading: now, so the hourly sync's window lands on real data. */ const NOW = Math.floor(Date.now() / MINUTE) * MINUTE; -/** Three years of heart rate, one reading a minute. */ -const POINTS = 3 * 365 * 24 * 60; /** * How much the collected heap may grow over the second half of the history: @@ -64,6 +81,47 @@ const POINTS = 3 * 365 * 24 * 60; */ const GROWTH_BUDGET_BYTES = 16 * 1024 * 1024; +/** + * The run's size (see the header): the full three years with production + * slices when `HEALTHLOG_BACKFILL_TEST_DAYS` is at least 1095, else that + * many days (120 unless set) with 50-page slices and heart rate the only + * collection left to walk. + */ +const size = vi.hoisted(() => { + const raw = process.env.HEALTHLOG_BACKFILL_TEST_DAYS; + const days = raw && /^\d+$/.test(raw) && Number(raw) > 0 ? Number(raw) : 120; + const full = days >= 3 * 365; + return { days, full, slicePages: full ? 500 : 50 }; +}); + +/** Days of heart rate the first run walks; the second walks a third more. */ +const DAYS = size.days; +/** One reading a minute. */ +const POINTS = DAYS * 24 * 60; +/** Pages a slice reads, and the faults' places scaled to it. */ +const SLICE_PAGES = size.slicePages; +const SCALE = SLICE_PAGES / 500; + +vi.mock("@/lib/jobs/google-health-backfill", async (importOriginal) => { + const actual = + await importOriginal(); + if (size.slicePages === actual.GOOGLE_HEALTH_BACKFILL_SLICE_PAGES) { + return actual; + } + // The drain runs each slice with the production budget; a shorter run + // gives it shorter slices through the job's own test seam. + return { + ...actual, + runGoogleHealthBackfillForUser: ( + ...args: Parameters + ) => + actual.runGoogleHealthBackfillForUser(args[0], args[1], args[2], { + ...args[3], + slicePages: size.slicePages, + }), + }; +}); + const fake = vi.hoisted(() => ({ now: 0, total: 0, @@ -239,6 +297,19 @@ async function seed(points: number): Promise { accessToken: encrypt("access"), refreshToken: encrypt("refresh"), tokenExpiresAt: new Date(Date.now() + 7 * 86_400_000), + // A shorter run starts with every light collection walked, so its + // short first slice goes to heart rate (see the header). + ...(size.full + ? {} + : { + backfillProgress: { + done: ( + await import("@/lib/google-health/collection-keys") + ).GOOGLE_HEALTH_COLLECTION_KEYS.filter( + (k) => k !== "fetchHeartRate", + ), + }, + }), }, }); fake.now = NOW; @@ -284,7 +355,7 @@ function nextSlice(): AdmissionPayload | null { return next ? (next.data as AdmissionPayload) : null; } -describe("Google Health backfill over three years of heart rate (#1220)", () => { +describe(`Google Health backfill over ${DAYS} days of heart rate (#1220)`, () => { it("completes slice by slice, imports every reading once, and keeps the hourly sync running", async () => { const { drainIntegrationBackfillAdmission } = await import("@/lib/jobs/integration-backfill-drain"); @@ -292,15 +363,13 @@ describe("Google Health backfill over three years of heart rate (#1220)", () => await import("@/lib/jobs/reminder/google-health-sync"); const { GOOGLE_HEALTH_TRANSIENT_RETRY } = await import("@/lib/google-health/client"); - const { GOOGLE_HEALTH_BACKFILL_SLICE_PAGES } = - await import("@/lib/jobs/google-health-backfill"); GOOGLE_HEALTH_TRANSIENT_RETRY.baseDelayMs = 0; // A server error and a lost request in the first slice, and a rate limit // Google will not lift for an hour in the second. - fake.faults.set(40, "503"); - fake.faults.set(80, "timeout"); - fake.faults.set(GOOGLE_HEALTH_BACKFILL_SLICE_PAGES + 120, "429-long"); + fake.faults.set(Math.round(40 * SCALE), "503"); + fake.faults.set(Math.round(80 * SCALE), "timeout"); + fake.faults.set(SLICE_PAGES + Math.round(120 * SCALE), "429-long"); // Heap samples from inside the page requests, between two pages of the // work being measured, once the second slice is over (see the budget). @@ -309,7 +378,9 @@ describe("Google Health backfill over three years of heart rate (#1220)", () => let peak = 0; let samples = 0; fake.onRequest = () => { - if (!measuring || fake.heartRateRequests % 50 !== 0) return; + if (!measuring || fake.heartRateRequests % Math.round(50 * SCALE) !== 0) { + return; + } const live = liveHeapBytes(); if (anchor === null) { anchor = peak = live; @@ -431,15 +502,21 @@ describe("Google Health backfill over three years of heart rate (#1220)", () => `${mb(v8.getHeapStatistics().heap_size_limit)} MB, peak ${mb(peak)} MB)`, ); expect(samples).toBeGreaterThan(5); - expect(growth).toBeLessThan(GROWTH_BUDGET_BYTES); + // Only over the full history: a short run is still warming up (one + // measured 47 MB of growth over 120 days with nothing held), and a walk + // that held what it read would hold too little of a short history to + // stand out. On every run the walk's memory is pinned by + // google-health-dense-backfill-memory.test.ts. + if (size.full) expect(growth).toBeLessThan(GROWTH_BUDGET_BYTES); }, 3_600_000); it("completes when Google refuses the resume edge's upper bound, meeting the refusal once", async () => { - // Four years: the first slice reads without an edge, the second resumes - // with the bound and is refused, every later one reads without it. Long - // enough that passing over everything already written on every slice - // would read the history several times over. - const points = 4 * 365 * 24 * 60; + // A third longer than the first run (four years in the full run): the + // first slice reads without an edge, the second resumes with the bound + // and is refused, every later one reads without it. Long enough that + // passing over everything already written on every slice would read the + // history several times over. + const points = Math.round((DAYS * 4) / 3) * 24 * 60; await seed(points); fake.refuseUpperBound = true; @@ -449,8 +526,6 @@ describe("Google Health backfill over three years of heart rate (#1220)", () => await import("@/lib/jobs/reminder/google-health-sync"); const { GOOGLE_HEALTH_TRANSIENT_RETRY } = await import("@/lib/google-health/client"); - const { GOOGLE_HEALTH_BACKFILL_SLICE_PAGES } = - await import("@/lib/jobs/google-health-backfill"); GOOGLE_HEALTH_TRANSIENT_RETRY.baseDelayMs = 0; const prisma = getPrismaClient(); @@ -528,7 +603,7 @@ describe("Google Health backfill over three years of heart rate (#1220)", () => `history pages=${historyPages} slices=${slices.length}`, ); expect(fake.backfillRequests).toBeLessThanOrEqual( - historyPages + GOOGLE_HEALTH_BACKFILL_SLICE_PAGES + 2 * slices.length, + historyPages + SLICE_PAGES + 2 * slices.length, ); // The bound was asked for once, refused once, and never again: the