From d1db4e59cbfa4b6a0930fc7d41ee993802adbdf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 17 Nov 2026 21:24:39 +0100 Subject: [PATCH] fix(apple-health): stop holding route and lab references past what a pass reads The import collected every workout-route and Health Records reference the export's XML named until the parse ended, although the passes read at most 20,000 routes and 50,000 lab files. Past those ceilings a reference is now only counted, and the passes report the count as failed. --- .../__tests__/bounded-references.test.ts | 25 ++++++++++ .../__tests__/clinical-labs-import.test.ts | 12 +++++ .../__tests__/route-import.test.ts | 15 ++++++ src/lib/apple-health/bounded-references.ts | 47 +++++++++++++++++++ src/lib/apple-health/clinical-labs-import.ts | 12 +++-- src/lib/apple-health/route-import.ts | 8 +++- src/lib/jobs/apple-health-import-worker.ts | 35 ++++++++++---- 7 files changed, 139 insertions(+), 15 deletions(-) create mode 100644 src/lib/apple-health/__tests__/bounded-references.test.ts create mode 100644 src/lib/apple-health/bounded-references.ts diff --git a/src/lib/apple-health/__tests__/bounded-references.test.ts b/src/lib/apple-health/__tests__/bounded-references.test.ts new file mode 100644 index 000000000..669ea476b --- /dev/null +++ b/src/lib/apple-health/__tests__/bounded-references.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; + +import { boundedReferences } from "../bounded-references"; + +describe("boundedReferences", () => { + it("keeps references up to the cap and counts the rest", () => { + const list = boundedReferences(3); + for (let i = 0; i < 1_000; i += 1) list.push(i); + expect(list.items).toEqual([0, 1, 2]); + expect(list.overflow).toBe(997); + expect(list.total).toBe(1_000); + expect(list.handOver()).toEqual({ references: [0, 1, 2], overflow: 997 }); + }); + + it("forgets both once the pass has read them", () => { + const list = boundedReferences(2); + list.push("a"); + list.push("b"); + list.push("c"); + list.clear(); + expect(list.items).toEqual([]); + expect(list.overflow).toBe(0); + expect(list.total).toBe(0); + }); +}); diff --git a/src/lib/apple-health/__tests__/clinical-labs-import.test.ts b/src/lib/apple-health/__tests__/clinical-labs-import.test.ts index d9af8f6d5..d4a36e5d3 100644 --- a/src/lib/apple-health/__tests__/clinical-labs-import.test.ts +++ b/src/lib/apple-health/__tests__/clinical-labs-import.test.ts @@ -183,6 +183,18 @@ describe("importAppleHealthClinicalLabs", () => { expect(counts).toMatchObject({ discovered: 2, imported: 1, failed: 1 }); }); + it("counts references the job did not keep as refused", async () => { + const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") }); + const counts = await importAppleHealthClinicalLabs({ + archivePath: path, + userId: "user-1", + tz: "Europe/Berlin", + references: [ref("/clinical-records/Observation-a.json")], + overflow: 4, + }); + expect(counts).toMatchObject({ discovered: 5, imported: 1, failed: 4 }); + }); + it("fails soft when the archive breaks a limit", async () => { const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") }); const counts = await run( diff --git a/src/lib/apple-health/__tests__/route-import.test.ts b/src/lib/apple-health/__tests__/route-import.test.ts index e24f33fe6..e27439488 100644 --- a/src/lib/apple-health/__tests__/route-import.test.ts +++ b/src/lib/apple-health/__tests__/route-import.test.ts @@ -230,6 +230,21 @@ describe("importAppleHealthWorkoutRoutes", () => { expect(prisma.workoutRoute.createMany).not.toHaveBeenCalled(); }); + it("counts references the job did not keep as failed", async () => { + const prisma = fakePrisma({ id: "w1", elevationM: null, route: null }); + const counts = await importAppleHealthWorkoutRoutes({ + archivePath: archive(files), + userId: "u1", + references: [ + { externalId: "ext-1", path: "/workout-routes/route_1.gpx" }, + ], + overflow: 3, + prisma: prisma as never, + }); + expect(counts.attached).toBe(1); + expect(counts.failed).toBe(3); + }); + it("fails soft on a member over the byte cap", async () => { const prisma = fakePrisma({ id: "w1", elevationM: null, route: null }); const counts = await importAppleHealthWorkoutRoutes({ diff --git a/src/lib/apple-health/bounded-references.ts b/src/lib/apple-health/bounded-references.ts new file mode 100644 index 000000000..a713fcc2e --- /dev/null +++ b/src/lib/apple-health/bounded-references.ts @@ -0,0 +1,47 @@ +/** + * The references an export's XML names for a later archive pass (workout + * routes, Health Records files), kept up to what that pass reads. + * + * The XML parser hands over one reference per element, and an export can + * name far more than the pass will ever read, each one held until the parse + * is done. Past the cap a reference is only counted, so the pass can report + * it as refused without the job having held it. + */ +export interface BoundedReferences { + /** The references kept, in the order they arrived. */ + readonly items: T[]; + /** How many arrived after the cap. */ + readonly overflow: number; + /** Every reference that arrived. */ + readonly total: number; + push(item: T): void; + /** What the pass takes: the references kept and how many were not. */ + handOver(): { references: T[]; overflow: number }; + /** Drop what is held once the pass has read it. */ + clear(): void; +} + +export function boundedReferences(cap: number): BoundedReferences { + const items: T[] = []; + let overflow = 0; + return { + items, + get overflow() { + return overflow; + }, + get total() { + return items.length + overflow; + }, + push(item) { + if (items.length < cap) items.push(item); + else overflow += 1; + }, + handOver() { + return { references: items, overflow }; + }, + clear() { + items.length = 0; + overflow = 0; + }, + }; +} diff --git a/src/lib/apple-health/clinical-labs-import.ts b/src/lib/apple-health/clinical-labs-import.ts index 2d50252c3..e9a2843d2 100644 --- a/src/lib/apple-health/clinical-labs-import.ts +++ b/src/lib/apple-health/clinical-labs-import.ts @@ -82,6 +82,8 @@ export async function importAppleHealthClinicalLabs(input: { userId: string; tz: string; references: readonly ClinicalLabReference[]; + /** References the job counted past its cap and did not keep. */ + overflow?: number; prisma?: Pick< PrismaClient, "labResult" | "biomarker" | "biomarkerAlias" | "labResultImportKey" @@ -90,8 +92,9 @@ export async function importAppleHealthClinicalLabs(input: { }): Promise { const counts = emptyFhirLabCounts(); const limits = input.limits ?? APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS; - counts.discovered = input.references.length; - if (input.references.length === 0) return counts; + const dropped = input.overflow ?? 0; + counts.discovered = input.references.length + dropped; + if (counts.discovered === 0) return counts; const tally: Record = {}; const note = (reason: string) => { @@ -99,9 +102,10 @@ export async function importAppleHealthClinicalLabs(input: { }; // More references than the archive may hold clinical files: the overflow - // is refused, the rest is read. + // is refused, the rest is read. The job stops keeping references at the + // same ceiling and hands over how many it only counted. const usable = input.references.slice(0, limits.maxClinicalMembers); - const overflow = input.references.length - usable.length; + const overflow = input.references.length - usable.length + dropped; counts.failed += overflow; if (overflow > 0) tally.over_limit = overflow; diff --git a/src/lib/apple-health/route-import.ts b/src/lib/apple-health/route-import.ts index 19c5211d3..c0b676024 100644 --- a/src/lib/apple-health/route-import.ts +++ b/src/lib/apple-health/route-import.ts @@ -181,6 +181,8 @@ export async function importAppleHealthWorkoutRoutes(input: { archivePath: string; userId: string; references: readonly WorkoutRouteReference[]; + /** References the job counted past its cap and did not keep. */ + overflow?: number; prisma: Pick; limits?: AppleHealthRouteArchiveLimits; }): Promise { @@ -188,7 +190,8 @@ export async function importAppleHealthWorkoutRoutes(input: { discovered: 0, attached: 0, skipped: 0, - failed: 0, + // A track the job did not keep is over the ceiling, never read. + failed: input.overflow ?? 0, }; const byMember = new Map(); for (const ref of input.references) { @@ -260,7 +263,8 @@ export async function importAppleHealthWorkoutRoutes(input: { // The archive itself is unreadable for this pass (a malformed directory, // a limit hit in it). Every referenced track not yet handled // counts as failed; the import stays done. - const handled = counts.attached + counts.skipped + counts.failed; + const handled = + counts.attached + counts.skipped + counts.failed - (input.overflow ?? 0); counts.failed += Math.max(0, byMember.size - handled); } return counts; diff --git a/src/lib/jobs/apple-health-import-worker.ts b/src/lib/jobs/apple-health-import-worker.ts index c8a2986ec..a83a4de86 100644 --- a/src/lib/jobs/apple-health-import-worker.ts +++ b/src/lib/jobs/apple-health-import-worker.ts @@ -30,7 +30,9 @@ import type { PrismaClient } from "@/generated/prisma/client"; import { prisma, toJson } from "@/lib/db"; import type { Job } from "pg-boss"; +import { boundedReferences } from "@/lib/apple-health/bounded-references"; import { + APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS, importAppleHealthClinicalLabs, type AppleHealthImportJobResult, } from "@/lib/apple-health/clinical-labs-import"; @@ -39,6 +41,7 @@ import { importAppleHealthArchiveEcgs } from "@/lib/apple-health/ecg-import"; import { extractExportXml } from "@/lib/import/unzip-export-xml"; import { annotate } from "@/lib/logging/context"; import { + APPLE_HEALTH_ROUTE_ARCHIVE_LIMITS, importAppleHealthWorkoutRoutes, type WorkoutRouteReference, } from "@/lib/apple-health/route-import"; @@ -311,6 +314,20 @@ async function writeProgress( }); } +/** + * The route and Health Records references the XML names, kept up to what + * each archive pass reads; past that a reference is only counted, so an + * export naming far more files does not hold them all until the parse ends. + */ +const routeReferenceList = () => + boundedReferences( + APPLE_HEALTH_ROUTE_ARCHIVE_LIMITS.maxRouteMembers, + ); +const labReferenceList = () => + boundedReferences( + APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS.maxClinicalMembers, + ); + /** * Top-level handler. Thrown errors land the job in `failed` with the * error message recorded on `ImportJob.failureReason` and re-thrown @@ -400,8 +417,8 @@ export async function handleAppleHealthImport( }); // GPS tracks are read from the archive only when the person keeps them. const keepRoutes = userRow?.storeWorkoutRoutes === true; - const routeReferences: WorkoutRouteReference[] = []; - const labReferences: ClinicalLabReference[] = []; + const routeReferences = routeReferenceList(); + const labReferences = labReferenceList(); const userTimezone = userRow?.timezone && userRow.timezone.length > 0 ? userRow.timezone @@ -493,38 +510,38 @@ export async function handleAppleHealthImport( // workouts are committed, a track that cannot be read is only counted. // Read one member at a time from the archive; nothing about a track // reaches the result but the count. - if (routeReferences.length > 0) { + if (routeReferences.total > 0) { const routes = await importAppleHealthWorkoutRoutes({ archivePath: uploadPath, userId, - references: routeReferences, + ...routeReferences.handOver(), prisma, }); result.workouts.routesAttached = routes.attached; annotate({ action: { name: "apple_health.import.workout_routes" }, meta: { - referenced: routeReferences.length, + referenced: routeReferences.total, discovered: routes.discovered, attached: routes.attached, skipped: routes.skipped, failed: routes.failed, }, }); - routeReferences.length = 0; + routeReferences.clear(); } // Lab results from Health Records, one FHIR file at a time; only counts // reach the result. - if (labReferences.length > 0) { + if (labReferences.total > 0) { result.labs = await importAppleHealthClinicalLabs({ archivePath: uploadPath, userId, tz: userTimezone, - references: labReferences, + ...labReferences.handOver(), prisma, }); - labReferences.length = 0; + labReferences.clear(); } // Done. Persist the terminal envelope.