fix(apple-health): stop holding route and lab references past what a pass reads

The import collected every workout-route and Health Records reference
the export's XML named until the parse ended, although the passes read
at most 20,000 routes and 50,000 lab files. Past those ceilings a
reference is now only counted, and the passes report the count as
failed.
This commit is contained in:
Marc-André Bombeck committed 2026-11-17 21:24:39 +01:00
1 parent 3572645544
commit d1db4e59cb
7 files changed
+139 -15

No files matched your search

@@ -0,0 +1,25 @@
import { describe, expect, it } from "vitest";
import { boundedReferences } from "../bounded-references";
describe("boundedReferences", () => {
it("keeps references up to the cap and counts the rest", () => {
const list = boundedReferences<number>(3);
for (let i = 0; i < 1_000; i += 1) list.push(i);
expect(list.items).toEqual([0, 1, 2]);
expect(list.overflow).toBe(997);
expect(list.total).toBe(1_000);
expect(list.handOver()).toEqual({ references: [0, 1, 2], overflow: 997 });
});
it("forgets both once the pass has read them", () => {
const list = boundedReferences<string>(2);
list.push("a");
list.push("b");
list.push("c");
list.clear();
expect(list.items).toEqual([]);
expect(list.overflow).toBe(0);
expect(list.total).toBe(0);
});
});
@@ -183,6 +183,18 @@ describe("importAppleHealthClinicalLabs", () => {
expect(counts).toMatchObject({ discovered: 2, imported: 1, failed: 1 });
});
it("counts references the job did not keep as refused", async () => {
const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") });
const counts = await importAppleHealthClinicalLabs({
archivePath: path,
userId: "user-1",
tz: "Europe/Berlin",
references: [ref("/clinical-records/Observation-a.json")],
overflow: 4,
});
expect(counts).toMatchObject({ discovered: 5, imported: 1, failed: 4 });
});
it("fails soft when the archive breaks a limit", async () => {
const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") });
const counts = await run(
@@ -230,6 +230,21 @@ describe("importAppleHealthWorkoutRoutes", () => {
expect(prisma.workoutRoute.createMany).not.toHaveBeenCalled();
});
it("counts references the job did not keep as failed", async () => {
const prisma = fakePrisma({ id: "w1", elevationM: null, route: null });
const counts = await importAppleHealthWorkoutRoutes({
archivePath: archive(files),
userId: "u1",
references: [
{ externalId: "ext-1", path: "/workout-routes/route_1.gpx" },
],
overflow: 3,
prisma: prisma as never,
});
expect(counts.attached).toBe(1);
expect(counts.failed).toBe(3);
});
it("fails soft on a member over the byte cap", async () => {
const prisma = fakePrisma({ id: "w1", elevationM: null, route: null });
const counts = await importAppleHealthWorkoutRoutes({
@@ -0,0 +1,47 @@
/**
* The references an export's XML names for a later archive pass (workout
* routes, Health Records files), kept up to what that pass reads.
*
* The XML parser hands over one reference per element, and an export can
* name far more than the pass will ever read, each one held until the parse
* is done. Past the cap a reference is only counted, so the pass can report
* it as refused without the job having held it.
*/
export interface BoundedReferences<T> {
/** The references kept, in the order they arrived. */
readonly items: T[];
/** How many arrived after the cap. */
readonly overflow: number;
/** Every reference that arrived. */
readonly total: number;
push(item: T): void;
/** What the pass takes: the references kept and how many were not. */
handOver(): { references: T[]; overflow: number };
/** Drop what is held once the pass has read it. */
clear(): void;
}
export function boundedReferences<T>(cap: number): BoundedReferences<T> {
const items: T[] = [];
let overflow = 0;
return {
items,
get overflow() {
return overflow;
},
get total() {
return items.length + overflow;
},
push(item) {
if (items.length < cap) items.push(item);
else overflow += 1;
},
handOver() {
return { references: items, overflow };
},
clear() {
items.length = 0;
overflow = 0;
},
};
}
+8 -4
View File
@@ -82,6 +82,8 @@ export async function importAppleHealthClinicalLabs(input: {
userId: string;
tz: string;
references: readonly ClinicalLabReference[];
/** References the job counted past its cap and did not keep. */
overflow?: number;
prisma?: Pick<
PrismaClient,
"labResult" | "biomarker" | "biomarkerAlias" | "labResultImportKey"
@@ -90,8 +92,9 @@ export async function importAppleHealthClinicalLabs(input: {
}): Promise<FhirLabCounts> {
const counts = emptyFhirLabCounts();
const limits = input.limits ?? APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS;
counts.discovered = input.references.length;
if (input.references.length === 0) return counts;
const dropped = input.overflow ?? 0;
counts.discovered = input.references.length + dropped;
if (counts.discovered === 0) return counts;
const tally: Record<string, number> = {};
const note = (reason: string) => {
@@ -99,9 +102,10 @@ export async function importAppleHealthClinicalLabs(input: {
};
// More references than the archive may hold clinical files: the overflow
// is refused, the rest is read.
// is refused, the rest is read. The job stops keeping references at the
// same ceiling and hands over how many it only counted.
const usable = input.references.slice(0, limits.maxClinicalMembers);
const overflow = input.references.length - usable.length;
const overflow = input.references.length - usable.length + dropped;
counts.failed += overflow;
if (overflow > 0) tally.over_limit = overflow;
+6 -2
View File
@@ -181,6 +181,8 @@ export async function importAppleHealthWorkoutRoutes(input: {
archivePath: string;
userId: string;
references: readonly WorkoutRouteReference[];
/** References the job counted past its cap and did not keep. */
overflow?: number;
prisma: Pick<PrismaClient, "workout" | "workoutRoute">;
limits?: AppleHealthRouteArchiveLimits;
}): Promise<WorkoutRouteImportCounts> {
@@ -188,7 +190,8 @@ export async function importAppleHealthWorkoutRoutes(input: {
discovered: 0,
attached: 0,
skipped: 0,
failed: 0,
// A track the job did not keep is over the ceiling, never read.
failed: input.overflow ?? 0,
};
const byMember = new Map<string, string>();
for (const ref of input.references) {
@@ -260,7 +263,8 @@ export async function importAppleHealthWorkoutRoutes(input: {
// The archive itself is unreadable for this pass (a malformed directory,
// a limit hit in it). Every referenced track not yet handled
// counts as failed; the import stays done.
const handled = counts.attached + counts.skipped + counts.failed;
const handled =
counts.attached + counts.skipped + counts.failed - (input.overflow ?? 0);
counts.failed += Math.max(0, byMember.size - handled);
}
return counts;
+26 -9
View File
@@ -30,7 +30,9 @@ import type { PrismaClient } from "@/generated/prisma/client";
import { prisma, toJson } from "@/lib/db";
import type { Job } from "pg-boss";
import { boundedReferences } from "@/lib/apple-health/bounded-references";
import {
APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS,
importAppleHealthClinicalLabs,
type AppleHealthImportJobResult,
} from "@/lib/apple-health/clinical-labs-import";
@@ -39,6 +41,7 @@ import { importAppleHealthArchiveEcgs } from "@/lib/apple-health/ecg-import";
import { extractExportXml } from "@/lib/import/unzip-export-xml";
import { annotate } from "@/lib/logging/context";
import {
APPLE_HEALTH_ROUTE_ARCHIVE_LIMITS,
importAppleHealthWorkoutRoutes,
type WorkoutRouteReference,
} from "@/lib/apple-health/route-import";
@@ -311,6 +314,20 @@ async function writeProgress(
});
}
/**
* The route and Health Records references the XML names, kept up to what
* each archive pass reads; past that a reference is only counted, so an
* export naming far more files does not hold them all until the parse ends.
*/
const routeReferenceList = () =>
boundedReferences<WorkoutRouteReference>(
APPLE_HEALTH_ROUTE_ARCHIVE_LIMITS.maxRouteMembers,
);
const labReferenceList = () =>
boundedReferences<ClinicalLabReference>(
APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS.maxClinicalMembers,
);
/**
* Top-level handler. Thrown errors land the job in `failed` with the
* error message recorded on `ImportJob.failureReason` and re-thrown
@@ -400,8 +417,8 @@ export async function handleAppleHealthImport(
});
// GPS tracks are read from the archive only when the person keeps them.
const keepRoutes = userRow?.storeWorkoutRoutes === true;
const routeReferences: WorkoutRouteReference[] = [];
const labReferences: ClinicalLabReference[] = [];
const routeReferences = routeReferenceList();
const labReferences = labReferenceList();
const userTimezone =
userRow?.timezone && userRow.timezone.length > 0
? userRow.timezone
@@ -493,38 +510,38 @@ export async function handleAppleHealthImport(
// workouts are committed, a track that cannot be read is only counted.
// Read one member at a time from the archive; nothing about a track
// reaches the result but the count.
if (routeReferences.length > 0) {
if (routeReferences.total > 0) {
const routes = await importAppleHealthWorkoutRoutes({
archivePath: uploadPath,
userId,
references: routeReferences,
...routeReferences.handOver(),
prisma,
});
result.workouts.routesAttached = routes.attached;
annotate({
action: { name: "apple_health.import.workout_routes" },
meta: {
referenced: routeReferences.length,
referenced: routeReferences.total,
discovered: routes.discovered,
attached: routes.attached,
skipped: routes.skipped,
failed: routes.failed,
},
});
routeReferences.length = 0;
routeReferences.clear();
}
// Lab results from Health Records, one FHIR file at a time; only counts
// reach the result.
if (labReferences.length > 0) {
if (labReferences.total > 0) {
result.labs = await importAppleHealthClinicalLabs({
archivePath: uploadPath,
userId,
tz: userTimezone,
references: labReferences,
...labReferences.handOver(),
prisma,
});
labReferences.length = 0;
labReferences.clear();
}
// Done. Persist the terminal envelope.