mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-19 08:12:52 +00:00
fix(apple-health): stop holding route and lab references past what a pass reads
The import collected every workout-route and Health Records reference the export's XML named until the parse ended, although the passes read at most 20,000 routes and 50,000 lab files. Past those ceilings a reference is now only counted, and the passes report the count as failed.
This commit is contained in:
1 parent
3572645544
commit
d1db4e59cb
7 files changed
+139
-15
No files matched your search
@@ -0,0 +1,25 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { boundedReferences } from "../bounded-references";
|
||||
|
||||
describe("boundedReferences", () => {
|
||||
it("keeps references up to the cap and counts the rest", () => {
|
||||
const list = boundedReferences<number>(3);
|
||||
for (let i = 0; i < 1_000; i += 1) list.push(i);
|
||||
expect(list.items).toEqual([0, 1, 2]);
|
||||
expect(list.overflow).toBe(997);
|
||||
expect(list.total).toBe(1_000);
|
||||
expect(list.handOver()).toEqual({ references: [0, 1, 2], overflow: 997 });
|
||||
});
|
||||
|
||||
it("forgets both once the pass has read them", () => {
|
||||
const list = boundedReferences<string>(2);
|
||||
list.push("a");
|
||||
list.push("b");
|
||||
list.push("c");
|
||||
list.clear();
|
||||
expect(list.items).toEqual([]);
|
||||
expect(list.overflow).toBe(0);
|
||||
expect(list.total).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -183,6 +183,18 @@ describe("importAppleHealthClinicalLabs", () => {
|
||||
expect(counts).toMatchObject({ discovered: 2, imported: 1, failed: 1 });
|
||||
});
|
||||
|
||||
it("counts references the job did not keep as refused", async () => {
|
||||
const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") });
|
||||
const counts = await importAppleHealthClinicalLabs({
|
||||
archivePath: path,
|
||||
userId: "user-1",
|
||||
tz: "Europe/Berlin",
|
||||
references: [ref("/clinical-records/Observation-a.json")],
|
||||
overflow: 4,
|
||||
});
|
||||
expect(counts).toMatchObject({ discovered: 5, imported: 1, failed: 4 });
|
||||
});
|
||||
|
||||
it("fails soft when the archive breaks a limit", async () => {
|
||||
const path = archive({ [`${ROOT}Observation-a.json`]: observation("a") });
|
||||
const counts = await run(
|
||||
|
||||
@@ -230,6 +230,21 @@ describe("importAppleHealthWorkoutRoutes", () => {
|
||||
expect(prisma.workoutRoute.createMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("counts references the job did not keep as failed", async () => {
|
||||
const prisma = fakePrisma({ id: "w1", elevationM: null, route: null });
|
||||
const counts = await importAppleHealthWorkoutRoutes({
|
||||
archivePath: archive(files),
|
||||
userId: "u1",
|
||||
references: [
|
||||
{ externalId: "ext-1", path: "/workout-routes/route_1.gpx" },
|
||||
],
|
||||
overflow: 3,
|
||||
prisma: prisma as never,
|
||||
});
|
||||
expect(counts.attached).toBe(1);
|
||||
expect(counts.failed).toBe(3);
|
||||
});
|
||||
|
||||
it("fails soft on a member over the byte cap", async () => {
|
||||
const prisma = fakePrisma({ id: "w1", elevationM: null, route: null });
|
||||
const counts = await importAppleHealthWorkoutRoutes({
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* The references an export's XML names for a later archive pass (workout
|
||||
* routes, Health Records files), kept up to what that pass reads.
|
||||
*
|
||||
* The XML parser hands over one reference per element, and an export can
|
||||
* name far more than the pass will ever read, each one held until the parse
|
||||
* is done. Past the cap a reference is only counted, so the pass can report
|
||||
* it as refused without the job having held it.
|
||||
*/
|
||||
export interface BoundedReferences<T> {
|
||||
/** The references kept, in the order they arrived. */
|
||||
readonly items: T[];
|
||||
/** How many arrived after the cap. */
|
||||
readonly overflow: number;
|
||||
/** Every reference that arrived. */
|
||||
readonly total: number;
|
||||
push(item: T): void;
|
||||
/** What the pass takes: the references kept and how many were not. */
|
||||
handOver(): { references: T[]; overflow: number };
|
||||
/** Drop what is held once the pass has read it. */
|
||||
clear(): void;
|
||||
}
|
||||
|
||||
export function boundedReferences<T>(cap: number): BoundedReferences<T> {
|
||||
const items: T[] = [];
|
||||
let overflow = 0;
|
||||
return {
|
||||
items,
|
||||
get overflow() {
|
||||
return overflow;
|
||||
},
|
||||
get total() {
|
||||
return items.length + overflow;
|
||||
},
|
||||
push(item) {
|
||||
if (items.length < cap) items.push(item);
|
||||
else overflow += 1;
|
||||
},
|
||||
handOver() {
|
||||
return { references: items, overflow };
|
||||
},
|
||||
clear() {
|
||||
items.length = 0;
|
||||
overflow = 0;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -82,6 +82,8 @@ export async function importAppleHealthClinicalLabs(input: {
|
||||
userId: string;
|
||||
tz: string;
|
||||
references: readonly ClinicalLabReference[];
|
||||
/** References the job counted past its cap and did not keep. */
|
||||
overflow?: number;
|
||||
prisma?: Pick<
|
||||
PrismaClient,
|
||||
"labResult" | "biomarker" | "biomarkerAlias" | "labResultImportKey"
|
||||
@@ -90,8 +92,9 @@ export async function importAppleHealthClinicalLabs(input: {
|
||||
}): Promise<FhirLabCounts> {
|
||||
const counts = emptyFhirLabCounts();
|
||||
const limits = input.limits ?? APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS;
|
||||
counts.discovered = input.references.length;
|
||||
if (input.references.length === 0) return counts;
|
||||
const dropped = input.overflow ?? 0;
|
||||
counts.discovered = input.references.length + dropped;
|
||||
if (counts.discovered === 0) return counts;
|
||||
|
||||
const tally: Record<string, number> = {};
|
||||
const note = (reason: string) => {
|
||||
@@ -99,9 +102,10 @@ export async function importAppleHealthClinicalLabs(input: {
|
||||
};
|
||||
|
||||
// More references than the archive may hold clinical files: the overflow
|
||||
// is refused, the rest is read.
|
||||
// is refused, the rest is read. The job stops keeping references at the
|
||||
// same ceiling and hands over how many it only counted.
|
||||
const usable = input.references.slice(0, limits.maxClinicalMembers);
|
||||
const overflow = input.references.length - usable.length;
|
||||
const overflow = input.references.length - usable.length + dropped;
|
||||
counts.failed += overflow;
|
||||
if (overflow > 0) tally.over_limit = overflow;
|
||||
|
||||
|
||||
@@ -181,6 +181,8 @@ export async function importAppleHealthWorkoutRoutes(input: {
|
||||
archivePath: string;
|
||||
userId: string;
|
||||
references: readonly WorkoutRouteReference[];
|
||||
/** References the job counted past its cap and did not keep. */
|
||||
overflow?: number;
|
||||
prisma: Pick<PrismaClient, "workout" | "workoutRoute">;
|
||||
limits?: AppleHealthRouteArchiveLimits;
|
||||
}): Promise<WorkoutRouteImportCounts> {
|
||||
@@ -188,7 +190,8 @@ export async function importAppleHealthWorkoutRoutes(input: {
|
||||
discovered: 0,
|
||||
attached: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
// A track the job did not keep is over the ceiling, never read.
|
||||
failed: input.overflow ?? 0,
|
||||
};
|
||||
const byMember = new Map<string, string>();
|
||||
for (const ref of input.references) {
|
||||
@@ -260,7 +263,8 @@ export async function importAppleHealthWorkoutRoutes(input: {
|
||||
// The archive itself is unreadable for this pass (a malformed directory,
|
||||
// a limit hit in it). Every referenced track not yet handled
|
||||
// counts as failed; the import stays done.
|
||||
const handled = counts.attached + counts.skipped + counts.failed;
|
||||
const handled =
|
||||
counts.attached + counts.skipped + counts.failed - (input.overflow ?? 0);
|
||||
counts.failed += Math.max(0, byMember.size - handled);
|
||||
}
|
||||
return counts;
|
||||
|
||||
@@ -30,7 +30,9 @@ import type { PrismaClient } from "@/generated/prisma/client";
|
||||
import { prisma, toJson } from "@/lib/db";
|
||||
import type { Job } from "pg-boss";
|
||||
|
||||
import { boundedReferences } from "@/lib/apple-health/bounded-references";
|
||||
import {
|
||||
APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS,
|
||||
importAppleHealthClinicalLabs,
|
||||
type AppleHealthImportJobResult,
|
||||
} from "@/lib/apple-health/clinical-labs-import";
|
||||
@@ -39,6 +41,7 @@ import { importAppleHealthArchiveEcgs } from "@/lib/apple-health/ecg-import";
|
||||
import { extractExportXml } from "@/lib/import/unzip-export-xml";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import {
|
||||
APPLE_HEALTH_ROUTE_ARCHIVE_LIMITS,
|
||||
importAppleHealthWorkoutRoutes,
|
||||
type WorkoutRouteReference,
|
||||
} from "@/lib/apple-health/route-import";
|
||||
@@ -311,6 +314,20 @@ async function writeProgress(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The route and Health Records references the XML names, kept up to what
|
||||
* each archive pass reads; past that a reference is only counted, so an
|
||||
* export naming far more files does not hold them all until the parse ends.
|
||||
*/
|
||||
const routeReferenceList = () =>
|
||||
boundedReferences<WorkoutRouteReference>(
|
||||
APPLE_HEALTH_ROUTE_ARCHIVE_LIMITS.maxRouteMembers,
|
||||
);
|
||||
const labReferenceList = () =>
|
||||
boundedReferences<ClinicalLabReference>(
|
||||
APPLE_HEALTH_CLINICAL_ARCHIVE_LIMITS.maxClinicalMembers,
|
||||
);
|
||||
|
||||
/**
|
||||
* Top-level handler. Thrown errors land the job in `failed` with the
|
||||
* error message recorded on `ImportJob.failureReason` and re-thrown
|
||||
@@ -400,8 +417,8 @@ export async function handleAppleHealthImport(
|
||||
});
|
||||
// GPS tracks are read from the archive only when the person keeps them.
|
||||
const keepRoutes = userRow?.storeWorkoutRoutes === true;
|
||||
const routeReferences: WorkoutRouteReference[] = [];
|
||||
const labReferences: ClinicalLabReference[] = [];
|
||||
const routeReferences = routeReferenceList();
|
||||
const labReferences = labReferenceList();
|
||||
const userTimezone =
|
||||
userRow?.timezone && userRow.timezone.length > 0
|
||||
? userRow.timezone
|
||||
@@ -493,38 +510,38 @@ export async function handleAppleHealthImport(
|
||||
// workouts are committed, a track that cannot be read is only counted.
|
||||
// Read one member at a time from the archive; nothing about a track
|
||||
// reaches the result but the count.
|
||||
if (routeReferences.length > 0) {
|
||||
if (routeReferences.total > 0) {
|
||||
const routes = await importAppleHealthWorkoutRoutes({
|
||||
archivePath: uploadPath,
|
||||
userId,
|
||||
references: routeReferences,
|
||||
...routeReferences.handOver(),
|
||||
prisma,
|
||||
});
|
||||
result.workouts.routesAttached = routes.attached;
|
||||
annotate({
|
||||
action: { name: "apple_health.import.workout_routes" },
|
||||
meta: {
|
||||
referenced: routeReferences.length,
|
||||
referenced: routeReferences.total,
|
||||
discovered: routes.discovered,
|
||||
attached: routes.attached,
|
||||
skipped: routes.skipped,
|
||||
failed: routes.failed,
|
||||
},
|
||||
});
|
||||
routeReferences.length = 0;
|
||||
routeReferences.clear();
|
||||
}
|
||||
|
||||
// Lab results from Health Records, one FHIR file at a time; only counts
|
||||
// reach the result.
|
||||
if (labReferences.length > 0) {
|
||||
if (labReferences.total > 0) {
|
||||
result.labs = await importAppleHealthClinicalLabs({
|
||||
archivePath: uploadPath,
|
||||
userId,
|
||||
tz: userTimezone,
|
||||
references: labReferences,
|
||||
...labReferences.handOver(),
|
||||
prisma,
|
||||
});
|
||||
labReferences.length = 0;
|
||||
labReferences.clear();
|
||||
}
|
||||
|
||||
// Done. Persist the terminal envelope.
|
||||
|
||||
Reference in new issue
Block a user