From e40de2a654cbdf59670c51a8a36168b436d33333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 17 Nov 2026 18:39:15 +0100 Subject: [PATCH] feat(apple-health): read ZIP64 exports and Health Records members The auxiliary member reader refused ZIP64 archives, which is what the Health app writes for a large export, so ECGs and workout routes were counted as failed exactly for the accounts with the most history. It now resolves the ZIP64 end record and the per-member extended sizes, with the same bounds as before. It also learns the clinical-records/ JSON members, and a stored (uncompressed) member is no longer returned as zeros: its bytes were the read buffer the reader wipes after use. --- .../__tests__/archive-stream-zip64.test.ts | 155 +++++++++++++++ .../apple-health/__tests__/helpers/zip64.ts | 103 ++++++++++ src/lib/apple-health/archive-stream.ts | 181 ++++++++++++++++-- src/lib/apple-health/route-import.ts | 4 +- 4 files changed, 425 insertions(+), 18 deletions(-) create mode 100644 src/lib/apple-health/__tests__/archive-stream-zip64.test.ts create mode 100644 src/lib/apple-health/__tests__/helpers/zip64.ts diff --git a/src/lib/apple-health/__tests__/archive-stream-zip64.test.ts b/src/lib/apple-health/__tests__/archive-stream-zip64.test.ts new file mode 100644 index 000000000..45cfe36c4 --- /dev/null +++ b/src/lib/apple-health/__tests__/archive-stream-zip64.test.ts @@ -0,0 +1,155 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + clinicalMemberNameFromReference, + isClinicalRecordJson, + streamAppleHealthClinicalMembers, + streamAppleHealthEcgMembers, + streamAppleHealthRouteMembers, +} from "../archive-stream"; +import { zip64Archive } from "./helpers/zip64"; + +const dirs: string[] = []; +afterEach(() => { + for (const dir of dirs.splice(0)) + rmSync(dir, { recursive: true, force: true }); +}); + +function write(bytes: Buffer): string { + const dir = mkdtempSync(join(tmpdir(), "healthlog-zip64-")); + dirs.push(dir); + const path = join(dir, "export.zip"); + writeFileSync(path, bytes); + return path; +} + +async function text(stream: AsyncIterable): Promise { + let out = ""; + for await (const chunk of stream) out += String(chunk); + return out; +} + +const LIMITS = { + maxMembers: 10, + maxMemberBytes: 4096, + maxCompressionRatio: 100, +}; + +const FILES = { + "apple_health_export/export.xml": "", + "apple_health_export/electrocardiograms/ecg_2025-01-01.csv": "Name,x\n", + "apple_health_export/workout-routes/route_1.gpx": "", + "apple_health_export/clinical-records/Observation-1.json": '{"id":"1"}', +}; + +describe("archive-stream on a ZIP64 archive", () => { + it("streams ECG, route and clinical members", async () => { + const path = write(zip64Archive(FILES)); + const ecg: string[] = []; + for await (const m of streamAppleHealthEcgMembers({ + archivePath: path, + limits: { ...LIMITS, maxEcgMembers: 5, maxTotalEcgBytes: 8192 }, + })) { + ecg.push(`${m.name}=${await text(m.stream)}`); + } + expect(ecg).toEqual([ + "apple_health_export/electrocardiograms/ecg_2025-01-01.csv=Name,x\n", + ]); + + const routes: string[] = []; + for await (const m of streamAppleHealthRouteMembers({ + archivePath: path, + limits: { ...LIMITS, maxRouteMembers: 5, maxTotalRouteBytes: 8192 }, + wanted: () => true, + })) { + routes.push(await text(m.stream)); + } + expect(routes).toEqual([""]); + + const clinical: string[] = []; + for await (const m of streamAppleHealthClinicalMembers({ + archivePath: path, + limits: { ...LIMITS, maxClinicalMembers: 5, maxTotalClinicalBytes: 8192 }, + wanted: () => true, + })) { + clinical.push(await text(m.stream)); + } + expect(clinical).toEqual(['{"id":"1"}']); + }); + + it("refuses ZIP64 sentinels without the locator", async () => { + const path = write(zip64Archive(FILES, { omitLocator: true })); + const run = async () => { + for await (const m of streamAppleHealthEcgMembers({ + archivePath: path, + limits: { ...LIMITS, maxEcgMembers: 5, maxTotalEcgBytes: 8192 }, + })) { + void m; + } + }; + await expect(run()).rejects.toThrow(/ZIP64 locator/); + }); + + it("refuses a ZIP64 member without its extra field", async () => { + const path = write(zip64Archive(FILES, { omitExtra: true })); + const run = async () => { + for await (const m of streamAppleHealthEcgMembers({ + archivePath: path, + limits: { ...LIMITS, maxEcgMembers: 5, maxTotalEcgBytes: 8192 }, + })) { + void m; + } + }; + await expect(run()).rejects.toThrow(/malformed ZIP64 member/); + }); + + it("still enforces the member count from the ZIP64 record", async () => { + const path = write(zip64Archive(FILES)); + const run = async () => { + for await (const m of streamAppleHealthEcgMembers({ + archivePath: path, + limits: { + ...LIMITS, + maxMembers: 3, + maxEcgMembers: 5, + maxTotalEcgBytes: 8192, + }, + })) { + void m; + } + }; + await expect(run()).rejects.toThrow(/member count limit/); + }); +}); + +describe("clinical member names", () => { + it("maps Apple's reference path onto the archive member", () => { + expect( + clinicalMemberNameFromReference("/clinical-records/Observation-A.json"), + ).toBe("apple_health_export/clinical-records/Observation-A.json"); + expect( + clinicalMemberNameFromReference("clinical-records/Observation-A.json"), + ).toBe("apple_health_export/clinical-records/Observation-A.json"); + }); + + it.each([ + "/clinical-records/../export.xml", + "/clinical-records/sub/Observation.json", + "/workout-routes/route.gpx", + "/clinical-records/Observation.txt", + ])("refuses %s", (path) => { + expect(clinicalMemberNameFromReference(path)).toBeNull(); + }); + + it("recognises only direct JSON children", () => { + expect( + isClinicalRecordJson("apple_health_export/clinical-records/a.JSON"), + ).toBe(true); + expect(isClinicalRecordJson("apple_health_export/clinical-records/")).toBe( + false, + ); + }); +}); diff --git a/src/lib/apple-health/__tests__/helpers/zip64.ts b/src/lib/apple-health/__tests__/helpers/zip64.ts new file mode 100644 index 000000000..ef6729f16 --- /dev/null +++ b/src/lib/apple-health/__tests__/helpers/zip64.ts @@ -0,0 +1,103 @@ +/** + * A minimal ZIP64 writer for tests: stored members, every size and offset + * moved into the ZIP64 extra field and the ZIP64 end record, the way a large + * Health app export is written. fflate writes classic ZIP only. + */ +export function zip64Archive( + files: Record, + options: { omitLocator?: boolean; omitExtra?: boolean } = {}, +): Buffer { + const locals: Buffer[] = []; + const centrals: Buffer[] = []; + let offset = 0; + for (const [name, body] of Object.entries(files)) { + const nameBytes = Buffer.from(name, "utf8"); + const data = Buffer.from(body, "utf8"); + const crc = crc32(data); + + const localExtra = Buffer.alloc(20); + localExtra.writeUInt16LE(0x0001, 0); + localExtra.writeUInt16LE(16, 2); + localExtra.writeBigUInt64LE(BigInt(data.length), 4); + localExtra.writeBigUInt64LE(BigInt(data.length), 12); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(45, 4); + local.writeUInt16LE(0, 6); + local.writeUInt16LE(0, 8); + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(0xffffffff, 18); + local.writeUInt32LE(0xffffffff, 22); + local.writeUInt16LE(nameBytes.length, 26); + local.writeUInt16LE(localExtra.length, 28); + const localRecord = Buffer.concat([local, nameBytes, localExtra, data]); + + const centralExtra = Buffer.alloc(28); + centralExtra.writeUInt16LE(0x0001, 0); + centralExtra.writeUInt16LE(24, 2); + centralExtra.writeBigUInt64LE(BigInt(data.length), 4); + centralExtra.writeBigUInt64LE(BigInt(data.length), 12); + centralExtra.writeBigUInt64LE(BigInt(offset), 20); + const extra = options.omitExtra ? Buffer.alloc(0) : centralExtra; + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(45, 4); + central.writeUInt16LE(45, 6); + central.writeUInt16LE(0, 8); + central.writeUInt16LE(0, 10); + central.writeUInt32LE(crc, 16); + central.writeUInt32LE(0xffffffff, 20); + central.writeUInt32LE(0xffffffff, 24); + central.writeUInt16LE(nameBytes.length, 28); + central.writeUInt16LE(extra.length, 30); + central.writeUInt32LE(0xffffffff, 42); + centrals.push(Buffer.concat([central, nameBytes, extra])); + + locals.push(localRecord); + offset += localRecord.length; + } + const centralDirectory = Buffer.concat(centrals); + const centralOffset = offset; + const count = Object.keys(files).length; + + const record = Buffer.alloc(56); + record.writeUInt32LE(0x06064b50, 0); + record.writeBigUInt64LE(44n, 4); + record.writeUInt16LE(45, 12); + record.writeUInt16LE(45, 14); + record.writeBigUInt64LE(BigInt(count), 24); + record.writeBigUInt64LE(BigInt(count), 32); + record.writeBigUInt64LE(BigInt(centralDirectory.length), 40); + record.writeBigUInt64LE(BigInt(centralOffset), 48); + + const locator = Buffer.alloc(20); + locator.writeUInt32LE(0x07064b50, 0); + locator.writeBigUInt64LE(BigInt(centralOffset + centralDirectory.length), 8); + locator.writeUInt32LE(1, 16); + + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(0xffff, 8); + eocd.writeUInt16LE(0xffff, 10); + eocd.writeUInt32LE(0xffffffff, 12); + eocd.writeUInt32LE(0xffffffff, 16); + + return Buffer.concat([ + ...locals, + centralDirectory, + record, + ...(options.omitLocator ? [] : [locator]), + eocd, + ]); +} + +function crc32(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = crc & 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1; + } + } + return (crc ^ 0xffffffff) >>> 0; +} diff --git a/src/lib/apple-health/archive-stream.ts b/src/lib/apple-health/archive-stream.ts index 7af968af6..1fc0461cd 100644 --- a/src/lib/apple-health/archive-stream.ts +++ b/src/lib/apple-health/archive-stream.ts @@ -1,6 +1,7 @@ /** - * Bounded ZIP reader for Apple Health's auxiliary members: the ECG CSVs and - * the workout-route GPX files. + * Bounded ZIP reader for Apple Health's auxiliary members: the ECG CSVs, the + * workout-route GPX files and the Health Records FHIR JSON files. Reads ZIP64 + * archives too, which is what the Health app writes for a large export. * * ZIP entry names are treated as untrusted identifiers only: no archive member * is ever materialised on disk. Central-directory metadata is checked before @@ -36,9 +37,13 @@ interface ZipEntry { const EOCD_SIGNATURE = 0x06054b50; const CENTRAL_SIGNATURE = 0x02014b50; const LOCAL_SIGNATURE = 0x04034b50; +const ZIP64_EOCD_SIGNATURE = 0x06064b50; +const ZIP64_LOCATOR_SIGNATURE = 0x07064b50; +const ZIP64_EXTRA_ID = 0x0001; const MAX_EOCD_SEARCH = 65_535 + 22; const ECG_PREFIX = "apple_health_export/electrocardiograms/"; const ROUTE_PREFIX = "apple_health_export/workout-routes/"; +const CLINICAL_PREFIX = "apple_health_export/clinical-records/"; /** * One family of auxiliary members: which names belong to it, the label its @@ -93,6 +98,48 @@ async function readExactly( return buffer; } +/** A 64-bit ZIP field as a number, refused past the safe-integer range. */ +function safeNumber(value: bigint): number { + if (value > BigInt(Number.MAX_SAFE_INTEGER)) { + throw archiveError("ZIP64 value is too large to address safely"); + } + return Number(value); +} + +/** + * The ZIP64 extended-information values a central-directory entry asked for. + * A sentinel without its value in the extra field is a malformed archive. + */ +function zip64ExtraValues( + extra: Buffer, + wanted: { uncompressed: boolean; compressed: boolean; offset: boolean }, +): { uncompressed?: number; compressed?: number; offset?: number } { + let cursor = 0; + while (cursor + 4 <= extra.length) { + const id = extra.readUInt16LE(cursor); + const size = extra.readUInt16LE(cursor + 2); + const end = cursor + 4 + size; + if (end > extra.length) break; + if (id === ZIP64_EXTRA_ID) { + const out: { + uncompressed?: number; + compressed?: number; + offset?: number; + } = {}; + let at = cursor + 4; + for (const field of ["uncompressed", "compressed", "offset"] as const) { + if (!wanted[field]) continue; + if (at + 8 > end) throw archiveError("malformed ZIP64 member"); + out[field] = safeNumber(extra.readBigUInt64LE(at)); + at += 8; + } + return out; + } + cursor = end; + } + throw archiveError("malformed ZIP64 member"); +} + function isUnsafeName(name: string): boolean { if ( name.length === 0 || @@ -138,6 +185,23 @@ export function routeMemberNameFromReference(path: string): string | null { return isWorkoutRouteGpx(name) ? name : null; } +/** A Health Records resource as the archive names it: `clinical-records/.json`. */ +export function isClinicalRecordJson(name: string): boolean { + return isDirectChild(name, CLINICAL_PREFIX, ".json"); +} + +/** + * The archive member a `` points at. + * Apple writes the path from the export root + * (`/clinical-records/Observation-….json`); the member carries the export + * folder in front. Null for anything that is not a direct clinical file. + */ +export function clinicalMemberNameFromReference(path: string): string | null { + const trimmed = path.startsWith("/") ? path.slice(1) : path; + const name = `apple_health_export/${trimmed}`; + return isClinicalRecordJson(name) ? name : null; +} + async function readCentralDirectory( handle: FileHandle, fileSize: number, @@ -160,18 +224,47 @@ async function readCentralDirectory( const disk = tail.readUInt16LE(eocdOffset + 4); const centralDisk = tail.readUInt16LE(eocdOffset + 6); const diskEntries = tail.readUInt16LE(eocdOffset + 8); - const totalEntries = tail.readUInt16LE(eocdOffset + 10); - const centralSize = tail.readUInt32LE(eocdOffset + 12); - const centralOffset = tail.readUInt32LE(eocdOffset + 16); + let totalEntries = tail.readUInt16LE(eocdOffset + 10); + let centralSize = tail.readUInt32LE(eocdOffset + 12); + let centralOffset = tail.readUInt32LE(eocdOffset + 16); + if (disk !== 0 || centralDisk !== 0 || diskEntries !== totalEntries) { + throw archiveError("unsupported multi-disk archive"); + } + // ZIP64: a sentinel in any of the three fields means the real values live + // in the ZIP64 end record, found through the locator just before the end + // record. The Health app writes one for a large export. if ( - disk !== 0 || - centralDisk !== 0 || - diskEntries !== totalEntries || totalEntries === 0xffff || centralSize === 0xffffffff || centralOffset === 0xffffffff ) { - throw archiveError("unsupported multi-disk or ZIP64 archive"); + const locatorOffset = fileSize - tailLength + eocdOffset - 20; + if (locatorOffset < 0) throw archiveError("missing ZIP64 locator"); + const locator = await readExactly(handle, 20, locatorOffset); + if (locator.readUInt32LE(0) !== ZIP64_LOCATOR_SIGNATURE) { + throw archiveError("missing ZIP64 locator"); + } + if (locator.readUInt32LE(4) !== 0 || locator.readUInt32LE(16) > 1) { + throw archiveError("unsupported multi-disk archive"); + } + const recordOffset = safeNumber(locator.readBigUInt64LE(8)); + if (recordOffset + 56 > fileSize) { + throw archiveError("invalid ZIP64 directory bounds"); + } + const record = await readExactly(handle, 56, recordOffset); + if (record.readUInt32LE(0) !== ZIP64_EOCD_SIGNATURE) { + throw archiveError("malformed ZIP64 directory"); + } + if (record.readUInt32LE(16) !== 0 || record.readUInt32LE(20) !== 0) { + throw archiveError("unsupported multi-disk archive"); + } + const recordDiskEntries = safeNumber(record.readBigUInt64LE(24)); + totalEntries = safeNumber(record.readBigUInt64LE(32)); + centralSize = safeNumber(record.readBigUInt64LE(40)); + centralOffset = safeNumber(record.readBigUInt64LE(48)); + if (recordDiskEntries !== totalEntries) { + throw archiveError("unsupported multi-disk archive"); + } } if (totalEntries > limits.maxMembers) { throw archiveError("member count limit exceeded"); @@ -192,21 +285,37 @@ async function readCentralDirectory( } const flags = header.readUInt16LE(8); const method = header.readUInt16LE(10); - const compressedSize = header.readUInt32LE(20); - const uncompressedSize = header.readUInt32LE(24); + let compressedSize = header.readUInt32LE(20); + let uncompressedSize = header.readUInt32LE(24); const nameLength = header.readUInt16LE(28); const extraLength = header.readUInt16LE(30); const commentLength = header.readUInt16LE(32); - const localHeaderOffset = header.readUInt32LE(42); + let localHeaderOffset = header.readUInt32LE(42); + + const nameBuffer = await readExactly(handle, nameLength, cursor + 46); if ( compressedSize === 0xffffffff || uncompressedSize === 0xffffffff || localHeaderOffset === 0xffffffff ) { - throw archiveError("unsupported ZIP64 member"); + // ZIP64 member: the real values sit in the extended-information extra + // field (id 0x0001), in this order, each present only when its 32-bit + // field holds the sentinel. + const extra = await readExactly( + handle, + extraLength, + cursor + 46 + nameLength, + ); + const sizes = zip64ExtraValues(extra, { + uncompressed: uncompressedSize === 0xffffffff, + compressed: compressedSize === 0xffffffff, + offset: localHeaderOffset === 0xffffffff, + }); + if (sizes.uncompressed !== undefined) + uncompressedSize = sizes.uncompressed; + if (sizes.compressed !== undefined) compressedSize = sizes.compressed; + if (sizes.offset !== undefined) localHeaderOffset = sizes.offset; } - - const nameBuffer = await readExactly(handle, nameLength, cursor + 46); const name = nameBuffer.toString("utf8"); if (isUnsafeName(name)) throw archiveError("unsafe member path"); if (names.has(name)) throw archiveError("duplicate member name"); @@ -284,9 +393,11 @@ async function decompressEntry( entry.compressedSize, dataOffset, ); + // A stored member is read through a copy: the read buffer is wiped below, + // and the stored bytes are the member's content. const source = entry.method === 0 - ? Readable.from([compressed]) + ? Readable.from([Buffer.from(compressed)]) : Readable.from([compressed]).pipe(createInflateRaw()); const chunks: Buffer[] = []; let memberBytes = 0; @@ -403,3 +514,41 @@ export async function* streamAppleHealthRouteMembers(input: { input.wanted, ); } + +/** Ceilings for the Health Records members; the shared ones keep their meaning. */ +export interface AppleHealthClinicalArchiveLimits { + maxMembers: number; + maxClinicalMembers: number; + maxMemberBytes: number; + maxTotalClinicalBytes: number; + maxCompressionRatio: number; +} + +/** + * Yield the Health Records JSON members `wanted` names, one bounded buffer at + * a time: the caller parses and drops each before the next is inflated. + */ +export async function* streamAppleHealthClinicalMembers(input: { + archivePath: string; + limits: AppleHealthClinicalArchiveLimits; + wanted: (name: string) => boolean; +}): AsyncGenerator { + const { limits } = input; + yield* streamMembers( + input.archivePath, + { + maxMembers: limits.maxMembers, + maxEcgMembers: limits.maxClinicalMembers, + maxMemberBytes: limits.maxMemberBytes, + maxTotalEcgBytes: limits.maxTotalClinicalBytes, + maxCompressionRatio: limits.maxCompressionRatio, + }, + { + label: "clinical record", + matches: isClinicalRecordJson, + maxMembers: limits.maxClinicalMembers, + maxTotalBytes: limits.maxTotalClinicalBytes, + }, + input.wanted, + ); +} diff --git a/src/lib/apple-health/route-import.ts b/src/lib/apple-health/route-import.ts index c392d2dc0..1a51e2c40 100644 --- a/src/lib/apple-health/route-import.ts +++ b/src/lib/apple-health/route-import.ts @@ -256,8 +256,8 @@ export async function importAppleHealthWorkoutRoutes(input: { } } } catch { - // The archive itself is unreadable for this pass (a ZIP64 export, a - // limit hit in the directory). Every referenced track not yet handled + // The archive itself is unreadable for this pass (a malformed directory, + // a limit hit in it). Every referenced track not yet handled // counts as failed; the import stays done. const handled = counts.attached + counts.skipped + counts.failed; counts.failed += Math.max(0, byMember.size - handled);