feat(apple-health): read ZIP64 exports and Health Records members

The auxiliary member reader refused ZIP64 archives, which is what the
Health app writes for a large export, so ECGs and workout routes were
counted as failed exactly for the accounts with the most history. It
now resolves the ZIP64 end record and the per-member extended sizes,
with the same bounds as before.

It also learns the clinical-records/ JSON members, and a stored
(uncompressed) member is no longer returned as zeros: its bytes were
the read buffer the reader wipes after use.
This commit is contained in:
Marc-André Bombeck committed 2026-11-17 18:39:15 +01:00
1 parent 5994762c74
commit e40de2a654
4 files changed
+425 -18

No files matched your search

@@ -0,0 +1,155 @@
import { afterEach, describe, expect, it } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
clinicalMemberNameFromReference,
isClinicalRecordJson,
streamAppleHealthClinicalMembers,
streamAppleHealthEcgMembers,
streamAppleHealthRouteMembers,
} from "../archive-stream";
import { zip64Archive } from "./helpers/zip64";
const dirs: string[] = [];
afterEach(() => {
for (const dir of dirs.splice(0))
rmSync(dir, { recursive: true, force: true });
});
function write(bytes: Buffer): string {
const dir = mkdtempSync(join(tmpdir(), "healthlog-zip64-"));
dirs.push(dir);
const path = join(dir, "export.zip");
writeFileSync(path, bytes);
return path;
}
async function text(stream: AsyncIterable<unknown>): Promise<string> {
let out = "";
for await (const chunk of stream) out += String(chunk);
return out;
}
const LIMITS = {
maxMembers: 10,
maxMemberBytes: 4096,
maxCompressionRatio: 100,
};
const FILES = {
"apple_health_export/export.xml": "<HealthData/>",
"apple_health_export/electrocardiograms/ecg_2025-01-01.csv": "Name,x\n",
"apple_health_export/workout-routes/route_1.gpx": "<gpx/>",
"apple_health_export/clinical-records/Observation-1.json": '{"id":"1"}',
};
describe("archive-stream on a ZIP64 archive", () => {
it("streams ECG, route and clinical members", async () => {
const path = write(zip64Archive(FILES));
const ecg: string[] = [];
for await (const m of streamAppleHealthEcgMembers({
archivePath: path,
limits: { ...LIMITS, maxEcgMembers: 5, maxTotalEcgBytes: 8192 },
})) {
ecg.push(`${m.name}=${await text(m.stream)}`);
}
expect(ecg).toEqual([
"apple_health_export/electrocardiograms/ecg_2025-01-01.csv=Name,x\n",
]);
const routes: string[] = [];
for await (const m of streamAppleHealthRouteMembers({
archivePath: path,
limits: { ...LIMITS, maxRouteMembers: 5, maxTotalRouteBytes: 8192 },
wanted: () => true,
})) {
routes.push(await text(m.stream));
}
expect(routes).toEqual(["<gpx/>"]);
const clinical: string[] = [];
for await (const m of streamAppleHealthClinicalMembers({
archivePath: path,
limits: { ...LIMITS, maxClinicalMembers: 5, maxTotalClinicalBytes: 8192 },
wanted: () => true,
})) {
clinical.push(await text(m.stream));
}
expect(clinical).toEqual(['{"id":"1"}']);
});
it("refuses ZIP64 sentinels without the locator", async () => {
const path = write(zip64Archive(FILES, { omitLocator: true }));
const run = async () => {
for await (const m of streamAppleHealthEcgMembers({
archivePath: path,
limits: { ...LIMITS, maxEcgMembers: 5, maxTotalEcgBytes: 8192 },
})) {
void m;
}
};
await expect(run()).rejects.toThrow(/ZIP64 locator/);
});
it("refuses a ZIP64 member without its extra field", async () => {
const path = write(zip64Archive(FILES, { omitExtra: true }));
const run = async () => {
for await (const m of streamAppleHealthEcgMembers({
archivePath: path,
limits: { ...LIMITS, maxEcgMembers: 5, maxTotalEcgBytes: 8192 },
})) {
void m;
}
};
await expect(run()).rejects.toThrow(/malformed ZIP64 member/);
});
it("still enforces the member count from the ZIP64 record", async () => {
const path = write(zip64Archive(FILES));
const run = async () => {
for await (const m of streamAppleHealthEcgMembers({
archivePath: path,
limits: {
...LIMITS,
maxMembers: 3,
maxEcgMembers: 5,
maxTotalEcgBytes: 8192,
},
})) {
void m;
}
};
await expect(run()).rejects.toThrow(/member count limit/);
});
});
describe("clinical member names", () => {
it("maps Apple's reference path onto the archive member", () => {
expect(
clinicalMemberNameFromReference("/clinical-records/Observation-A.json"),
).toBe("apple_health_export/clinical-records/Observation-A.json");
expect(
clinicalMemberNameFromReference("clinical-records/Observation-A.json"),
).toBe("apple_health_export/clinical-records/Observation-A.json");
});
it.each([
"/clinical-records/../export.xml",
"/clinical-records/sub/Observation.json",
"/workout-routes/route.gpx",
"/clinical-records/Observation.txt",
])("refuses %s", (path) => {
expect(clinicalMemberNameFromReference(path)).toBeNull();
});
it("recognises only direct JSON children", () => {
expect(
isClinicalRecordJson("apple_health_export/clinical-records/a.JSON"),
).toBe(true);
expect(isClinicalRecordJson("apple_health_export/clinical-records/")).toBe(
false,
);
});
});
@@ -0,0 +1,103 @@
/**
* A minimal ZIP64 writer for tests: stored members, every size and offset
* moved into the ZIP64 extra field and the ZIP64 end record, the way a large
* Health app export is written. fflate writes classic ZIP only.
*/
export function zip64Archive(
files: Record<string, string>,
options: { omitLocator?: boolean; omitExtra?: boolean } = {},
): Buffer {
const locals: Buffer[] = [];
const centrals: Buffer[] = [];
let offset = 0;
for (const [name, body] of Object.entries(files)) {
const nameBytes = Buffer.from(name, "utf8");
const data = Buffer.from(body, "utf8");
const crc = crc32(data);
const localExtra = Buffer.alloc(20);
localExtra.writeUInt16LE(0x0001, 0);
localExtra.writeUInt16LE(16, 2);
localExtra.writeBigUInt64LE(BigInt(data.length), 4);
localExtra.writeBigUInt64LE(BigInt(data.length), 12);
const local = Buffer.alloc(30);
local.writeUInt32LE(0x04034b50, 0);
local.writeUInt16LE(45, 4);
local.writeUInt16LE(0, 6);
local.writeUInt16LE(0, 8);
local.writeUInt32LE(crc, 14);
local.writeUInt32LE(0xffffffff, 18);
local.writeUInt32LE(0xffffffff, 22);
local.writeUInt16LE(nameBytes.length, 26);
local.writeUInt16LE(localExtra.length, 28);
const localRecord = Buffer.concat([local, nameBytes, localExtra, data]);
const centralExtra = Buffer.alloc(28);
centralExtra.writeUInt16LE(0x0001, 0);
centralExtra.writeUInt16LE(24, 2);
centralExtra.writeBigUInt64LE(BigInt(data.length), 4);
centralExtra.writeBigUInt64LE(BigInt(data.length), 12);
centralExtra.writeBigUInt64LE(BigInt(offset), 20);
const extra = options.omitExtra ? Buffer.alloc(0) : centralExtra;
const central = Buffer.alloc(46);
central.writeUInt32LE(0x02014b50, 0);
central.writeUInt16LE(45, 4);
central.writeUInt16LE(45, 6);
central.writeUInt16LE(0, 8);
central.writeUInt16LE(0, 10);
central.writeUInt32LE(crc, 16);
central.writeUInt32LE(0xffffffff, 20);
central.writeUInt32LE(0xffffffff, 24);
central.writeUInt16LE(nameBytes.length, 28);
central.writeUInt16LE(extra.length, 30);
central.writeUInt32LE(0xffffffff, 42);
centrals.push(Buffer.concat([central, nameBytes, extra]));
locals.push(localRecord);
offset += localRecord.length;
}
const centralDirectory = Buffer.concat(centrals);
const centralOffset = offset;
const count = Object.keys(files).length;
const record = Buffer.alloc(56);
record.writeUInt32LE(0x06064b50, 0);
record.writeBigUInt64LE(44n, 4);
record.writeUInt16LE(45, 12);
record.writeUInt16LE(45, 14);
record.writeBigUInt64LE(BigInt(count), 24);
record.writeBigUInt64LE(BigInt(count), 32);
record.writeBigUInt64LE(BigInt(centralDirectory.length), 40);
record.writeBigUInt64LE(BigInt(centralOffset), 48);
const locator = Buffer.alloc(20);
locator.writeUInt32LE(0x07064b50, 0);
locator.writeBigUInt64LE(BigInt(centralOffset + centralDirectory.length), 8);
locator.writeUInt32LE(1, 16);
const eocd = Buffer.alloc(22);
eocd.writeUInt32LE(0x06054b50, 0);
eocd.writeUInt16LE(0xffff, 8);
eocd.writeUInt16LE(0xffff, 10);
eocd.writeUInt32LE(0xffffffff, 12);
eocd.writeUInt32LE(0xffffffff, 16);
return Buffer.concat([
...locals,
centralDirectory,
record,
...(options.omitLocator ? [] : [locator]),
eocd,
]);
}
function crc32(data: Buffer): number {
let crc = 0xffffffff;
for (const byte of data) {
crc ^= byte;
for (let bit = 0; bit < 8; bit += 1) {
crc = crc & 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
}
}
return (crc ^ 0xffffffff) >>> 0;
}
+165 -16
View File
@@ -1,6 +1,7 @@
/**
* Bounded ZIP reader for Apple Health's auxiliary members: the ECG CSVs and
* the workout-route GPX files.
* Bounded ZIP reader for Apple Health's auxiliary members: the ECG CSVs, the
* workout-route GPX files and the Health Records FHIR JSON files. Reads ZIP64
* archives too, which is what the Health app writes for a large export.
*
* ZIP entry names are treated as untrusted identifiers only: no archive member
* is ever materialised on disk. Central-directory metadata is checked before
@@ -36,9 +37,13 @@ interface ZipEntry {
const EOCD_SIGNATURE = 0x06054b50;
const CENTRAL_SIGNATURE = 0x02014b50;
const LOCAL_SIGNATURE = 0x04034b50;
const ZIP64_EOCD_SIGNATURE = 0x06064b50;
const ZIP64_LOCATOR_SIGNATURE = 0x07064b50;
const ZIP64_EXTRA_ID = 0x0001;
const MAX_EOCD_SEARCH = 65_535 + 22;
const ECG_PREFIX = "apple_health_export/electrocardiograms/";
const ROUTE_PREFIX = "apple_health_export/workout-routes/";
const CLINICAL_PREFIX = "apple_health_export/clinical-records/";
/**
* One family of auxiliary members: which names belong to it, the label its
@@ -93,6 +98,48 @@ async function readExactly(
return buffer;
}
/** A 64-bit ZIP field as a number, refused past the safe-integer range. */
function safeNumber(value: bigint): number {
if (value > BigInt(Number.MAX_SAFE_INTEGER)) {
throw archiveError("ZIP64 value is too large to address safely");
}
return Number(value);
}
/**
* The ZIP64 extended-information values a central-directory entry asked for.
* A sentinel without its value in the extra field is a malformed archive.
*/
function zip64ExtraValues(
extra: Buffer,
wanted: { uncompressed: boolean; compressed: boolean; offset: boolean },
): { uncompressed?: number; compressed?: number; offset?: number } {
let cursor = 0;
while (cursor + 4 <= extra.length) {
const id = extra.readUInt16LE(cursor);
const size = extra.readUInt16LE(cursor + 2);
const end = cursor + 4 + size;
if (end > extra.length) break;
if (id === ZIP64_EXTRA_ID) {
const out: {
uncompressed?: number;
compressed?: number;
offset?: number;
} = {};
let at = cursor + 4;
for (const field of ["uncompressed", "compressed", "offset"] as const) {
if (!wanted[field]) continue;
if (at + 8 > end) throw archiveError("malformed ZIP64 member");
out[field] = safeNumber(extra.readBigUInt64LE(at));
at += 8;
}
return out;
}
cursor = end;
}
throw archiveError("malformed ZIP64 member");
}
function isUnsafeName(name: string): boolean {
if (
name.length === 0 ||
@@ -138,6 +185,23 @@ export function routeMemberNameFromReference(path: string): string | null {
return isWorkoutRouteGpx(name) ? name : null;
}
/** A Health Records resource as the archive names it: `clinical-records/<file>.json`. */
export function isClinicalRecordJson(name: string): boolean {
return isDirectChild(name, CLINICAL_PREFIX, ".json");
}
/**
* The archive member a `<ClinicalRecord resourceFilePath="…">` points at.
* Apple writes the path from the export root
* (`/clinical-records/Observation-….json`); the member carries the export
* folder in front. Null for anything that is not a direct clinical file.
*/
export function clinicalMemberNameFromReference(path: string): string | null {
const trimmed = path.startsWith("/") ? path.slice(1) : path;
const name = `apple_health_export/${trimmed}`;
return isClinicalRecordJson(name) ? name : null;
}
async function readCentralDirectory(
handle: FileHandle,
fileSize: number,
@@ -160,18 +224,47 @@ async function readCentralDirectory(
const disk = tail.readUInt16LE(eocdOffset + 4);
const centralDisk = tail.readUInt16LE(eocdOffset + 6);
const diskEntries = tail.readUInt16LE(eocdOffset + 8);
const totalEntries = tail.readUInt16LE(eocdOffset + 10);
const centralSize = tail.readUInt32LE(eocdOffset + 12);
const centralOffset = tail.readUInt32LE(eocdOffset + 16);
let totalEntries = tail.readUInt16LE(eocdOffset + 10);
let centralSize = tail.readUInt32LE(eocdOffset + 12);
let centralOffset = tail.readUInt32LE(eocdOffset + 16);
if (disk !== 0 || centralDisk !== 0 || diskEntries !== totalEntries) {
throw archiveError("unsupported multi-disk archive");
}
// ZIP64: a sentinel in any of the three fields means the real values live
// in the ZIP64 end record, found through the locator just before the end
// record. The Health app writes one for a large export.
if (
disk !== 0 ||
centralDisk !== 0 ||
diskEntries !== totalEntries ||
totalEntries === 0xffff ||
centralSize === 0xffffffff ||
centralOffset === 0xffffffff
) {
throw archiveError("unsupported multi-disk or ZIP64 archive");
const locatorOffset = fileSize - tailLength + eocdOffset - 20;
if (locatorOffset < 0) throw archiveError("missing ZIP64 locator");
const locator = await readExactly(handle, 20, locatorOffset);
if (locator.readUInt32LE(0) !== ZIP64_LOCATOR_SIGNATURE) {
throw archiveError("missing ZIP64 locator");
}
if (locator.readUInt32LE(4) !== 0 || locator.readUInt32LE(16) > 1) {
throw archiveError("unsupported multi-disk archive");
}
const recordOffset = safeNumber(locator.readBigUInt64LE(8));
if (recordOffset + 56 > fileSize) {
throw archiveError("invalid ZIP64 directory bounds");
}
const record = await readExactly(handle, 56, recordOffset);
if (record.readUInt32LE(0) !== ZIP64_EOCD_SIGNATURE) {
throw archiveError("malformed ZIP64 directory");
}
if (record.readUInt32LE(16) !== 0 || record.readUInt32LE(20) !== 0) {
throw archiveError("unsupported multi-disk archive");
}
const recordDiskEntries = safeNumber(record.readBigUInt64LE(24));
totalEntries = safeNumber(record.readBigUInt64LE(32));
centralSize = safeNumber(record.readBigUInt64LE(40));
centralOffset = safeNumber(record.readBigUInt64LE(48));
if (recordDiskEntries !== totalEntries) {
throw archiveError("unsupported multi-disk archive");
}
}
if (totalEntries > limits.maxMembers) {
throw archiveError("member count limit exceeded");
@@ -192,21 +285,37 @@ async function readCentralDirectory(
}
const flags = header.readUInt16LE(8);
const method = header.readUInt16LE(10);
const compressedSize = header.readUInt32LE(20);
const uncompressedSize = header.readUInt32LE(24);
let compressedSize = header.readUInt32LE(20);
let uncompressedSize = header.readUInt32LE(24);
const nameLength = header.readUInt16LE(28);
const extraLength = header.readUInt16LE(30);
const commentLength = header.readUInt16LE(32);
const localHeaderOffset = header.readUInt32LE(42);
let localHeaderOffset = header.readUInt32LE(42);
const nameBuffer = await readExactly(handle, nameLength, cursor + 46);
if (
compressedSize === 0xffffffff ||
uncompressedSize === 0xffffffff ||
localHeaderOffset === 0xffffffff
) {
throw archiveError("unsupported ZIP64 member");
// ZIP64 member: the real values sit in the extended-information extra
// field (id 0x0001), in this order, each present only when its 32-bit
// field holds the sentinel.
const extra = await readExactly(
handle,
extraLength,
cursor + 46 + nameLength,
);
const sizes = zip64ExtraValues(extra, {
uncompressed: uncompressedSize === 0xffffffff,
compressed: compressedSize === 0xffffffff,
offset: localHeaderOffset === 0xffffffff,
});
if (sizes.uncompressed !== undefined)
uncompressedSize = sizes.uncompressed;
if (sizes.compressed !== undefined) compressedSize = sizes.compressed;
if (sizes.offset !== undefined) localHeaderOffset = sizes.offset;
}
const nameBuffer = await readExactly(handle, nameLength, cursor + 46);
const name = nameBuffer.toString("utf8");
if (isUnsafeName(name)) throw archiveError("unsafe member path");
if (names.has(name)) throw archiveError("duplicate member name");
@@ -284,9 +393,11 @@ async function decompressEntry(
entry.compressedSize,
dataOffset,
);
// A stored member is read through a copy: the read buffer is wiped below,
// and the stored bytes are the member's content.
const source =
entry.method === 0
? Readable.from([compressed])
? Readable.from([Buffer.from(compressed)])
: Readable.from([compressed]).pipe(createInflateRaw());
const chunks: Buffer[] = [];
let memberBytes = 0;
@@ -403,3 +514,41 @@ export async function* streamAppleHealthRouteMembers(input: {
input.wanted,
);
}
/** Ceilings for the Health Records members; the shared ones keep their meaning. */
export interface AppleHealthClinicalArchiveLimits {
maxMembers: number;
maxClinicalMembers: number;
maxMemberBytes: number;
maxTotalClinicalBytes: number;
maxCompressionRatio: number;
}
/**
* Yield the Health Records JSON members `wanted` names, one bounded buffer at
* a time: the caller parses and drops each before the next is inflated.
*/
export async function* streamAppleHealthClinicalMembers(input: {
archivePath: string;
limits: AppleHealthClinicalArchiveLimits;
wanted: (name: string) => boolean;
}): AsyncGenerator<AppleHealthArchiveMember> {
const { limits } = input;
yield* streamMembers(
input.archivePath,
{
maxMembers: limits.maxMembers,
maxEcgMembers: limits.maxClinicalMembers,
maxMemberBytes: limits.maxMemberBytes,
maxTotalEcgBytes: limits.maxTotalClinicalBytes,
maxCompressionRatio: limits.maxCompressionRatio,
},
{
label: "clinical record",
matches: isClinicalRecordJson,
maxMembers: limits.maxClinicalMembers,
maxTotalBytes: limits.maxTotalClinicalBytes,
},
input.wanted,
);
}
+2 -2
View File
@@ -256,8 +256,8 @@ export async function importAppleHealthWorkoutRoutes(input: {
}
}
} catch {
// The archive itself is unreadable for this pass (a ZIP64 export, a
// limit hit in the directory). Every referenced track not yet handled
// The archive itself is unreadable for this pass (a malformed directory,
// a limit hit in it). Every referenced track not yet handled
// counts as failed; the import stays done.
const handled = counts.attached + counts.skipped + counts.failed;
counts.failed += Math.max(0, byMember.size - handled);