diff --git a/src/lib/apple-health/ecg-csv.ts b/src/lib/apple-health/ecg-csv.ts index 6f3edba14..d1b003b75 100644 --- a/src/lib/apple-health/ecg-csv.ts +++ b/src/lib/apple-health/ecg-csv.ts @@ -38,23 +38,111 @@ function parseRecordedAt(value: string | undefined): Date { return parsed; } +/** + * Split a measured quantity into its magnitude and its unit word. + * + * The unit is matched loosely on purpose. A real export writes the sampling + * rate as "511,422 hertz" — the word spelled out, lower case, and separated by + * a NO-BREAK space in the German files — where the fixtures had always assumed + * "512 Hz". The magnitude is read through the shared decimal helper, so a + * comma-region watch's "511,562" is the same rate as "511.422". + * + * `\s` covers U+00A0, so the no-break space needs no special case. + */ +function splitQuantity( + raw: string, + unit: RegExp, +): { magnitude: number; ok: boolean } { + const match = /^([+-]?[\d.,]+)\s*([^\s\d]+)$/.exec(raw.trim()); + if (!match) return { magnitude: Number.NaN, ok: false }; + const magnitude = parseDecimal(match[1]); + if (magnitude === null) return { magnitude: Number.NaN, ok: false }; + return { magnitude, ok: unit.test(match[2]) }; +} + +/** "Hz", "hz", "hertz", "Hertz" — the same unit, spelled as the region does. */ +const HERTZ = /^(?:hz|hertz)$/i; + +/** "bpm" and the spelled-out forms a localised export may carry. */ +const BEATS_PER_MINUTE = /^(?:bpm|spm)$/i; + function parseRate(value: string | undefined): number { - const match = /^(\d+(?:\.\d+)?)\s*Hz$/i.exec(value?.trim() ?? ""); - const rate = match ? Number(match[1]) : Number.NaN; - if (!Number.isFinite(rate) || rate <= 0 || rate > 10_000) { + const { magnitude, ok } = splitQuantity(value ?? "", HERTZ); + if ( + !ok || + !Number.isFinite(magnitude) || + magnitude <= 0 || + magnitude > 10_000 + ) { throw parserError("sample rate is invalid"); } - return Math.round(rate); + return Math.round(magnitude); } function parseHeartRate(value: string | undefined): number | null { if (value === undefined || value.trim() === "") return null; - const match = /^(\d+(?:\.\d+)?)\s*bpm$/i.exec(value.trim()); - const rate = match ? Number(match[1]) : Number.NaN; - if (!Number.isFinite(rate) || rate <= 0 || rate > 300) { + const { magnitude, ok } = splitQuantity(value, BEATS_PER_MINUTE); + if (!ok || !Number.isFinite(magnitude) || magnitude <= 0 || magnitude > 300) { throw parserError("average heart rate is invalid"); } - return Math.round(rate); + return Math.round(magnitude); +} + +/* ────────────────────────────────────────────────────────────────────────── + * Localised exports. + * + * The CSV is written in the device's language, and not only in its values — + * the metadata KEYS are translated too. A German watch writes + * `Klassifizierung,Sinusrhythmus` and `Aufzeichnungsdatum,…`, so a parser + * indexing on the English keys found none of them, never crossed into the + * waveform, and rejected the whole recording. + * + * Both maps below are OBSERVED, from real exports: nine keys and two verdicts, + * consistent across every German file seen. The other four shipped languages + * are deliberately ABSENT rather than guessed. A wrong key silently mis-files a + * value and a wrong clinical verdict is worse than no verdict, so a localised + * file this parser cannot place is refused — which is visible — instead of + * being read into the wrong column, which is not. + * + * Adding a language is a data addition to these two maps and nothing else. + * + * Two gaps worth naming. No observed file carries an average-heart-rate row in + * any language, so there is no localised key for it here. And every observed + * localised file uses the single-column layout, so the paired layout's + * `Lead,Voltage` column header has only ever been seen in English; its + * translated form is unknown and is not invented here. + * ────────────────────────────────────────────────────────────────────────── */ + +/** Folded localised metadata key → the canonical English key we index on. */ +const METADATA_KEY_ALIASES: Readonly> = { + // de — observed across real exports + geburtstag: "Date of Birth", + aufzeichnungsdatum: "Recorded Date", + klassifizierung: "Classification", + symptome: "Symptoms", + softwareversion: "Software Version", + gerat: "Device", + messrate: "Sample Rate", + ableitung: "Lead", + einheit: "Unit", +}; + +/** + * Folded localised verdict → its English wording. + * + * Deliberately an indirection onto the English vocabulary rather than a second + * verdict-to-enum table: one place decides what a verdict MEANS, and + * translations only feed it. + */ +const CLASSIFICATION_ALIASES: Readonly> = { + // de — observed across real exports + sinusrhythmus: "Sinus Rhythm", + uneindeutig: "Inconclusive", +}; + +/** Resolve a metadata key to the English one this parser indexes on. */ +function canonicalMetadataKey(raw: string): string { + return METADATA_KEY_ALIASES[foldForMatch(raw)] ?? raw; } /* ────────────────────────────────────────────────────────────────────────── @@ -164,8 +252,14 @@ export function classifyEcgRhythm( } function mapClassification( - value: string | undefined, + raw: string | undefined, ): RhythmClassification | null { + // Translate first, classify second. The alias table maps an observed + // localised verdict onto Apple's own English wording and stops there; + // `classifyEcgRhythm` remains the single place that decides what a verdict + // MEANS. A per-language verdict table would drift from this one the first + // time Apple adds a case, and only one of them would get updated. + const value = CLASSIFICATION_ALIASES[foldForMatch(raw ?? "")] ?? raw; const outcome = classifyEcgRhythm(value); return outcome.kind === "mapped" ? outcome.value : null; } @@ -193,6 +287,43 @@ function splitPair(line: string): [string, string] | null { return [line.slice(0, comma).trim(), line.slice(comma + 1).trim()]; } +/** + * Which section of the file the reader is in. + * + * The two waveform layouts disagree about what a comma MEANS, so the parser + * cannot decide that per line — it has to know which side of the boundary it is + * on. Naming the mode is what makes the two readings safe to hold at once. + */ +type Section = + /** Key/value metadata rows. A comma separates the key from the value. */ + | "header" + /** One bare number per row. A comma is a decimal mark. */ + | "single-column" + /** `lead,voltage` per row. A comma separates the two fields. */ + | "paired"; + +/** + * Read one scalar written in either decimal dialect: "-180.596" and "-180,596" + * are the same measurement, recorded by watches set to different regions. + * + * Accepts at most ONE separator. A number carrying thousands grouping + * ("1.234,5") is refused rather than guessed at, because resolving it needs to + * know the file's region and nothing in the file states it. + * + * The single ambiguous case is a lone separator followed by exactly three + * digits, where "-180,596" could in principle be grouping rather than a + * decimal. It is read as a decimal, for two reasons that agree. Observed + * waveforms carry two- and three-digit fractions in the same file + * ("-199,83" next to "-217,333"), and grouping is always three, so the + * two-digit rows settle what the separator is. And a machine writing one bare + * value per row has no reason to group thousands at all. + */ +function parseDecimal(raw: string): number | null { + if (!/^[+-]?\d+(?:[.,]\d+)?$/.test(raw)) return null; + const value = Number(raw.replace(",", ".")); + return Number.isFinite(value) ? value : null; +} + export async function parseAppleHealthEcgCsv(input: { memberName: string; stream: Readable; @@ -205,8 +336,7 @@ export async function parseAppleHealthEcgCsv(input: { const metadata = new Map(); const samples: number[] = []; - let inSamples = false; - let singleColumn = false; + let section: Section = "header"; let unitScale = 1; let lead: string | null = null; const lines = createInterface({ @@ -218,10 +348,23 @@ export async function parseAppleHealthEcgCsv(input: { for await (const rawLine of lines) { const line = rawLine.replace(/^\uFEFF/, "").trim(); if (line === "") { - // Apple's single-column layout declares the lead and unit as - // metadata rows and separates them from the waveform with a blank - // line; one bare number per row follows. - if (!inSamples && metadata.has("Lead") && metadata.has("Unit")) { + // THE CROSSING into the single-column waveform. + // + // Apple's single-column layout declares the lead and unit as metadata + // rows and separates them from the waveform with a blank line; one + // bare number per row follows. A blank line on its own does not mean + // anything — real exports carry two of them mid-header — so the + // boundary is "a blank line once Lead and Unit have both been + // declared", and it is taken exactly once, from the header. + // + // Everything after this point reads a comma as a decimal mark rather + // than as a field separator, which is why the parser tracks the + // section instead of deciding line by line. + if ( + section === "header" && + metadata.has("Lead") && + metadata.has("Unit") + ) { const scale = resolveUnitScale(metadata.get("Unit")); if (scale === null) { throw parserError("sample unit is unsupported"); @@ -231,20 +374,17 @@ export async function parseAppleHealthEcgCsv(input: { if (declaredLead.length > 0 && declaredLead.length <= 32) { lead = declaredLead; } - inSamples = true; - singleColumn = true; + section = "single-column"; } continue; } - if (singleColumn) { - if (line.includes(",")) { - throw parserError("row is malformed"); - } + if (section === "single-column") { if (samples.length >= input.maxSamples) { throw parserError("sample limit exceeded"); } - const value = Number(line); - if (!Number.isFinite(value)) { + // One value per row, in whichever decimal dialect the watch was set to. + const value = parseDecimal(line); + if (value === null) { throw parserError("sample value is invalid"); } const microvolts = Math.round(value * unitScale); @@ -259,23 +399,48 @@ export async function parseAppleHealthEcgCsv(input: { } const pair = splitPair(line); if (!pair) { - throw parserError("row is malformed"); + // A HEADER row whose value is empty is written without the trailing + // comma. Every real export opens with a bare `Name` when the name + // field is blank, which this parser used to reject on line 1 — before + // it had read anything at all. + // + // Only the header section may do this. Inside the single-column + // waveform a comma-less row is a sample and was handled above; inside + // the paired waveform every row is `lead,voltage`, so a missing comma + // there is still malformed. + // + // The row contributes NO metadata entry rather than an empty-string + // one: `Lead` and `Unit` presence is what switches the parser into the + // single-column mode below, and a valueless key must not be able to + // trip that switch. + if (section !== "header") { + throw parserError("row is malformed"); + } + continue; } const [key, value] = pair; - if (!inSamples) { - if (key.toLowerCase() === "lead" && value.toLowerCase() === "voltage") { - inSamples = true; + if (section === "header") { + // THE CROSSING into the paired waveform: the `Lead,Voltage` column + // header. Unlike the single-column boundary this one is explicit in + // the file, so there is nothing to infer. + if ( + canonicalMetadataKey(key) === "Lead" && + value.toLowerCase() === "voltage" + ) { + section = "paired"; continue; } - if (key !== "Name") metadata.set(key, value); + if (key !== "Name") metadata.set(canonicalMetadataKey(key), value); continue; } if (samples.length >= input.maxSamples) { throw parserError("sample limit exceeded"); } - const voltage = Number(value); - if (!Number.isFinite(voltage) || Math.abs(voltage) > 100) { + // `splitPair` cut on the FIRST comma, so a comma still inside the value + // is this row's decimal mark, exactly as in the single-column layout. + const voltage = parseDecimal(value); + if (voltage === null || Math.abs(voltage) > 100) { throw parserError("sample value is invalid"); } const microvolts = Math.round(voltage * 1_000); @@ -286,7 +451,7 @@ export async function parseAppleHealthEcgCsv(input: { samples.push(microvolts); } - if (!inSamples || samples.length === 0) { + if (section === "header" || samples.length === 0) { throw parserError("samples are missing"); } return { diff --git a/src/lib/measurements/__tests__/apple-health-ecg-import.test.ts b/src/lib/measurements/__tests__/apple-health-ecg-import.test.ts index 5873cf1cf..f89025fc8 100644 --- a/src/lib/measurements/__tests__/apple-health-ecg-import.test.ts +++ b/src/lib/measurements/__tests__/apple-health-ecg-import.test.ts @@ -480,3 +480,209 @@ describe("Apple Health HKElectrocardiogram CSV parser", () => { expect(spies.every((spy) => spy.mock.calls.length === 0)).toBe(true); }); }); + +/** + * The shapes real exports actually use. + * + * The fixtures above were written from the format's documentation rather than + * from a file off a watch, and they drifted from it in four independent ways. + * Each block below pins one of them, and — because the differences are dialect + * rather than version — pins BOTH dialects side by side, so a later edit cannot + * quietly trade one for the other. + */ +describe("Apple Health ECG CSV — real-export shapes", () => { + async function parse(csv: string, maxSamples = 8) { + const subject = await loadParserSubject(); + return subject.parseAppleHealthEcgCsv({ + memberName: "apple_health_export/electrocardiograms/ecg_2026-07-18.csv", + stream: Readable.from([Buffer.from(csv)]), + maxSamples, + }); + } + + describe("a header row with no value", () => { + it("accepts the bare `Name` row every observed export opens with", async () => { + // The name field is blank and Apple omits the comma entirely, so the + // parser used to fail on line 1, before reading anything. + await expect( + parse(validCsv().replace("Name,Private Patient Name", "Name")), + ).resolves.toMatchObject({ samplingFrequency: 512 }); + }); + + it("keeps a valueless header row out of the metadata", async () => { + // `Lead` + `Unit` presence is what switches the parser into the + // single-column mode; a valueless key must not be able to trip it. + await expect( + parse(`${singleColumnCsv().replace("Unit,µV", "Unit")}`), + ).rejects.toThrow(/samples are missing|malformed/i); + }); + + it("still rejects a comma-less row inside the paired waveform", async () => { + await expect( + parse(validCsv().replace("I,-0.002", "-0.002")), + ).rejects.toThrow(/malformed/i); + }); + }); + + describe("both decimal dialects", () => { + /** The same three samples, written by a dot-region and a comma-region watch. */ + const DOT_SAMPLES = ["1.5", "-2.4", "3"]; + const COMMA_SAMPLES = ["1,5", "-2,4", "3"]; + + function withSamples(rows: readonly string[]): string { + return `${singleColumnCsv().split("\n").slice(0, -3).join("\n")}\n${rows.join("\n")}`; + } + + it.each([ + ["dot", DOT_SAMPLES], + ["comma", COMMA_SAMPLES], + ])( + "reads a %s-decimal waveform to the same microvolts", + async (_d, rows) => { + await expect(parse(withSamples(rows))).resolves.toMatchObject({ + samples: [2, -2, 3], + }); + }, + ); + + it("reads both dialects of the paired layout identically", async () => { + // splitPair cuts on the first comma, so the value keeps its own mark. + await expect(parse(validCsv())).resolves.toMatchObject({ + samples: [1, -2, 3], + }); + const commaPaired = validCsv() + .replace("I,0.001", "I,0,001") + .replace("I,-0.002", "I,-0,002") + .replace("I,0.003", "I,0,003"); + await expect(parse(commaPaired)).resolves.toMatchObject({ + samples: [1, -2, 3], + }); + }); + + it("refuses a grouped number rather than guessing its region", async () => { + // "1.234,5" needs the file's region to resolve and the file never says. + await expect( + parse(withSamples(["1.234,5", "-2,4", "3"])), + ).rejects.toThrow(/sample value is invalid/i); + }); + + it.each([ + ["512 Hz", 512], + ["512 hz", 512], + ["512 hertz", 512], + ["511,422 hertz", 511], + ["511.422 hertz", 511], + ["511,562\u00a0Hertz", 512], + ["512Hz", 512], + ])("reads the sample rate written as %s", async (raw, expected) => { + // Observed exports spell the unit out, lower-case it, and (in the German + // files) separate it with a no-break space. The fixtures had only ever + // used "512 Hz". + await expect( + parse(validCsv({ "Sample Rate": raw })), + ).resolves.toMatchObject({ samplingFrequency: expected }); + }); + + it.each([ + ["a missing unit", "512"], + ["the wrong unit", "512 bpm"], + ["a rate above the bound", "20000 hertz"], + ["a grouped rate", "1.234,5 hertz"], + ])("still rejects a sample rate with %s", async (_label, raw) => { + await expect(parse(validCsv({ "Sample Rate": raw }))).rejects.toThrow( + /sample rate is invalid/i, + ); + }); + + it("reads the average heart rate in both dialects", async () => { + await expect( + parse(validCsv({ "Average Heart Rate": "64 bpm" })), + ).resolves.toMatchObject({ averageHeartRate: 64 }); + await expect( + parse(validCsv({ "Average Heart Rate": "63,7 bpm" })), + ).resolves.toMatchObject({ averageHeartRate: 64 }); + }); + + it("still rejects a paired row that follows the single-column waveform", async () => { + await expect(parse(`${singleColumnCsv()}\nI,0.001`)).rejects.toThrow( + /sample value is invalid/i, + ); + }); + }); + + describe("a localised export", () => { + /** + * The header of a real German export, key for key. Every row here was + * observed; nothing is translated by hand. The blank rows mid-header and + * the no-break space before "Hertz" are the file's, not a typo. + */ + function germanCsv( + overrides: Partial> = {}, + ): string { + const metadata: Record = { + Geburtstag: '"27.03.1987"', + Aufzeichnungsdatum: "2020-07-31 22:48:19 +0200", + Klassifizierung: "Sinusrhythmus", + Symptome: "", + Softwareversion: "1.13", + Gerät: '"Watch4,2"', + Messrate: "511,562\u00a0Hertz", + ...overrides, + }; + return [ + "Name", + ...Object.entries(metadata).map(([k, v]) => `${k},${v}`), + "", + "", + "Ableitung,Ableitung I", + "Einheit,µV", + "", + "-180,596", + "-199,83", + "-217,333", + ].join("\n"); + } + + it("reads a German export end to end", async () => { + await expect(parse(germanCsv())).resolves.toMatchObject({ + recordedAt: new Date("2020-07-31T20:48:19.000Z"), + samplingFrequency: 512, + samples: [-181, -200, -217], + lead: "Ableitung I", + rhythmClassification: "NOT_DETECTED", + }); + }); + + it("reads the other observed German verdict", async () => { + await expect( + parse(germanCsv({ Klassifizierung: "Uneindeutig" })), + ).resolves.toMatchObject({ rhythmClassification: "INCONCLUSIVE" }); + }); + + it("returns no verdict for a language that is not mapped yet", async () => { + // French is not in the alias map and is not guessed at. The recording + // still imports; only the verdict is withheld. + await expect( + parse(germanCsv({ Klassifizierung: "Rythme sinusal" })), + ).resolves.toMatchObject({ rhythmClassification: null }); + }); + + it("refuses a file whose keys it cannot place, rather than mis-filing it", async () => { + // An unmapped language's KEYS mean the parser never finds the waveform. + // Refusing is visible; reading values into the wrong column is not. + const french = germanCsv() + .replace("Aufzeichnungsdatum,", "Date d'enregistrement,") + .replace("Messrate,", "Fréquence d'échantillonnage,") + .replace("Ableitung,Ableitung I", "Dérivation,Dérivation I") + .replace("Einheit,µV", "Unité,µV"); + await expect(parse(french)).rejects.toThrow(/samples are missing/i); + }); + + it("leaves the English header working unchanged", async () => { + await expect(parse(validCsv())).resolves.toMatchObject({ + samplingFrequency: 512, + rhythmClassification: "NOT_DETECTED", + }); + }); + }); +});