Merge pull request #1140 from MBombeck/ready/pr-1131

fix(import): leave out Apple Health samples HealthLog wrote itself
This commit is contained in:
Marc-André Bombeck authored and GitHub committed 2026-10-04 17:33:47 +02:00
commit 92f17c8ca4
15 files changed
+1095 -46

No files matched your search

@@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
SYNTHETIC FIXTURE. Not an export from a real person or device.
Written by hand to the shape Apple's export.xml documents in its own DTD
(abridged below): a <Record> may hold <MetadataEntry key= value=> children.
Every value, date and id is invented. What it cannot show is whether Apple's
exporter writes a custom metadata key such as dev.healthlog.app.origin
verbatim; that has to be checked against a real export (see #1109).
Records, in order:
1. weight 80.1 carries the HealthLog origin marker -> left out
2. weight 80.2 user-entered, no marker -> imported
3. weight 80.3 marker among other metadata entries -> left out
4. weight 80.4 no metadata at all -> imported
5. weight 80.5 origin key with another value -> imported
-->
<!DOCTYPE HealthData [
<!ELEMENT HealthData (ExportDate, Me, Record*)>
<!ATTLIST HealthData locale CDATA #REQUIRED>
<!ELEMENT ExportDate EMPTY>
<!ATTLIST ExportDate value CDATA #REQUIRED>
<!ELEMENT Me EMPTY>
<!ELEMENT Record (MetadataEntry*)>
<!ATTLIST Record
type CDATA #REQUIRED
unit CDATA #IMPLIED
value CDATA #IMPLIED
sourceName CDATA #REQUIRED
sourceVersion CDATA #IMPLIED
device CDATA #IMPLIED
creationDate CDATA #IMPLIED
startDate CDATA #REQUIRED
endDate CDATA #REQUIRED>
<!ELEMENT MetadataEntry EMPTY>
<!ATTLIST MetadataEntry key CDATA #REQUIRED value CDATA #REQUIRED>
]>
<HealthData locale="en_US">
<ExportDate value="2026-05-20 12:00:00 +0200"/>
<Me/>
<Record type="HKQuantityTypeIdentifierBodyMass" sourceName="SyntheticApp" unit="kg" creationDate="2026-05-14 08:00:02 +0200" startDate="2026-05-14 08:00:00 +0200" endDate="2026-05-14 08:00:00 +0200" value="80.1">
<MetadataEntry key="dev.healthlog.app.origin" value="healthlog"/>
<MetadataEntry key="HKExternalUUID" value="00000000-0000-0000-0000-000000000001"/>
</Record>
<Record type="HKQuantityTypeIdentifierBodyMass" sourceName="SyntheticApp" unit="kg" creationDate="2026-05-14 09:00:02 +0200" startDate="2026-05-14 09:00:00 +0200" endDate="2026-05-14 09:00:00 +0200" value="80.2">
<MetadataEntry key="HKWasUserEntered" value="1"/>
</Record>
<Record type="HKQuantityTypeIdentifierBodyMass" sourceName="SyntheticApp" unit="kg" creationDate="2026-05-14 10:00:02 +0200" startDate="2026-05-14 10:00:00 +0200" endDate="2026-05-14 10:00:00 +0200" value="80.3">
<MetadataEntry key="HKWasUserEntered" value="1"/>
<MetadataEntry key="dev.healthlog.app.origin" value="healthlog"/>
</Record>
<Record type="HKQuantityTypeIdentifierBodyMass" sourceName="SyntheticApp" unit="kg" creationDate="2026-05-14 11:00:02 +0200" startDate="2026-05-14 11:00:00 +0200" endDate="2026-05-14 11:00:00 +0200" value="80.4"/>
<Record type="HKQuantityTypeIdentifierBodyMass" sourceName="SyntheticApp" unit="kg" creationDate="2026-05-14 12:00:02 +0200" startDate="2026-05-14 12:00:00 +0200" endDate="2026-05-14 12:00:00 +0200" value="80.5">
<MetadataEntry key="dev.healthlog.app.origin" value="some-other-app"/>
</Record>
</HealthData>
@@ -0,0 +1,572 @@
/**
* The Apple Health export must not bring back what HealthLog wrote into Apple
* Health itself.
*
* The iOS app writes manual entries into Apple Health, and mirrors Withings and
* import rows into it, each stamped with `dev.healthlog.app.origin = healthlog`.
* The server already holds those readings under their own source, so the same
* sample arriving through an export is a second copy: the series shows it
* twice and the rollups count it twice.
*
* Asserted against a real Postgres:
* - a record carrying the marker is left out, and counted;
* - a record without it lands as before;
* - the marker is read from a child `<MetadataEntry>` that follows the
* record's open tag, so the record is committed at its close tag;
* - a sample from before the marker carries the HealthLog row id as
* `HKExternalUUID`; a hit on one of the account's rows of the same kind
* leaves it out, either id of a blood pressure pair covering both halves;
* - a manual entry mirrored with neither (a MANUAL row with the same type and
* value within 2 s) is left out and counted separately;
* - a MANUAL row with another value, or 3 s away, is not a match;
* - a Withings row of the same value and time is not a match either (those
* are mirrored with their row id, so a match would only add false
* positives);
* - the value match asks nothing for a type the account has no MANUAL rows
* of, and each sample left out is counted once.
*/
import { mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { beforeEach, describe, expect, it } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import { streamParseExportXml } from "@/lib/measurements/import-apple-health-export";
const prisma = getPrismaClient();
beforeEach(async () => {
await truncateAllTables(prisma);
});
function writeXml(records: string): string {
const dir = mkdtempSync(join(tmpdir(), "healthlog-import-own-"));
const xmlPath = join(dir, "export.xml");
writeFileSync(
xmlPath,
`<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE HealthData [<!ELEMENT HealthData (Record)*>]>
<HealthData locale="en_US">
${records}
</HealthData>`,
);
return xmlPath;
}
const weight = (value: number, end: string, children = "") =>
` <Record type="HKQuantityTypeIdentifierBodyMass" sourceName="Health" unit="kg" startDate="${end}" endDate="${end}" value="${value}">${children}</Record>`;
const OWN = `<MetadataEntry key="dev.healthlog.app.origin" value="healthlog"/>`;
const OTHER = `<MetadataEntry key="HKWasUserEntered" value="1"/>`;
async function run(records: string) {
const user = await prisma.user.create({
data: { username: "own-samples", email: "own@example.test", role: "USER" },
});
const result = await streamParseExportXml({
xmlPath: writeXml(records),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
const rows = await prisma.measurement.findMany({
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
orderBy: { measuredAt: "asc" },
});
return { user, result, rows };
}
describe("a record HealthLog wrote itself (the origin marker)", () => {
it("is left out and counted; an ordinary record still lands", async () => {
const { result, rows } = await run(
[
weight(80.1, "2026-05-14 08:00:00 +0200", `\n ${OWN}\n `),
weight(80.2, "2026-05-14 09:00:00 +0200", `\n ${OTHER}\n `),
weight(80.3, "2026-05-14 10:00:00 +0200"),
].join("\n"),
);
expect(rows.map((r) => r.value)).toEqual([80.2, 80.3]);
expect(result.writtenByHealthLog).toEqual({
byMarker: 1,
byExternalId: 0,
matchedManual: 0,
});
});
it("finds the marker among other metadata entries", async () => {
const { result, rows } = await run(
weight(80.1, "2026-05-14 08:00:00 +0200", `\n ${OTHER}\n ${OWN}\n`),
);
expect(rows).toHaveLength(0);
expect(result.writtenByHealthLog.byMarker).toBe(1);
});
it("does not let one record's marker spill onto the next", async () => {
const { rows } = await run(
[
weight(80.1, "2026-05-14 08:00:00 +0200", `\n ${OWN}\n`),
weight(80.2, "2026-05-14 09:00:00 +0200"),
].join("\n"),
);
expect(rows.map((r) => r.value)).toEqual([80.2]);
});
it("needs the exact value: another origin does not count", async () => {
const { rows } = await run(
weight(
80.1,
"2026-05-14 08:00:00 +0200",
`<MetadataEntry key="dev.healthlog.app.origin" value="other"/>`,
),
);
expect(rows).toHaveLength(1);
});
});
const bpRecord = (
type: "Systolic" | "Diastolic",
value: number,
at: string,
children = "",
) =>
` <Record type="HKQuantityTypeIdentifierBloodPressure${type}" sourceName="Health" unit="mmHg" startDate="${at}" endDate="${at}" value="${value}">${children}</Record>`;
const correlation = (inner: string, children = "") =>
` <Correlation type="HKCorrelationTypeIdentifierBloodPressure" sourceName="Health" startDate="2026-05-14 08:00:00 +0200" endDate="2026-05-14 08:00:00 +0200">${children}\n${inner}\n </Correlation>`;
describe("blood pressure correlations and self-closing records", () => {
const AT = "2026-05-14 08:00:00 +0200";
async function runBp(records: string) {
const user = await prisma.user.create({
data: { username: "own-bp", email: "bp@example.test", role: "USER" },
});
const result = await streamParseExportXml({
xmlPath: writeXml(records),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
const rows = await prisma.measurement.findMany({
where: {
userId: user.id,
type: { in: ["BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA"] },
},
orderBy: [{ type: "asc" }],
});
return { result, values: rows.map((r) => r.value) };
}
it("imports an unmarked correlation's two records", async () => {
const { values, result } = await runBp(
correlation(
[bpRecord("Systolic", 121, AT), bpRecord("Diastolic", 79, AT)].join(
"\n",
),
),
);
expect(values).toEqual([121, 79]);
expect(result.writtenByHealthLog.byMarker).toBe(0);
});
it("leaves out both records when the marker is on the correlation, before them", async () => {
const { values, result } = await runBp(
correlation(
[bpRecord("Systolic", 121, AT), bpRecord("Diastolic", 79, AT)].join(
"\n",
),
`\n ${OWN}`,
),
);
expect(values).toEqual([]);
expect(result.writtenByHealthLog.byMarker).toBe(2);
});
it("leaves out both records when the marker is on the correlation, after them", async () => {
const records = [
bpRecord("Systolic", 121, AT),
bpRecord("Diastolic", 79, AT),
].join("\n");
const { values } = await runBp(
` <Correlation type="HKCorrelationTypeIdentifierBloodPressure" sourceName="Health" startDate="${AT}" endDate="${AT}">\n${records}\n ${OWN}\n </Correlation>`,
);
expect(values).toEqual([]);
});
it("leaves out only the record that carries the marker", async () => {
const { values } = await runBp(
correlation(
[
bpRecord("Systolic", 121, AT, `\n ${OWN}\n `),
bpRecord("Diastolic", 79, AT),
].join("\n"),
),
);
expect(values).toEqual([79]);
});
it("does not let a correlation's marker spill onto the next record", async () => {
const { values } = await runBp(
[
correlation(
[bpRecord("Systolic", 121, AT), bpRecord("Diastolic", 79, AT)].join(
"\n",
),
`\n ${OWN}`,
),
bpRecord("Systolic", 130, "2026-05-14 09:00:00 +0200"),
].join("\n"),
);
expect(values).toEqual([130]);
});
it("imports a self-closing record, with no children to wait for", async () => {
const { rows } = await run(
` <Record type="HKQuantityTypeIdentifierBodyMass" sourceName="Health" unit="kg" startDate="${AT}" endDate="${AT}" value="80.9"/>`,
);
expect(rows.map((r) => r.value)).toEqual([80.9]);
});
});
describe("the synthetic export fixture", () => {
// `export-own-origin.synthetic.xml` is invented, in the shape Apple's DTD
// documents. It shows the parser reads a file laid out like an export; it
// cannot show that Apple writes a custom metadata key verbatim.
it("leaves out the two marked records and imports the other three", async () => {
const user = await prisma.user.create({
data: { username: "own-fixture", email: "fx@example.test", role: "USER" },
});
const result = await streamParseExportXml({
xmlPath: join(
process.cwd(),
"tests/fixtures/apple-health/export-own-origin.synthetic.xml",
),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
const rows = await prisma.measurement.findMany({
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
orderBy: { measuredAt: "asc" },
});
expect(rows.map((r) => r.value)).toEqual([80.2, 80.4, 80.5]);
expect(result.writtenByHealthLog).toEqual({
byMarker: 2,
byExternalId: 0,
matchedManual: 0,
});
});
});
describe("a manual entry mirrored before the marker existed", () => {
async function runAgainst(
manual: { value: number; at: string; source?: "MANUAL" | "WITHINGS" },
record: { value: number; at: string },
) {
const user = await prisma.user.create({
data: { username: "own-fallback", email: "f@example.test", role: "USER" },
});
await prisma.measurement.create({
data: {
userId: user.id,
type: "WEIGHT",
unit: "kg",
source: manual.source ?? "MANUAL",
value: manual.value,
measuredAt: new Date(manual.at),
},
});
const result = await streamParseExportXml({
xmlPath: writeXml(weight(record.value, "2026-05-14 08:00:00 +0200")),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
const rows = await prisma.measurement.findMany({
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
});
return { result, rows };
}
it("is left out when a MANUAL row has the same value within 2 s", async () => {
const { result, rows } = await runAgainst(
{ value: 80.4, at: "2026-05-14T06:00:01.500Z" },
{ value: 80.4, at: "" },
);
expect(rows).toHaveLength(0);
expect(result.writtenByHealthLog).toEqual({
byMarker: 0,
byExternalId: 0,
matchedManual: 1,
});
});
it("is kept when the value differs", async () => {
const { result, rows } = await runAgainst(
{ value: 80.9, at: "2026-05-14T06:00:00.000Z" },
{ value: 80.4, at: "" },
);
expect(rows).toHaveLength(1);
expect(result.writtenByHealthLog.matchedManual).toBe(0);
});
it("is kept when the MANUAL row is 3 s away", async () => {
const { rows } = await runAgainst(
{ value: 80.4, at: "2026-05-14T06:00:03.000Z" },
{ value: 80.4, at: "" },
);
expect(rows).toHaveLength(1);
});
it("is kept against a Withings row of the same value and time", async () => {
const { result, rows } = await runAgainst(
{ value: 80.4, at: "2026-05-14T06:00:00.000Z", source: "WITHINGS" },
{ value: 80.4, at: "" },
);
expect(rows).toHaveLength(1);
expect(result.writtenByHealthLog.matchedManual).toBe(0);
});
});
/**
* Count every `measurement.findMany` the import issues against MANUAL rows,
* by wrapping the client the import is handed.
*/
function countingManualQueries() {
const manualQueries: Array<{ where?: Record<string, unknown> }> = [];
const measurement = new Proxy(prisma.measurement, {
get(target, prop, receiver) {
const value = Reflect.get(target, prop, receiver);
if (prop !== "findMany" || typeof value !== "function") return value;
return (args: { where?: Record<string, unknown> }) => {
if (args?.where?.source === "MANUAL") manualQueries.push(args);
return value.call(target, args);
};
},
});
const client = new Proxy(prisma, {
get(target, prop, receiver) {
if (prop === "measurement") return measurement;
const value = Reflect.get(target, prop, receiver);
return typeof value === "function" ? value.bind(target) : value;
},
}) as typeof prisma;
return { client, manualQueries };
}
describe("the manual-mirror lookup stays off the hot path", () => {
const AT = (i: number) =>
`2026-05-14 ${String(8 + Math.floor(i / 60)).padStart(2, "0")}:${String(i % 60).padStart(2, "0")}:00 +0200`;
const heartRate = (value: number, at: string) =>
` <Record type="HKQuantityTypeIdentifierHeartRate" sourceName="Health" unit="count/min" startDate="${at}" endDate="${at}" value="${value}"/>`;
async function runCounting(records: string, manualType?: "WEIGHT") {
const user = await prisma.user.create({
data: { username: "own-hot", email: "hot@example.test", role: "USER" },
});
if (manualType) {
await prisma.measurement.create({
data: {
userId: user.id,
type: manualType,
unit: "kg",
source: "MANUAL",
value: 80.4,
measuredAt: new Date("2026-05-14T06:00:00.000Z"),
},
});
}
const { client, manualQueries } = countingManualQueries();
const result = await streamParseExportXml({
xmlPath: writeXml(records),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma: client,
spotBatchSize: 10,
});
return { result, manualQueries, user };
}
it("asks nothing per batch when the account has no MANUAL rows", async () => {
const records = Array.from({ length: 40 }, (_, i) =>
heartRate(60 + (i % 20), AT(i)),
).join("\n");
const { manualQueries, result } = await runCounting(records);
expect(manualQueries).toHaveLength(0);
expect(result.writtenByHealthLog.matchedManual).toBe(0);
});
it("asks nothing for a type without MANUAL rows, and one bounded window per batch for one with them", async () => {
const records = [
...Array.from({ length: 40 }, (_, i) => heartRate(60 + (i % 20), AT(i))),
weight(80.4, "2026-05-14 08:00:00 +0200"),
weight(80.6, "2026-05-14 09:00:00 +0200"),
].join("\n");
const { manualQueries, result, user } = await runCounting(
records,
"WEIGHT",
);
// The heart rates ask nothing; the weights ask once, with one window for
// their type, not one clause per row.
expect(manualQueries).toHaveLength(1);
const or = manualQueries[0].where?.OR as Array<{ type: string }>;
expect(or).toHaveLength(1);
expect(or[0].type).toBe("WEIGHT");
// The mirror is still caught; the other weight still lands.
expect(result.writtenByHealthLog.matchedManual).toBe(1);
const weights = await prisma.measurement.findMany({
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
});
expect(weights.map((w) => w.value)).toEqual([80.6]);
});
});
describe("a sample whose HKExternalUUID is a HealthLog row id", () => {
const AT = "2026-05-14 08:00:00 +0200";
const ext = (id: string) =>
`<MetadataEntry key="HKExternalUUID" value="${id}"/>`;
async function runWith(
seed: (userId: string) => Promise<Record<string, string>>,
records: (ids: Record<string, string>) => string,
) {
const user = await prisma.user.create({
data: { username: "own-ext", email: "ext@example.test", role: "USER" },
});
const ids = await seed(user.id);
const result = await streamParseExportXml({
xmlPath: writeXml(records(ids)),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
const rows = await prisma.measurement.findMany({
where: { userId: user.id, source: "APPLE_HEALTH" },
orderBy: [{ type: "asc" }],
});
return { result, rows };
}
const seedRow = async (
userId: string,
type: "WEIGHT" | "BLOOD_PRESSURE_SYS" | "BLOOD_PRESSURE_DIA",
value: number,
source: "MANUAL" | "WITHINGS" = "MANUAL",
) =>
(
await prisma.measurement.create({
data: {
userId,
type,
value,
unit: type === "WEIGHT" ? "kg" : "mmHg",
source,
// Ten minutes off the sample: only the id can tie them together.
measuredAt: new Date("2026-05-14T06:10:00.000Z"),
},
})
).id;
it("is left out without the marker, for a Withings row the value match never covers", async () => {
const { result, rows } = await runWith(
async (userId) => ({
w: await seedRow(userId, "WEIGHT", 81, "WITHINGS"),
}),
(ids) => weight(80.4, AT, ext(ids.w)),
);
expect(rows).toHaveLength(0);
expect(result.writtenByHealthLog).toEqual({
byMarker: 0,
byExternalId: 1,
matchedManual: 0,
});
});
it("leaves out both halves of a pressure pair stamped with either half's id", async () => {
const { result, rows } = await runWith(
async (userId) => ({
sys: await seedRow(userId, "BLOOD_PRESSURE_SYS", 121),
dia: await seedRow(userId, "BLOOD_PRESSURE_DIA", 79),
}),
(ids) =>
[
// Current builds stamp both samples with the systolic id ...
bpRecord("Systolic", 121, AT, ext(ids.sys)),
bpRecord("Diastolic", 79, AT, ext(ids.sys)),
// ... older builds stamped both with the diastolic id.
bpRecord("Systolic", 122, "2026-05-14 09:00:00 +0200", ext(ids.dia)),
bpRecord("Diastolic", 80, "2026-05-14 09:00:00 +0200", ext(ids.dia)),
].join("\n"),
);
expect(rows).toHaveLength(0);
expect(result.writtenByHealthLog.byExternalId).toBe(4);
});
it("is imported when the id is unknown, belongs to another account, or names another kind", async () => {
const other = await prisma.user.create({
data: { username: "own-ext-2", email: "ext2@example.test", role: "USER" },
});
const foreignId = await seedRow(other.id, "WEIGHT", 80.4);
const { result, rows } = await runWith(
async (userId) => ({
sys: await seedRow(userId, "BLOOD_PRESSURE_SYS", 121),
}),
(ids) =>
[
weight(80.1, AT, ext("00000000-0000-4000-8000-000000000001")),
weight(80.2, "2026-05-14 09:00:00 +0200", ext(foreignId)),
weight(80.3, "2026-05-14 10:00:00 +0200", ext(ids.sys)),
].join("\n"),
);
expect(rows.map((r) => r.value)).toEqual([80.1, 80.2, 80.3]);
expect(result.writtenByHealthLog.byExternalId).toBe(0);
});
});
describe("each sample left out is counted once", () => {
it("does not count a marked record twice when it appears top level and inside its correlation", async () => {
const AT = "2026-05-14 08:00:00 +0200";
const sys = bpRecord("Systolic", 121, AT, `\n ${OWN}\n `);
const dia = bpRecord("Diastolic", 79, AT, `\n ${OWN}\n `);
const user = await prisma.user.create({
data: { username: "own-once", email: "once@example.test", role: "USER" },
});
const result = await streamParseExportXml({
xmlPath: writeXml(
[sys, dia, correlation([sys, dia].join("\n"))].join("\n"),
),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
expect(result.writtenByHealthLog.byMarker).toBe(2);
});
});
describe("a cycle sample HealthLog wrote itself", () => {
it("is left out of the day-log import", async () => {
const user = await prisma.user.create({
data: {
username: "own-cycle",
email: "cycle@example.test",
role: "USER",
gender: "FEMALE",
timezone: "Europe/Berlin",
},
});
const flow = (day: string, children = "") =>
` <Record type="HKCategoryTypeIdentifierMenstrualFlow" sourceName="Health" value="HKCategoryValueMenstrualFlowMedium" startDate="${day} 08:00:00 +0000" endDate="${day} 08:00:00 +0000">${children}</Record>`;
const result = await streamParseExportXml({
xmlPath: writeXml(
[flow("2026-03-02", OWN), flow("2026-03-03")].join("\n"),
),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
const days = await prisma.cycleDayLog.findMany({
where: { userId: user.id },
orderBy: { date: "asc" },
});
expect(days.map((d) => d.date)).toEqual(["2026-03-03"]);
expect(result.writtenByHealthLog.byMarker).toBe(1);
});
});