fix(import): leave out Apple Health samples HealthLog wrote itself

The iOS app writes manual entries into Apple Health, and mirrors Withings and
import rows into it, each stamped with dev.healthlog.app.origin = healthlog.
The server already holds those readings under their own source, so the same
sample arriving through an Apple Health export is a second copy: the series
shows it twice and the rollups count it twice.

The export import now leaves out a record carrying that marker. The marker is a
child <MetadataEntry>, read after the record's open tag, so a record is
committed at its close tag. As a fallback for manual entries mirrored before the
app set the marker, a record is also left out when a MANUAL row of the same
type and value sits within 2 s (same rule and tolerance as the sync path).
Withings and import rows are not matched by value and time: they reach Apple
Health only through the mirror, which always sets the marker.

The import summary says how many records were left out for this reason.

Fixes #1109
This commit is contained in:
BernhardBuckel committed 2026-10-04 10:53:23 +02:00
1 parent 808b2672cd
commit a5e98c9e71
11 files changed
+353 -41

No files matched your search

@@ -31,6 +31,8 @@ interface JobStatus {
totals?: { recordsRead?: number; rowsUpserted?: number };
clinical?: { skipped?: number };
cumulativeEstimates?: { days?: number; rows?: number };
/** Records HealthLog wrote into Apple Health itself, left out here. */
writtenByHealthLog?: { byMarker?: number; matchedManual?: number };
/** HK types deferred by design — read but not (yet) imported. */
deferred?: Record<string, number>;
/** Per-key drop counters — plain HK types are unsupported-by-design,
@@ -342,6 +344,9 @@ export function AppleHealthImportCard() {
const status = statusQuery.data ?? null;
const isRunning = status !== null && !TERMINAL_STATES.includes(status.status);
const isDone = status?.status === "done";
const ownRecordsSkipped =
(status?.result?.writtenByHealthLog?.byMarker ?? 0) +
(status?.result?.writtenByHealthLog?.matchedManual ?? 0);
const isFailed = status?.status === "failed";
const busy = uploading || isRunning;
const percent = status?.progress?.percent ?? null;
@@ -492,6 +497,19 @@ export function AppleHealthImportCard() {
})}
</p>
)}
{isDone && ownRecordsSkipped > 0 && (
<p
data-testid="import-apple-health-own-skipped"
className="text-muted-foreground text-sm"
>
{t(
"settings.sections.export.import.appleHealth.writtenByHealthLog",
{
count: ownRecordsSkipped,
},
)}
</p>
)}
{isDone && (status?.result?.cumulativeEstimates?.days ?? 0) > 0 && (
<AppleHealthEstimateWarning
days={status?.result?.cumulativeEstimates?.days ?? 0}
+24
View File
@@ -0,0 +1,24 @@
/**
* The marker HealthLog's iOS app stamps on every HealthKit sample it writes
* itself (a manual entry, or a Withings or import row it mirrors into Apple
* Health). The sample also carries the HealthLog measurement id as
* `HKExternalUUID`.
*
* Such a sample is already in the database under its own source. When the same
* sample comes back through an Apple Health export it is a second copy of that
* reading, so the export import leaves it out. This is the same predicate the
* app's own sync filter uses, and a stable contract with the app.
*/
export const HEALTHLOG_ORIGIN_METADATA_KEY = "dev.healthlog.app.origin";
export const HEALTHLOG_ORIGIN_METADATA_VALUE = "healthlog";
/** Is this `<MetadataEntry>` the mark of a sample HealthLog wrote itself? */
export function isHealthLogOriginEntry(
key: string | undefined,
value: string | undefined,
): boolean {
return (
key === HEALTHLOG_ORIGIN_METADATA_KEY &&
value === HEALTHLOG_ORIGIN_METADATA_VALUE
);
}
@@ -40,6 +40,11 @@ import {
dayKeyForUserTz,
canonicalDailyTimestamp,
} from "@/lib/measurements/drain-per-sample-cumulative";
import { isHealthLogOriginEntry } from "@/lib/apple-health/own-origin";
import {
isSameReadingAcrossSource,
MEASURED_AT_TOLERANCE_MS,
} from "@/lib/measurements/cross-source-merge";
import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement";
import {
insertNewMeasurementRows,
@@ -115,6 +120,13 @@ export interface ImportJobResult {
durationMs: number;
};
clinical: { skipped: number };
/**
* Records left out because HealthLog wrote them into Apple Health itself and
* already holds them under their own source: `byMarker` carry the app's
* origin marker, `matchedManual` match a manual entry (same type and value,
* within 2 s) from before the marker existed.
*/
writtenByHealthLog: { byMarker: number; matchedManual: number };
/**
* v1.15.0 — reproductive HealthKit samples routed into CYCLE day-logs
* (NOT Measurement). Absent / zeroed when the account has no cycle
@@ -412,6 +424,11 @@ export async function streamParseExportXml(
MeasurementType,
Map<string, CumulativeSourceSubtotals>
>();
// A record's contribution is committed at its close tag, once its child
// `<MetadataEntry>` rows are read: one of them can mark the sample as written
// by HealthLog itself, which has to keep it out of the import.
let pendingRecord: { commit: () => void; ownOrigin: boolean } | null = null;
const writtenByHealthLog = { byMarker: 0, matchedManual: 0 };
// Spot-row batch awaiting flush.
const spotBatch: PreparedMeasurement[] = [];
// Workout-row batch awaiting flush.
@@ -468,9 +485,54 @@ export async function streamParseExportXml(
};
// ── Flush helpers ──────────────────────────────────────────
// A manual entry the app mirrored into Apple Health before it stamped its
// origin marker comes back in the export as a second copy of the reading.
// Leave it out when a MANUAL row of the same type and value sits within 2 s.
// Only MANUAL: Withings and import rows reach Apple Health through the app's
// mirror, which always sets the marker, so a value-and-time match against
// them would only add false positives. Same rule and tolerance as the sync
// path (`cross-source-merge.ts`).
const withoutManualMirrors = async (
rows: PreparedMeasurement[],
): Promise<PreparedMeasurement[]> => {
if (rows.length === 0) return rows;
const candidates = await prisma.measurement.findMany({
where: {
userId,
source: "MANUAL",
deletedAt: null,
OR: rows.map((row) => ({
type: row.type,
measuredAt: {
gte: new Date(row.measuredAt.getTime() - MEASURED_AT_TOLERANCE_MS),
lte: new Date(row.measuredAt.getTime() + MEASURED_AT_TOLERANCE_MS),
},
})),
},
select: { type: true, source: true, value: true, measuredAt: true },
});
if (candidates.length === 0) return rows;
return rows.filter((row) => {
const mirrored = candidates.some((candidate) =>
isSameReadingAcrossSource(
{
type: row.type,
source: "APPLE_HEALTH",
value: row.value,
measuredAt: row.measuredAt,
},
candidate,
),
);
if (mirrored) writtenByHealthLog.matchedManual += 1;
return !mirrored;
});
};
const flushSpotBatch = async (): Promise<void> => {
if (spotBatch.length === 0) return;
const chunk = spotBatch.splice(0, spotBatch.length);
const incoming = spotBatch.splice(0, spotBatch.length);
const chunk = await withoutManualMirrors(incoming);
for (const row of chunk) widenSpan(row.measuredAt);
const insertedArrivals: Array<{
id: string;
@@ -907,46 +969,51 @@ export async function streamParseExportXml(
const stat = bumpStat(mapped.type);
stat.read += 1;
if (CUMULATIVE_HK_TYPES.has(mapped.type)) {
const dayKey = dayKeyForUserTz(mapped.takenAt, userTimezone);
const sourceHash = hashCumulativeSourceIdentity(
attrs.sourceName,
attrs.device,
);
let byDay = cumulativeBucket.get(mapped.type);
if (!byDay) {
byDay = new Map();
cumulativeBucket.set(mapped.type, byDay);
}
let bySource = byDay.get(dayKey);
if (!bySource) {
bySource = new Map();
byDay.set(dayKey, bySource);
}
bySource.set(
sourceHash,
(bySource.get(sourceHash) ?? 0) + mapped.value,
);
} else {
// Spot row: derive a stable externalId, queue for flush.
const externalId = hashSampleKey(
hkType,
attrs.value ?? "",
attrs.startDate,
attrs.endDate,
);
spotBatch.push({
userId,
type: mapped.type,
value: mapped.value,
unit: mapped.unit,
measuredAt: mapped.takenAt,
externalId,
externalSourceVersion: attrs.sourceVersion ?? null,
sleepStage: mapped.sleepStage ?? null,
deviceType: null,
});
}
pendingRecord = {
ownOrigin: false,
commit: () => {
if (CUMULATIVE_HK_TYPES.has(mapped.type)) {
const dayKey = dayKeyForUserTz(mapped.takenAt, userTimezone);
const sourceHash = hashCumulativeSourceIdentity(
attrs.sourceName,
attrs.device,
);
let byDay = cumulativeBucket.get(mapped.type);
if (!byDay) {
byDay = new Map();
cumulativeBucket.set(mapped.type, byDay);
}
let bySource = byDay.get(dayKey);
if (!bySource) {
bySource = new Map();
byDay.set(dayKey, bySource);
}
bySource.set(
sourceHash,
(bySource.get(sourceHash) ?? 0) + mapped.value,
);
} else {
// Spot row: derive a stable externalId, queue for flush.
const externalId = hashSampleKey(
hkType,
attrs.value ?? "",
attrs.startDate,
attrs.endDate,
);
spotBatch.push({
userId,
type: mapped.type,
value: mapped.value,
unit: mapped.unit,
measuredAt: mapped.takenAt,
externalId,
externalSourceVersion: attrs.sourceVersion ?? null,
sleepStage: mapped.sleepStage ?? null,
deviceType: null,
});
}
},
};
return;
}
@@ -1025,6 +1092,9 @@ export async function streamParseExportXml(
}
if (name === "MetadataEntry") {
if (pendingRecord && isHealthLogOriginEntry(attrs.key, attrs.value)) {
pendingRecord.ownOrigin = true;
}
// Attach the SexualActivity protection flag to the open cycle record.
// Apple writes `HKMetadataKeySexualActivityProtectionUsed` with a
// `"0"`/`"1"` (or `"true"`/`"false"`) value.
@@ -1070,6 +1140,12 @@ export async function streamParseExportXml(
};
parser.onclosetag = async (tagName) => {
if (tagName === "Record" && pendingRecord) {
const rec = pendingRecord;
pendingRecord = null;
if (rec.ownOrigin) writtenByHealthLog.byMarker += 1;
else rec.commit();
}
if (tagName === "Workout" && currentWorkout) {
workoutBatch.push(currentWorkout);
currentWorkout = null;
@@ -1212,6 +1288,7 @@ export async function streamParseExportXml(
perType,
workouts,
clinical,
writtenByHealthLog,
cycle,
deferred,
unknown,