diff --git a/messages/de.json b/messages/de.json index 0c7173452..251f58dd9 100644 --- a/messages/de.json +++ b/messages/de.json @@ -5373,6 +5373,7 @@ "rowsImported": "Bisher {count} Einträge importiert", "doneSummary": "Fertig — {imported} von {read} Datensätzen importiert, {skipped} klinische Datensätze übersprungen.", "doneNothing": "Aus diesem Archiv wurde nichts importiert: 0 von {read} Datensätzen sind angekommen.", + "writtenByHealthLog": "{count} Einträge, die HealthLog selbst in Apple Health geschrieben hat, wurden ausgelassen, weil sie schon in deinen Daten sind.", "ecgResult": "EKGs: {discovered} gefunden, {imported} importiert, {updated} aktualisiert, {skipped} übersprungen, {failed} fehlgeschlagen. Fehler bei EKGs verwerfen keine unterstützten Datensätze, die aus export.xml importiert wurden.", "estimateWarning": "Kumulative Summen für {count} Tag(e) wurden anhand der größten Exportquelle geschätzt. Native Apple-Health-Statistiken bleiben maßgeblich.", "phase": { diff --git a/messages/en.json b/messages/en.json index 3cd9aa9a4..3196dad45 100644 --- a/messages/en.json +++ b/messages/en.json @@ -5373,6 +5373,7 @@ "rowsImported": "{count} rows imported so far", "doneSummary": "Done — {imported} of {read} records imported, {skipped} clinical records skipped.", "doneNothing": "Nothing was imported from that archive: 0 of {read} records landed.", + "writtenByHealthLog": "{count} records that HealthLog wrote into Apple Health itself were left out, because they are already in your data.", "ecgResult": "ECGs: {discovered} discovered, {imported} imported, {updated} updated, {skipped} skipped, {failed} failed. ECG failures do not discard supported records imported from export.xml.", "estimateWarning": "Cumulative totals for {count} day(s) were estimated from the largest export source. Native Apple Health statistics remain authoritative.", "phase": { diff --git a/messages/es.json b/messages/es.json index d40131208..2d6d6414c 100644 --- a/messages/es.json +++ b/messages/es.json @@ -5373,6 +5373,7 @@ "rowsImported": "{count} filas importadas hasta ahora", "doneSummary": "Listo: {imported} de {read} registros importados, {skipped} registros clínicos omitidos.", "doneNothing": "No se importó nada de ese archivo: llegaron 0 de {read} registros.", + "writtenByHealthLog": "Se omitieron {count} registros que HealthLog escribió en Apple Health, porque ya están en tus datos.", "ecgResult": "ECG: {discovered} detectados, {imported} importados, {updated} actualizados, {skipped} omitidos y {failed} fallidos. Los errores de ECG no descartan los registros compatibles importados desde export.xml.", "estimateWarning": "Los totales acumulados de {count} día(s) se estimaron a partir de la mayor fuente de la exportación. Las estadísticas nativas de Apple Health siguen siendo la referencia.", "phase": { diff --git a/messages/fr.json b/messages/fr.json index a5d0afa08..8e1bb35f1 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -5373,6 +5373,7 @@ "rowsImported": "{count} lignes importées jusqu'ici", "doneSummary": "Terminé — {imported} enregistrements sur {read} importés, {skipped} enregistrements cliniques ignorés.", "doneNothing": "Rien n'a été importé de cette archive : 0 enregistrement sur {read} est arrivé.", + "writtenByHealthLog": "{count} enregistrements que HealthLog a écrits lui-même dans Santé ont été ignorés, car ils figurent déjà dans vos données.", "ecgResult": "ECG : {discovered} détectés, {imported} importés, {updated} mis à jour, {skipped} ignorés et {failed} échoués. Les erreurs d’ECG n’annulent pas les enregistrements compatibles importés depuis export.xml.", "estimateWarning": "Les totaux cumulés de {count} jour(s) ont été estimés à partir de la source d’export la plus élevée. Les statistiques Apple Santé natives restent la référence.", "phase": { diff --git a/messages/it.json b/messages/it.json index e0e489863..bc6b6ab1a 100644 --- a/messages/it.json +++ b/messages/it.json @@ -5373,6 +5373,7 @@ "rowsImported": "{count} righe importate finora", "doneSummary": "Fatto: {imported} di {read} record importati, {skipped} record clinici saltati.", "doneNothing": "Da questo archivio non è stato importato nulla: sono arrivati 0 record su {read}.", + "writtenByHealthLog": "{count} registrazioni che HealthLog ha scritto in Salute sono state escluse, perché sono già nei tuoi dati.", "ecgResult": "ECG: {discovered} rilevati, {imported} importati, {updated} aggiornati, {skipped} ignorati e {failed} non riusciti. Gli errori ECG non eliminano i record supportati importati da export.xml.", "estimateWarning": "I totali cumulativi di {count} giorno/i sono stati stimati dalla fonte di esportazione più alta. Le statistiche native di Apple Salute restano autorevoli.", "phase": { diff --git a/messages/ko.json b/messages/ko.json index e51af7017..b416ab808 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -5373,6 +5373,7 @@ "rowsImported": "지금까지 {count}개 행을 불러왔어요", "doneSummary": "완료 — {read}개 중 {imported}개 기록을 불러왔고, 진료 기록 {skipped}개는 건너뛰었어요.", "doneNothing": "그 보관 파일에서 불러온 게 없어요: {read}개 중 0개만 들어왔어요.", + "writtenByHealthLog": "HealthLog가 직접 건강 앱에 기록한 {count}개 항목은 이미 데이터에 있으므로 제외했습니다.", "ecgResult": "심전도: 발견 {discovered}개, 불러옴 {imported}개, 갱신 {updated}개, 건너뜀 {skipped}개, 실패 {failed}개. 심전도가 실패해도 export.xml에서 불러온 지원 기록은 그대로 남아요.", "estimateWarning": "{count}일치 누적 합계는 가장 큰 내보내기 소스에서 추정했어요. Apple Health 자체 통계가 여전히 기준이에요.", "phase": { diff --git a/messages/pl.json b/messages/pl.json index caa7f9245..b907285fe 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -5373,6 +5373,7 @@ "rowsImported": "Zaimportowano dotąd {count} wierszy", "doneSummary": "Gotowe — zaimportowano {imported} z {read} rekordów, pominięto {skipped} rekordów klinicznych.", "doneNothing": "Z tego archiwum nic nie zaimportowano: dotarło 0 z {read} rekordów.", + "writtenByHealthLog": "Pominięto {count} wpisów, które HealthLog sam zapisał w Apple Health, ponieważ są już w Twoich danych.", "ecgResult": "EKG: wykryto {discovered}, zaimportowano {imported}, zaktualizowano {updated}, pominięto {skipped}, nie powiodło się {failed}. Błędy EKG nie usuwają obsługiwanych rekordów zaimportowanych z pliku export.xml.", "estimateWarning": "Łączne wartości dla {count} dni oszacowano na podstawie największego źródła w eksporcie. Natywne statystyki Apple Health pozostają nadrzędne.", "phase": { diff --git a/src/components/settings/import-panel/apple-health-import-card.tsx b/src/components/settings/import-panel/apple-health-import-card.tsx index 344299c81..dbe40953c 100644 --- a/src/components/settings/import-panel/apple-health-import-card.tsx +++ b/src/components/settings/import-panel/apple-health-import-card.tsx @@ -31,6 +31,8 @@ interface JobStatus { totals?: { recordsRead?: number; rowsUpserted?: number }; clinical?: { skipped?: number }; cumulativeEstimates?: { days?: number; rows?: number }; + /** Records HealthLog wrote into Apple Health itself, left out here. */ + writtenByHealthLog?: { byMarker?: number; matchedManual?: number }; /** HK types deferred by design — read but not (yet) imported. */ deferred?: Record; /** Per-key drop counters — plain HK types are unsupported-by-design, @@ -342,6 +344,9 @@ export function AppleHealthImportCard() { const status = statusQuery.data ?? null; const isRunning = status !== null && !TERMINAL_STATES.includes(status.status); const isDone = status?.status === "done"; + const ownRecordsSkipped = + (status?.result?.writtenByHealthLog?.byMarker ?? 0) + + (status?.result?.writtenByHealthLog?.matchedManual ?? 0); const isFailed = status?.status === "failed"; const busy = uploading || isRunning; const percent = status?.progress?.percent ?? null; @@ -492,6 +497,19 @@ export function AppleHealthImportCard() { })}

)} + {isDone && ownRecordsSkipped > 0 && ( +

+ {t( + "settings.sections.export.import.appleHealth.writtenByHealthLog", + { + count: ownRecordsSkipped, + }, + )} +

+ )} {isDone && (status?.result?.cumulativeEstimates?.days ?? 0) > 0 && ( ` the mark of a sample HealthLog wrote itself? */ +export function isHealthLogOriginEntry( + key: string | undefined, + value: string | undefined, +): boolean { + return ( + key === HEALTHLOG_ORIGIN_METADATA_KEY && + value === HEALTHLOG_ORIGIN_METADATA_VALUE + ); +} diff --git a/src/lib/measurements/import-apple-health-export.ts b/src/lib/measurements/import-apple-health-export.ts index 37b5b0c35..8801e4d49 100644 --- a/src/lib/measurements/import-apple-health-export.ts +++ b/src/lib/measurements/import-apple-health-export.ts @@ -40,6 +40,11 @@ import { dayKeyForUserTz, canonicalDailyTimestamp, } from "@/lib/measurements/drain-per-sample-cumulative"; +import { isHealthLogOriginEntry } from "@/lib/apple-health/own-origin"; +import { + isSameReadingAcrossSource, + MEASURED_AT_TOLERANCE_MS, +} from "@/lib/measurements/cross-source-merge"; import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement"; import { insertNewMeasurementRows, @@ -115,6 +120,13 @@ export interface ImportJobResult { durationMs: number; }; clinical: { skipped: number }; + /** + * Records left out because HealthLog wrote them into Apple Health itself and + * already holds them under their own source: `byMarker` carry the app's + * origin marker, `matchedManual` match a manual entry (same type and value, + * within 2 s) from before the marker existed. + */ + writtenByHealthLog: { byMarker: number; matchedManual: number }; /** * v1.15.0 — reproductive HealthKit samples routed into CYCLE day-logs * (NOT Measurement). Absent / zeroed when the account has no cycle @@ -412,6 +424,11 @@ export async function streamParseExportXml( MeasurementType, Map >(); + // A record's contribution is committed at its close tag, once its child + // `` rows are read: one of them can mark the sample as written + // by HealthLog itself, which has to keep it out of the import. + let pendingRecord: { commit: () => void; ownOrigin: boolean } | null = null; + const writtenByHealthLog = { byMarker: 0, matchedManual: 0 }; // Spot-row batch awaiting flush. const spotBatch: PreparedMeasurement[] = []; // Workout-row batch awaiting flush. @@ -468,9 +485,54 @@ export async function streamParseExportXml( }; // ── Flush helpers ────────────────────────────────────────── + // A manual entry the app mirrored into Apple Health before it stamped its + // origin marker comes back in the export as a second copy of the reading. + // Leave it out when a MANUAL row of the same type and value sits within 2 s. + // Only MANUAL: Withings and import rows reach Apple Health through the app's + // mirror, which always sets the marker, so a value-and-time match against + // them would only add false positives. Same rule and tolerance as the sync + // path (`cross-source-merge.ts`). + const withoutManualMirrors = async ( + rows: PreparedMeasurement[], + ): Promise => { + if (rows.length === 0) return rows; + const candidates = await prisma.measurement.findMany({ + where: { + userId, + source: "MANUAL", + deletedAt: null, + OR: rows.map((row) => ({ + type: row.type, + measuredAt: { + gte: new Date(row.measuredAt.getTime() - MEASURED_AT_TOLERANCE_MS), + lte: new Date(row.measuredAt.getTime() + MEASURED_AT_TOLERANCE_MS), + }, + })), + }, + select: { type: true, source: true, value: true, measuredAt: true }, + }); + if (candidates.length === 0) return rows; + return rows.filter((row) => { + const mirrored = candidates.some((candidate) => + isSameReadingAcrossSource( + { + type: row.type, + source: "APPLE_HEALTH", + value: row.value, + measuredAt: row.measuredAt, + }, + candidate, + ), + ); + if (mirrored) writtenByHealthLog.matchedManual += 1; + return !mirrored; + }); + }; + const flushSpotBatch = async (): Promise => { if (spotBatch.length === 0) return; - const chunk = spotBatch.splice(0, spotBatch.length); + const incoming = spotBatch.splice(0, spotBatch.length); + const chunk = await withoutManualMirrors(incoming); for (const row of chunk) widenSpan(row.measuredAt); const insertedArrivals: Array<{ id: string; @@ -907,46 +969,51 @@ export async function streamParseExportXml( const stat = bumpStat(mapped.type); stat.read += 1; - if (CUMULATIVE_HK_TYPES.has(mapped.type)) { - const dayKey = dayKeyForUserTz(mapped.takenAt, userTimezone); - const sourceHash = hashCumulativeSourceIdentity( - attrs.sourceName, - attrs.device, - ); - let byDay = cumulativeBucket.get(mapped.type); - if (!byDay) { - byDay = new Map(); - cumulativeBucket.set(mapped.type, byDay); - } - let bySource = byDay.get(dayKey); - if (!bySource) { - bySource = new Map(); - byDay.set(dayKey, bySource); - } - bySource.set( - sourceHash, - (bySource.get(sourceHash) ?? 0) + mapped.value, - ); - } else { - // Spot row: derive a stable externalId, queue for flush. - const externalId = hashSampleKey( - hkType, - attrs.value ?? "", - attrs.startDate, - attrs.endDate, - ); - spotBatch.push({ - userId, - type: mapped.type, - value: mapped.value, - unit: mapped.unit, - measuredAt: mapped.takenAt, - externalId, - externalSourceVersion: attrs.sourceVersion ?? null, - sleepStage: mapped.sleepStage ?? null, - deviceType: null, - }); - } + pendingRecord = { + ownOrigin: false, + commit: () => { + if (CUMULATIVE_HK_TYPES.has(mapped.type)) { + const dayKey = dayKeyForUserTz(mapped.takenAt, userTimezone); + const sourceHash = hashCumulativeSourceIdentity( + attrs.sourceName, + attrs.device, + ); + let byDay = cumulativeBucket.get(mapped.type); + if (!byDay) { + byDay = new Map(); + cumulativeBucket.set(mapped.type, byDay); + } + let bySource = byDay.get(dayKey); + if (!bySource) { + bySource = new Map(); + byDay.set(dayKey, bySource); + } + bySource.set( + sourceHash, + (bySource.get(sourceHash) ?? 0) + mapped.value, + ); + } else { + // Spot row: derive a stable externalId, queue for flush. + const externalId = hashSampleKey( + hkType, + attrs.value ?? "", + attrs.startDate, + attrs.endDate, + ); + spotBatch.push({ + userId, + type: mapped.type, + value: mapped.value, + unit: mapped.unit, + measuredAt: mapped.takenAt, + externalId, + externalSourceVersion: attrs.sourceVersion ?? null, + sleepStage: mapped.sleepStage ?? null, + deviceType: null, + }); + } + }, + }; return; } @@ -1025,6 +1092,9 @@ export async function streamParseExportXml( } if (name === "MetadataEntry") { + if (pendingRecord && isHealthLogOriginEntry(attrs.key, attrs.value)) { + pendingRecord.ownOrigin = true; + } // Attach the SexualActivity protection flag to the open cycle record. // Apple writes `HKMetadataKeySexualActivityProtectionUsed` with a // `"0"`/`"1"` (or `"true"`/`"false"`) value. @@ -1070,6 +1140,12 @@ export async function streamParseExportXml( }; parser.onclosetag = async (tagName) => { + if (tagName === "Record" && pendingRecord) { + const rec = pendingRecord; + pendingRecord = null; + if (rec.ownOrigin) writtenByHealthLog.byMarker += 1; + else rec.commit(); + } if (tagName === "Workout" && currentWorkout) { workoutBatch.push(currentWorkout); currentWorkout = null; @@ -1212,6 +1288,7 @@ export async function streamParseExportXml( perType, workouts, clinical, + writtenByHealthLog, cycle, deferred, unknown, diff --git a/tests/integration/apple-health-import-own-samples.test.ts b/tests/integration/apple-health-import-own-samples.test.ts new file mode 100644 index 000000000..d97d0dda8 --- /dev/null +++ b/tests/integration/apple-health-import-own-samples.test.ts @@ -0,0 +1,186 @@ +/** + * The Apple Health export must not bring back what HealthLog wrote into Apple + * Health itself. + * + * The iOS app writes manual entries into Apple Health, and mirrors Withings and + * import rows into it, each stamped with `dev.healthlog.app.origin = healthlog`. + * The server already holds those readings under their own source, so the same + * sample arriving through an export is a second copy: the series shows it + * twice and the rollups count it twice. + * + * Asserted against a real Postgres: + * - a record carrying the marker is left out, and counted; + * - a record without it lands as before; + * - the marker is read from a child `` that follows the + * record's open tag, so the record is committed at its close tag; + * - a manual entry mirrored before the marker existed (a MANUAL row with the + * same type and value within 2 s) is left out and counted separately; + * - a MANUAL row with another value, or 3 s away, is not a match; + * - a Withings row of the same value and time is not a match either (those + * are mirrored with the marker, so a match would only add false positives). + */ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { beforeEach, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { streamParseExportXml } from "@/lib/measurements/import-apple-health-export"; + +const prisma = getPrismaClient(); + +beforeEach(async () => { + await truncateAllTables(prisma); +}); + +function writeXml(records: string): string { + const dir = mkdtempSync(join(tmpdir(), "healthlog-import-own-")); + const xmlPath = join(dir, "export.xml"); + writeFileSync( + xmlPath, + ` +]> + +${records} +`, + ); + return xmlPath; +} + +const weight = (value: number, end: string, children = "") => + ` ${children}`; +const OWN = ``; +const OTHER = ``; + +async function run(records: string) { + const user = await prisma.user.create({ + data: { username: "own-samples", email: "own@example.test", role: "USER" }, + }); + const result = await streamParseExportXml({ + xmlPath: writeXml(records), + userId: user.id, + userTimezone: "Europe/Berlin", + prisma, + }); + const rows = await prisma.measurement.findMany({ + where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" }, + orderBy: { measuredAt: "asc" }, + }); + return { user, result, rows }; +} + +describe("a record HealthLog wrote itself (the origin marker)", () => { + it("is left out and counted; an ordinary record still lands", async () => { + const { result, rows } = await run( + [ + weight(80.1, "2026-05-14 08:00:00 +0200", `\n ${OWN}\n `), + weight(80.2, "2026-05-14 09:00:00 +0200", `\n ${OTHER}\n `), + weight(80.3, "2026-05-14 10:00:00 +0200"), + ].join("\n"), + ); + expect(rows.map((r) => r.value)).toEqual([80.2, 80.3]); + expect(result.writtenByHealthLog).toEqual({ + byMarker: 1, + matchedManual: 0, + }); + }); + + it("finds the marker among other metadata entries", async () => { + const { result, rows } = await run( + weight(80.1, "2026-05-14 08:00:00 +0200", `\n ${OTHER}\n ${OWN}\n`), + ); + expect(rows).toHaveLength(0); + expect(result.writtenByHealthLog.byMarker).toBe(1); + }); + + it("does not let one record's marker spill onto the next", async () => { + const { rows } = await run( + [ + weight(80.1, "2026-05-14 08:00:00 +0200", `\n ${OWN}\n`), + weight(80.2, "2026-05-14 09:00:00 +0200"), + ].join("\n"), + ); + expect(rows.map((r) => r.value)).toEqual([80.2]); + }); + + it("needs the exact value: another origin does not count", async () => { + const { rows } = await run( + weight( + 80.1, + "2026-05-14 08:00:00 +0200", + ``, + ), + ); + expect(rows).toHaveLength(1); + }); +}); + +describe("a manual entry mirrored before the marker existed", () => { + async function runAgainst( + manual: { value: number; at: string; source?: "MANUAL" | "WITHINGS" }, + record: { value: number; at: string }, + ) { + const user = await prisma.user.create({ + data: { username: "own-fallback", email: "f@example.test", role: "USER" }, + }); + await prisma.measurement.create({ + data: { + userId: user.id, + type: "WEIGHT", + unit: "kg", + source: manual.source ?? "MANUAL", + value: manual.value, + measuredAt: new Date(manual.at), + }, + }); + const result = await streamParseExportXml({ + xmlPath: writeXml(weight(record.value, "2026-05-14 08:00:00 +0200")), + userId: user.id, + userTimezone: "Europe/Berlin", + prisma, + }); + const rows = await prisma.measurement.findMany({ + where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" }, + }); + return { result, rows }; + } + + it("is left out when a MANUAL row has the same value within 2 s", async () => { + const { result, rows } = await runAgainst( + { value: 80.4, at: "2026-05-14T06:00:01.500Z" }, + { value: 80.4, at: "" }, + ); + expect(rows).toHaveLength(0); + expect(result.writtenByHealthLog).toEqual({ + byMarker: 0, + matchedManual: 1, + }); + }); + + it("is kept when the value differs", async () => { + const { result, rows } = await runAgainst( + { value: 80.9, at: "2026-05-14T06:00:00.000Z" }, + { value: 80.4, at: "" }, + ); + expect(rows).toHaveLength(1); + expect(result.writtenByHealthLog.matchedManual).toBe(0); + }); + + it("is kept when the MANUAL row is 3 s away", async () => { + const { rows } = await runAgainst( + { value: 80.4, at: "2026-05-14T06:00:03.000Z" }, + { value: 80.4, at: "" }, + ); + expect(rows).toHaveLength(1); + }); + + it("is kept against a Withings row of the same value and time", async () => { + const { result, rows } = await runAgainst( + { value: 80.4, at: "2026-05-14T06:00:00.000Z", source: "WITHINGS" }, + { value: 80.4, at: "" }, + ); + expect(rows).toHaveLength(1); + expect(result.writtenByHealthLog.matchedManual).toBe(0); + }); +});