Merge pull request #1140 from MBombeck/ready/pr-1131

fix(import): leave out Apple Health samples HealthLog wrote itself

src/components/settings/import-panel/__tests__/apple-health-own-records-note.test.tsx

@@ -0,0 +1,67 @@
+/**
+ * The note under a finished Apple Health import that says how many records
+ * were left out because HealthLog wrote them into Apple Health itself.
+ *
+ * The count is often 1, and a single string rendered "1 Einträge". The note
+ * reads the plural tier of the active locale, so each locale's own singular
+ * (and the Polish few form) reaches the screen.
+ */
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { renderToStaticMarkup } from "react-dom/server";
+
+import { I18nProvider } from "@/lib/i18n/context";
+import type { MessageBundle } from "@/lib/i18n/load-locale";
+import { locales, type Locale } from "@/lib/i18n/config";
+
+import { AppleHealthOwnRecordsNote } from "../apple-health-import-card";
+
+function bundle(locale: Locale): MessageBundle {
+  return JSON.parse(
+    readFileSync(join(process.cwd(), "messages", `${locale}.json`), "utf8"),
+  ) as MessageBundle;
+}
+
+function render(locale: Locale, count: number): string {
+  return (
+    renderToStaticMarkup(
+      <I18nProvider initialLocale={locale} initialMessages={bundle(locale)}>
+        <AppleHealthOwnRecordsNote count={count} />
+      </I18nProvider>,
+    )
+      // Text between the tags; a test reading its own static markup, not a
+      // sanitiser for untrusted input.
+      .split(/<[^>]*>/)
+      .join("")
+      .trim()
+  );
+}
+
+describe("<AppleHealthOwnRecordsNote>", () => {
+  it("uses the singular for one record", () => {
+    expect(render("de", 1)).toMatch(/^1 Eintrag,/);
+    expect(render("en", 1)).toMatch(/^1 record that/);
+    expect(render("es", 1)).toContain("1 registro que");
+    expect(render("fr", 1)).toMatch(/^1 enregistrement que/);
+    expect(render("it", 1)).toMatch(/^1 registrazione che/);
+    expect(render("pl", 1)).toContain("1 wpis,");
+  });
+
+  it("uses the plural for several, and the Polish few form for 2 to 4", () => {
+    expect(render("de", 5)).toMatch(/^5 Einträge,/);
+    expect(render("en", 5)).toMatch(/^5 records that/);
+    expect(render("pl", 3)).toContain("3 wpisy,");
+    expect(render("pl", 5)).toContain("5 wpisów,");
+  });
+
+  it("renders a sentence with the count in every locale", () => {
+    for (const locale of locales) {
+      for (const count of [1, 2, 5]) {
+        const text = render(locale, count);
+        expect(text, `${locale} ${count}`).toContain(String(count));
+        expect(text, `${locale} ${count}`).not.toContain("writtenByHealthLog");
+      }
+    }
+  });
+});

src/components/settings/import-panel/apple-health-import-card.tsx

@@ -31,6 +31,12 @@ interface JobStatus {
     totals?: { recordsRead?: number; rowsUpserted?: number };
     clinical?: { skipped?: number };
     cumulativeEstimates?: { days?: number; rows?: number };
+    /** Records HealthLog wrote into Apple Health itself, left out here. */
+    writtenByHealthLog?: {
+      byMarker?: number;
+      byExternalId?: number;
+      matchedManual?: number;
+    };
     /** HK types deferred by design — read but not (yet) imported. */
     deferred?: Record<string, number>;
     /** Per-key drop counters — plain HK types are unsupported-by-design,
@@ -247,6 +253,22 @@ export function AppleHealthSkipLines({
   );
 }
 
+/** How many records the import left out because HealthLog wrote them. */
+export function AppleHealthOwnRecordsNote({ count }: { count: number }) {
+  const { tCount } = useTranslations();
+  return (
+    <p
+      data-testid="import-apple-health-own-skipped"
+      className="text-muted-foreground text-sm"
+    >
+      {tCount(
+        "settings.sections.export.import.appleHealth.writtenByHealthLog",
+        count,
+      )}
+    </p>
+  );
+}
+
 export function AppleHealthImportCard() {
   const { t } = useTranslations();
   const fileInputRef = useRef<HTMLInputElement>(null);
@@ -342,6 +364,10 @@ export function AppleHealthImportCard() {
   const status = statusQuery.data ?? null;
   const isRunning = status !== null && !TERMINAL_STATES.includes(status.status);
   const isDone = status?.status === "done";
+  const ownRecordsSkipped =
+    (status?.result?.writtenByHealthLog?.byMarker ?? 0) +
+    (status?.result?.writtenByHealthLog?.byExternalId ?? 0) +
+    (status?.result?.writtenByHealthLog?.matchedManual ?? 0);
   const isFailed = status?.status === "failed";
   const busy = uploading || isRunning;
   const percent = status?.progress?.percent ?? null;
@@ -492,6 +518,9 @@ export function AppleHealthImportCard() {
             })}
           </p>
         )}
+        {isDone && ownRecordsSkipped > 0 && (
+          <AppleHealthOwnRecordsNote count={ownRecordsSkipped} />
+        )}
         {isDone && (status?.result?.cumulativeEstimates?.days ?? 0) > 0 && (
           <AppleHealthEstimateWarning
             days={status?.result?.cumulativeEstimates?.days ?? 0}

src/lib/apple-health/own-origin.ts

@@ -0,0 +1,32 @@
+/**
+ * The marker HealthLog's iOS app stamps on every HealthKit sample it writes
+ * itself, next to `HKExternalUUID` (the HealthLog row id): a manual entry saved
+ * from the app, and any Withings, import or manual row the server-to-Health
+ * mirror writes. Builds from before the marker existed wrote the id alone.
+ *
+ * Such a sample is already in the database under its own source. When the same
+ * sample comes back through an Apple Health export it is a second copy of that
+ * reading, so the export import leaves it out.
+ *
+ * This is not the predicate the app's own sync uses. The app decides "our own
+ * echo" from `HKExternalUUID` plus the sample's authoring source being the app
+ * itself (`HealthKitSampleOwnership.isOwnEcho`); it keeps this marker for the
+ * deletion path, where a deleted object exposes its metadata but no source. An
+ * export names its source only by a display name, so the import cannot repeat
+ * the source check and reads the marker instead, and for samples from before
+ * the marker, `HKExternalUUID` against the account's own row ids. The key and
+ * value are a stable contract with the app.
+ */
+export const HEALTHLOG_ORIGIN_METADATA_KEY = "dev.healthlog.app.origin";
+export const HEALTHLOG_ORIGIN_METADATA_VALUE = "healthlog";
+
+/** Is this `<MetadataEntry>` the mark of a sample HealthLog wrote itself? */
+export function isHealthLogOriginEntry(
+  key: string | undefined,
+  value: string | undefined,
+): boolean {
+  return (
+    key === HEALTHLOG_ORIGIN_METADATA_KEY &&
+    value === HEALTHLOG_ORIGIN_METADATA_VALUE
+  );
+}

src/lib/measurements/__tests__/import-apple-health-export.test.ts

@@ -271,6 +271,21 @@ function makeFakePrisma(
         }
         return returned;
       },
+      // The import reads once which types hold MANUAL rows.
+      groupBy: async ({
+        where,
+      }: {
+        where: { userId: string; source: string };
+      }) =>
+        [
+          ...new Set(
+            measurements
+              .filter(
+                (m) => m.userId === where.userId && m.source === where.source,
+              )
+              .map((m) => m.type),
+          ),
+        ].map((type) => ({ type })),
       findMany: async ({
         where,
       }: {

src/lib/measurements/__tests__/import-large-export-memory.test.ts

@@ -297,6 +297,7 @@ function makeDiscardingPrisma(sample: () => void) {
           externalId: row.externalId,
         }));
       },
+      groupBy: async () => [],
       findMany: async () => [],
       findFirst: async () => null,
       update: async () => {

src/lib/measurements/import-apple-health-export.ts

@@ -40,6 +40,11 @@ import {
   dayKeyForUserTz,
   canonicalDailyTimestamp,
 } from "@/lib/measurements/drain-per-sample-cumulative";
+import { isHealthLogOriginEntry } from "@/lib/apple-health/own-origin";
+import {
+  isSameReadingAcrossSource,
+  MEASURED_AT_TOLERANCE_MS,
+} from "@/lib/measurements/cross-source-merge";
 import { reconcileExternalMeasurement } from "@/lib/measurements/reconcile-external-measurement";
 import {
   insertNewMeasurementRows,
@@ -115,6 +120,19 @@ export interface ImportJobResult {
     durationMs: number;
   };
   clinical: { skipped: number };
+  /**
+   * Records left out because HealthLog wrote them into Apple Health itself and
+   * already holds them under their own source, each counted once however often
+   * the export repeats it: `byMarker` carry the app's origin marker,
+   * `byExternalId` carry a HealthLog row id as `HKExternalUUID` (samples from
+   * before the marker existed), `matchedManual` match a manual entry (same type
+   * and value, within 2 s) without either.
+   */
+  writtenByHealthLog: {
+    byMarker: number;
+    byExternalId: number;
+    matchedManual: number;
+  };
   /**
    * v1.15.0 — reproductive HealthKit samples routed into CYCLE day-logs
    * (NOT Measurement). Absent / zeroed when the account has no cycle
@@ -163,6 +181,29 @@ interface PreparedMeasurement {
   deviceType: string | null;
 }
 
+/**
+ * A quantity `<Record>` read up to its open tag. Its contribution is committed
+ * at the close tag, once its `<MetadataEntry>` children have been read.
+ */
+interface PendingRecord {
+  /** Queue the record; `healthLogId` is the `HKExternalUUID` it carried. */
+  commit: (healthLogId: string | null) => void;
+  ownOrigin: boolean;
+  healthLogId: string | null;
+  /** The record's sample identity, computed only when it is left out. */
+  sampleKey: () => string;
+}
+
+/** Apple's export name for `HKMetadataKeyExternalUUID`. */
+const HK_EXTERNAL_UUID_METADATA_KEY = "HKExternalUUID";
+
+/** The two halves of a blood pressure reading are one kind for an id match. */
+function kindOf(type: MeasurementType): string {
+  return type === "BLOOD_PRESSURE_SYS" || type === "BLOOD_PRESSURE_DIA"
+    ? "BLOOD_PRESSURE"
+    : type;
+}
+
 /** Flush row written to the `Workout` table. */
 interface PreparedWorkout {
   userId: string;
@@ -405,6 +446,8 @@ export async function streamParseExportXml(
     dayKey: string;
     rawValue: string | undefined;
     protectionUsed?: boolean;
+    sampleKey: string;
+    ownOrigin: boolean;
   } | null = null;
 
   // Cumulative fold: type -> local day -> hashed source identity -> subtotal.
@@ -412,6 +455,31 @@ export async function streamParseExportXml(
     MeasurementType,
     Map<string, CumulativeSourceSubtotals>
   >();
+  // A record's contribution is committed at its close tag, once its child
+  // `<MetadataEntry>` rows are read: one of them can mark the sample as written
+  // by HealthLog itself, which has to keep it out of the import.
+  let pendingRecord: PendingRecord | null = null;
+  const writtenByHealthLog = { byMarker: 0, byExternalId: 0, matchedManual: 0 };
+  // An export can carry the same sample twice (a correlation's records also
+  // stand on their own), so a left-out sample is counted by its sample key.
+  const countedOwn = new Set<string>();
+  const countOwn = (
+    reason: keyof typeof writtenByHealthLog,
+    sampleKey: string,
+  ): void => {
+    if (countedOwn.has(sampleKey)) return;
+    countedOwn.add(sampleKey);
+    writtenByHealthLog[reason] += 1;
+  };
+  // The `HKExternalUUID` a queued spot row carried, checked at flush.
+  const healthLogIdOf = new Map<PreparedMeasurement, string>();
+  // A `<Correlation>` (a blood pressure reading) wraps its records, and the
+  // marker may sit on the correlation itself, before or after them. Its records
+  // are held until it closes, then all kept or all left out together.
+  let currentCorrelation: {
+    ownOrigin: boolean;
+    held: PendingRecord[];
+  } | null = null;
   // Spot-row batch awaiting flush.
   const spotBatch: PreparedMeasurement[] = [];
   // Workout-row batch awaiting flush.
@@ -468,9 +536,134 @@ export async function streamParseExportXml(
   };
 
   // ── Flush helpers ──────────────────────────────────────────
+  // A sample the app wrote before it stamped its origin marker still carries
+  // the HealthLog row id as `HKExternalUUID`. A hit on one of this account's
+  // rows (deleted ones included: the sample is ours either way) of the same
+  // kind is an exact match. Blood pressure is one row per half, while the app
+  // stamps both samples of a reading with one id (the systolic row's, the
+  // diastolic row's in older builds), so either half matches either id.
+  const withoutHealthLogIds = async (
+    rows: PreparedMeasurement[],
+  ): Promise<PreparedMeasurement[]> => {
+    const ids = new Set<string>();
+    for (const row of rows) {
+      const id = healthLogIdOf.get(row);
+      if (id) ids.add(id);
+    }
+    if (ids.size === 0) return rows;
+    const known = await prisma.measurement.findMany({
+      where: { userId, id: { in: [...ids] } },
+      select: { id: true, type: true },
+    });
+    if (known.length === 0) return rows;
+    const typeOf = new Map(known.map((row) => [row.id, row.type]));
+    return rows.filter((row) => {
+      const id = healthLogIdOf.get(row);
+      const rowType = id ? typeOf.get(id) : undefined;
+      if (!rowType || kindOf(rowType) !== kindOf(row.type)) return true;
+      countOwn("byExternalId", row.externalId);
+      return false;
+    });
+  };
+
+  // A manual entry mirrored with neither the marker nor an id comes back as a
+  // second copy of the reading. Leave it out when a MANUAL row of the same
+  // type and value sits within 2 s. Only MANUAL: Withings and import rows are
+  // written into Apple Health by the server-to-Health mirror, which has always
+  // stamped the row id, so a value-and-time match against them would only add
+  // false positives. Same rule and tolerance as the sync path
+  // (`cross-source-merge.ts`).
+  //
+  // The types that hold MANUAL rows are read once per import, so a batch of a
+  // type the account never typed in asks nothing, and a batch that does asks
+  // one time window per type rather than one clause per row.
+  let manualTypes: Promise<Set<MeasurementType>> | null = null;
+  const withoutManualMirrors = async (
+    rows: PreparedMeasurement[],
+  ): Promise<PreparedMeasurement[]> => {
+    if (rows.length === 0) return rows;
+    manualTypes ??= prisma.measurement
+      .groupBy({
+        by: ["type"],
+        where: { userId, source: "MANUAL", deletedAt: null },
+      })
+      .then((groups) => new Set(groups.map((group) => group.type)));
+    const typed = await manualTypes;
+    const windows = new Map<MeasurementType, { from: number; to: number }>();
+    for (const row of rows) {
+      if (!typed.has(row.type)) continue;
+      const at = row.measuredAt.getTime();
+      const window = windows.get(row.type);
+      if (!window) windows.set(row.type, { from: at, to: at });
+      else {
+        if (at < window.from) window.from = at;
+        if (at > window.to) window.to = at;
+      }
+    }
+    if (windows.size === 0) return rows;
+    const candidates = await prisma.measurement.findMany({
+      where: {
+        userId,
+        source: "MANUAL",
+        deletedAt: null,
+        OR: [...windows].map(([type, window]) => ({
+          type,
+          measuredAt: {
+            gte: new Date(window.from - MEASURED_AT_TOLERANCE_MS),
+            lte: new Date(window.to + MEASURED_AT_TOLERANCE_MS),
+          },
+        })),
+      },
+      select: { type: true, source: true, value: true, measuredAt: true },
+      orderBy: { measuredAt: "asc" },
+    });
+    if (candidates.length === 0) return rows;
+    const byType = new Map<MeasurementType, typeof candidates>();
+    for (const candidate of candidates) {
+      const list = byType.get(candidate.type);
+      if (list) list.push(candidate);
+      else byType.set(candidate.type, [candidate]);
+    }
+    return rows.filter((row) => {
+      const list = byType.get(row.type);
+      if (!list) return true;
+      const at = row.measuredAt.getTime();
+      // First candidate not earlier than the tolerance window.
+      let lo = 0;
+      let hi = list.length;
+      while (lo < hi) {
+        const mid = (lo + hi) >> 1;
+        if (list[mid].measuredAt.getTime() < at - MEASURED_AT_TOLERANCE_MS)
+          lo = mid + 1;
+        else hi = mid;
+      }
+      for (let i = lo; i < list.length; i++) {
+        if (list[i].measuredAt.getTime() > at + MEASURED_AT_TOLERANCE_MS) break;
+        const mirrored = isSameReadingAcrossSource(
+          {
+            type: row.type,
+            source: "APPLE_HEALTH",
+            value: row.value,
+            measuredAt: row.measuredAt,
+          },
+          list[i],
+        );
+        if (mirrored) {
+          countOwn("matchedManual", row.externalId);
+          return false;
+        }
+      }
+      return true;
+    });
+  };
+
   const flushSpotBatch = async (): Promise<void> => {
     if (spotBatch.length === 0) return;
-    const chunk = spotBatch.splice(0, spotBatch.length);
+    const incoming = spotBatch.splice(0, spotBatch.length);
+    const chunk = await withoutManualMirrors(
+      await withoutHealthLogIds(incoming),
+    );
+    for (const row of incoming) healthLogIdOf.delete(row);
     for (const row of chunk) widenSpan(row.measuredAt);
     const insertedArrivals: Array<{
       id: string;
@@ -847,7 +1040,18 @@ export async function streamParseExportXml(
           Date.parse(attrs.endDate ?? attrs.startDate ?? ""),
         )
           ? null
-          : { hkType, dayKey, rawValue: attrs.value };
+          : {
+              hkType,
+              dayKey,
+              rawValue: attrs.value,
+              sampleKey: hashSampleKey(
+                hkType,
+                attrs.value ?? "",
+                attrs.startDate,
+                attrs.endDate,
+              ),
+              ownOrigin: false,
+            };
         return;
       }
 
@@ -907,46 +1111,61 @@ export async function streamParseExportXml(
       const stat = bumpStat(mapped.type);
       stat.read += 1;
 
-      if (CUMULATIVE_HK_TYPES.has(mapped.type)) {
-        const dayKey = dayKeyForUserTz(mapped.takenAt, userTimezone);
-        const sourceHash = hashCumulativeSourceIdentity(
-          attrs.sourceName,
-          attrs.device,
-        );
-        let byDay = cumulativeBucket.get(mapped.type);
-        if (!byDay) {
-          byDay = new Map();
-          cumulativeBucket.set(mapped.type, byDay);
-        }
-        let bySource = byDay.get(dayKey);
-        if (!bySource) {
-          bySource = new Map();
-          byDay.set(dayKey, bySource);
-        }
-        bySource.set(
-          sourceHash,
-          (bySource.get(sourceHash) ?? 0) + mapped.value,
-        );
-      } else {
-        // Spot row: derive a stable externalId, queue for flush.
-        const externalId = hashSampleKey(
-          hkType,
-          attrs.value ?? "",
-          attrs.startDate,
-          attrs.endDate,
-        );
-        spotBatch.push({
-          userId,
-          type: mapped.type,
-          value: mapped.value,
-          unit: mapped.unit,
-          measuredAt: mapped.takenAt,
-          externalId,
-          externalSourceVersion: attrs.sourceVersion ?? null,
-          sleepStage: mapped.sleepStage ?? null,
-          deviceType: null,
-        });
-      }
+      pendingRecord = {
+        ownOrigin: false,
+        healthLogId: null,
+        sampleKey: () =>
+          hashSampleKey(
+            hkType,
+            attrs.value ?? "",
+            attrs.startDate,
+            attrs.endDate,
+          ),
+        commit: (healthLogId) => {
+          if (CUMULATIVE_HK_TYPES.has(mapped.type)) {
+            const dayKey = dayKeyForUserTz(mapped.takenAt, userTimezone);
+            const sourceHash = hashCumulativeSourceIdentity(
+              attrs.sourceName,
+              attrs.device,
+            );
+            let byDay = cumulativeBucket.get(mapped.type);
+            if (!byDay) {
+              byDay = new Map();
+              cumulativeBucket.set(mapped.type, byDay);
+            }
+            let bySource = byDay.get(dayKey);
+            if (!bySource) {
+              bySource = new Map();
+              byDay.set(dayKey, bySource);
+            }
+            bySource.set(
+              sourceHash,
+              (bySource.get(sourceHash) ?? 0) + mapped.value,
+            );
+          } else {
+            // Spot row: derive a stable externalId, queue for flush.
+            const externalId = hashSampleKey(
+              hkType,
+              attrs.value ?? "",
+              attrs.startDate,
+              attrs.endDate,
+            );
+            const row: PreparedMeasurement = {
+              userId,
+              type: mapped.type,
+              value: mapped.value,
+              unit: mapped.unit,
+              measuredAt: mapped.takenAt,
+              externalId,
+              externalSourceVersion: attrs.sourceVersion ?? null,
+              sleepStage: mapped.sleepStage ?? null,
+              deviceType: null,
+            };
+            spotBatch.push(row);
+            if (healthLogId) healthLogIdOf.set(row, healthLogId);
+          }
+        },
+      };
       return;
     }
 
@@ -1024,7 +1243,21 @@ export async function streamParseExportXml(
       return;
     }
 
+    if (name === "Correlation") {
+      currentCorrelation = { ownOrigin: false, held: [] };
+      return;
+    }
+
     if (name === "MetadataEntry") {
+      if (isHealthLogOriginEntry(attrs.key, attrs.value)) {
+        // On a record, or on the correlation that wraps it.
+        if (pendingRecord) pendingRecord.ownOrigin = true;
+        else if (currentCycleRecord) currentCycleRecord.ownOrigin = true;
+        else if (currentCorrelation) currentCorrelation.ownOrigin = true;
+      }
+      if (pendingRecord && attrs.key === HK_EXTERNAL_UUID_METADATA_KEY) {
+        pendingRecord.healthLogId = attrs.value || null;
+      }
       // Attach the SexualActivity protection flag to the open cycle record.
       // Apple writes `HKMetadataKeySexualActivityProtectionUsed` with a
       // `"0"`/`"1"` (or `"true"`/`"false"`) value.
@@ -1051,16 +1284,15 @@ export async function streamParseExportXml(
 
     if (
       name === "Me" ||
-      name === "Correlation" ||
       name === "ActivitySummary" ||
       name === "WorkoutEvent" ||
       name === "WorkoutRoute" ||
       name === "FileReference" ||
       name === "HealthData"
     ) {
-      // Known elements we intentionally ignore at the open-tag stage.
-      // Correlation envelopes flatten naturally because their child
-      // `<Record>` elements still fire their own `onopentag`.
+      // Known elements we intentionally ignore at the open-tag stage. A
+      // `<Correlation>` is handled above: its child `<Record>` elements fire
+      // their own `onopentag`, and are held until it closes.
       return;
     }
 
@@ -1070,6 +1302,24 @@ export async function streamParseExportXml(
   };
 
   parser.onclosetag = async (tagName) => {
+    if (tagName === "Record" && pendingRecord) {
+      const rec = pendingRecord;
+      pendingRecord = null;
+      if (currentCorrelation) currentCorrelation.held.push(rec);
+      else if (rec.ownOrigin) countOwn("byMarker", rec.sampleKey());
+      else rec.commit(rec.healthLogId);
+    }
+    if (tagName === "Correlation" && currentCorrelation) {
+      const correlation = currentCorrelation;
+      currentCorrelation = null;
+      for (const rec of correlation.held) {
+        if (correlation.ownOrigin || rec.ownOrigin) {
+          countOwn("byMarker", rec.sampleKey());
+        } else {
+          rec.commit(rec.healthLogId);
+        }
+      }
+    }
     if (tagName === "Workout" && currentWorkout) {
       workoutBatch.push(currentWorkout);
       currentWorkout = null;
@@ -1077,6 +1327,11 @@ export async function streamParseExportXml(
     if (tagName === "Record" && currentCycleRecord) {
       const rec = currentCycleRecord;
       currentCycleRecord = null;
+      if (rec.ownOrigin) {
+        // A day HealthLog mirrored into Apple Health from its own day-log.
+        countOwn("byMarker", rec.sampleKey);
+        return;
+      }
       const consumed = cycleAccumulator.consume(
         rec.hkType,
         rec.dayKey,
@@ -1212,6 +1467,7 @@ export async function streamParseExportXml(
     perType,
     workouts,
     clinical,
+    writtenByHealthLog,
     cycle,
     deferred,
     unknown,