Merge pull request #507 from MBombeck/release/v1.28.49

release/v1.28.49 — Apple Health re-import no longer stops on a duplicate reading

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## [Unreleased]
 
+## [1.28.49] — 2026-07-16 — Apple Health re-import no longer stops on a duplicate reading
+
+Re-importing a cumulative export.zip could run for several minutes and then fail
+outright on a unique-constraint error when two readings shared the same time,
+type, and source but carried different internal ids — a single such reading
+aborted the whole import. The importer now recognises that case: it updates the
+existing reading in place (and brings back one that had been removed) instead of
+failing, and if it still can't place a single reading it skips just that one and
+carries on. A re-import completes cleanly.
+
 ## [1.28.48] — 2026-07-16 — Security follow-ups
 
 Two fixes from the campaign's security re-verification:

docs/api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: 3.1.0
 info:
   title: HealthLog API
-  version: 1.28.48
+  version: 1.28.49
   description: >-
     Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest.
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "healthlog",
-  "version": "1.28.48",
+  "version": "1.28.49",
   "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.",
   "license": "PolyForm-Noncommercial-1.0.0",
   "homepage": "https://healthlog.dev",

public/sw.js

@@ -36,7 +36,7 @@ try {
 // v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild.
 const CACHE_VERSION =
   (typeof self !== "undefined" && self.__APP_VERSION__) ||
-  /* @sw-version-fallback */ "v1.28.48";
+  /* @sw-version-fallback */ "v1.28.49";
 const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`;
 const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`;
 // v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*`

src/lib/measurements/__tests__/import-apple-health-export.test.ts

@@ -3,6 +3,7 @@ import { writeFileSync, mkdtempSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
 
+import { Prisma } from "@/generated/prisma/client";
 import {
   hashSampleKey,
   parseRecordValue,
@@ -96,6 +97,27 @@ function makeFakePrisma() {
   type Row = Record<string, unknown>;
   const measurements: Row[] = [];
   const workouts: Row[] = [];
+  let nextId = 1;
+
+  // The real `measurements` table carries a SECOND full unique index — the
+  // natural key `(userId, type, measuredAt, source, sleepStage)` (migration
+  // 0055, NULLS NOT DISTINCT) — on top of the `(userId, type, source,
+  // externalId)` one the upsert keys on. Model BOTH so a create that would
+  // collide on the natural key raises the exact P2002 the production DB does,
+  // which is what the natural-key rescue must catch. `measuredAt` compares by
+  // epoch; NULL `sleepStage` collides with NULL (NULLS NOT DISTINCT).
+  const sameMeasuredAt = (a: unknown, b: unknown): boolean =>
+    a instanceof Date && b instanceof Date
+      ? a.getTime() === b.getTime()
+      : a === b;
+  const sameStage = (a: unknown, b: unknown): boolean =>
+    (a ?? null) === (b ?? null);
+  const naturalKeyMatch = (m: Row, r: Row): boolean =>
+    m.userId === r.userId &&
+    m.type === r.type &&
+    m.source === r.source &&
+    sameMeasuredAt(m.measuredAt, r.measuredAt) &&
+    sameStage(m.sleepStage, r.sleepStage);
 
   return {
     _measurements: measurements,
@@ -137,6 +159,24 @@ function makeFakePrisma() {
           ) ?? null
         );
       },
+      // Natural-key probe used by the rescue path. Tombstone-inclusive: no
+      // `deletedAt` filter is applied here or by the caller.
+      findFirst: async ({ where }: { where: Record<string, unknown> }) => {
+        const match = measurements.find((m) => naturalKeyMatch(m, where));
+        return match ? { id: match.id } : null;
+      },
+      update: async ({
+        where,
+        data,
+      }: {
+        where: { id: unknown };
+        data: Row;
+      }) => {
+        const idx = measurements.findIndex((m) => m.id === where.id);
+        if (idx < 0) throw new Error(`no row with id ${String(where.id)}`);
+        measurements[idx] = { ...measurements[idx], ...data };
+        return measurements[idx];
+      },
       upsert: async ({
         where,
         create,
@@ -158,8 +198,17 @@ function makeFakePrisma() {
           measurements[idx] = { ...measurements[idx], ...update };
           return measurements[idx];
         }
-        measurements.push(create);
-        return create;
+        // The externalId leg missed → this is a create. Enforce the natural-key
+        // unique before inserting, exactly as Postgres would.
+        if (measurements.some((m) => naturalKeyMatch(m, create))) {
+          throw new Prisma.PrismaClientKnownRequestError(
+            "Unique constraint failed on the fields: (`user_id`,`type`,`measured_at`,`source`,`sleep_stage`)",
+            { code: "P2002", clientVersion: "test" },
+          );
+        }
+        const row = { id: `m${nextId++}`, ...create };
+        measurements.push(row);
+        return row;
       },
     },
     workout: {
@@ -398,6 +447,134 @@ describe("streamParseExportXml — end-to-end", () => {
   });
 });
 
+/**
+ * Issue #486 — two Apple samples that share the `measurements` natural key
+ * `(userId, type, measuredAt, source, sleepStage)` but carry DIFFERENT
+ * externalIds. The externalId upsert misses on the second → its create leg
+ * collides on the natural key → P2002. Before the natural-key rescue this
+ * aborted the whole import after minutes of work; after it, the run completes
+ * with the colliding row adopted in place.
+ */
+describe("streamParseExportXml — natural-key collision (issue #486)", () => {
+  // Both records share `endDate` (→ same `measuredAt`, the natural key) but
+  // differ in `startDate`/`value` (→ different `hashSampleKey` externalId).
+  function collidingExportXml(): string {
+    return `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE HealthData [<!ELEMENT HealthData (Record)*>]>
+<HealthData locale="en_US">
+  <Record type="HKQuantityTypeIdentifierBodyMass"
+          unit="kg"
+          startDate="2026-05-14 08:10:00 +0200"
+          endDate="2026-05-14 08:14:00 +0200"
+          value="78.4"
+          sourceName="Source App A"/>
+  <Record type="HKQuantityTypeIdentifierBodyMass"
+          unit="kg"
+          startDate="2026-05-14 08:12:00 +0200"
+          endDate="2026-05-14 08:14:00 +0200"
+          value="78.6"
+          sourceName="Source App B"/>
+</HealthData>`;
+  }
+
+  it("completes and adopts the colliding row instead of throwing P2002", async () => {
+    const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
+    const xmlPath = join(tmp, "export.xml");
+    writeFileSync(xmlPath, collidingExportXml());
+    const prisma = makeFakePrisma();
+
+    // On pre-fix code this rejects with the P2002 the fake raises when the
+    // second create hits the natural key; on fixed code it resolves.
+    const result = await streamParseExportXml({
+      xmlPath,
+      userId: "user-1",
+      userTimezone: "Europe/Berlin",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      prisma: prisma as any,
+    });
+
+    // Both samples were read; one row physically landed, the twin adopted it.
+    expect(result.perType.WEIGHT?.read).toBe(2);
+    expect(result.perType.WEIGHT?.inserted).toBe(1);
+    expect(result.perType.WEIGHT?.updated).toBe(1);
+
+    const weightRows = prisma._measurements.filter((m) => m.type === "WEIGHT");
+    expect(weightRows).toHaveLength(1);
+    // Last write wins: value + externalId re-keyed onto the second sample.
+    expect(weightRows[0]?.value).toBe(78.6);
+    const secondExternalId = hashSampleKey(
+      "HKQuantityTypeIdentifierBodyMass",
+      "78.6",
+      "2026-05-14 08:12:00 +0200",
+      "2026-05-14 08:14:00 +0200",
+    );
+    expect(weightRows[0]?.externalId).toBe(secondExternalId);
+
+    // The collision was rescued, not skipped.
+    expect(result.unknown["WEIGHT::natural_key_unresolved"]).toBeUndefined();
+    expect(result.unknown["WEIGHT::natural_key_rescue_failed"]).toBeUndefined();
+  });
+
+  it("re-import of overlapping data with a re-keyed externalId adopts the prior row", async () => {
+    const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
+    const prisma = makeFakePrisma();
+
+    // First import: single reading, externalId derived from its start/end.
+    const firstPath = join(tmp, "export-1.xml");
+    writeFileSync(
+      firstPath,
+      `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE HealthData [<!ELEMENT HealthData (Record)*>]>
+<HealthData locale="en_US">
+  <Record type="HKQuantityTypeIdentifierBodyMass"
+          unit="kg"
+          startDate="2026-05-14 08:10:00 +0200"
+          endDate="2026-05-14 08:14:00 +0200"
+          value="78.4"/>
+</HealthData>`,
+    );
+    const first = await streamParseExportXml({
+      xmlPath: firstPath,
+      userId: "user-1",
+      userTimezone: "Europe/Berlin",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      prisma: prisma as any,
+    });
+    expect(first.perType.WEIGHT?.inserted).toBe(1);
+    expect(prisma._measurements).toHaveLength(1);
+
+    // Re-export: SAME instant (endDate), but the sample now carries a
+    // different start window → a re-keyed externalId. The externalId upsert
+    // misses; the natural key is still held by the first row.
+    const secondPath = join(tmp, "export-2.xml");
+    writeFileSync(
+      secondPath,
+      `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE HealthData [<!ELEMENT HealthData (Record)*>]>
+<HealthData locale="en_US">
+  <Record type="HKQuantityTypeIdentifierBodyMass"
+          unit="kg"
+          startDate="2026-05-14 08:11:30 +0200"
+          endDate="2026-05-14 08:14:00 +0200"
+          value="79.0"/>
+</HealthData>`,
+    );
+    const second = await streamParseExportXml({
+      xmlPath: secondPath,
+      userId: "user-1",
+      userTimezone: "Europe/Berlin",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      prisma: prisma as any,
+    });
+
+    // No duplicate row; the prior row was adopted, counted as an update.
+    expect(second.perType.WEIGHT?.inserted).toBe(0);
+    expect(second.perType.WEIGHT?.updated).toBe(1);
+    expect(prisma._measurements).toHaveLength(1);
+    expect(prisma._measurements[0]?.value).toBe(79.0);
+  });
+});
+
 /**
  * Streaming-memory ceiling: synthesise a ~5 MB export and assert peak
  * heap stays bounded. Smaller than the §11 §5 fixture goal (50 MB) to

src/lib/measurements/import-apple-health-export.ts

@@ -374,39 +374,112 @@ export async function streamParseExportXml(
       const stat = bumpStat(row.type);
       const rowStart = Date.now();
       const exists = existingKey.has(`${row.type}::${row.externalId}`);
-      await prisma.measurement.upsert({
-        where: {
-          userId_type_source_externalId: {
+      try {
+        await prisma.measurement.upsert({
+          where: {
+            userId_type_source_externalId: {
+              userId,
+              type: row.type,
+              source: "APPLE_HEALTH",
+              externalId: row.externalId,
+            },
+          },
+          create: {
             userId,
             type: row.type,
+            value: row.value,
+            unit: row.unit,
             source: "APPLE_HEALTH",
+            measuredAt: row.measuredAt,
             externalId: row.externalId,
+            externalSourceVersion: row.externalSourceVersion,
+            sleepStage: row.sleepStage ?? null,
+            deviceType: row.deviceType,
           },
-        },
-        create: {
-          userId,
-          type: row.type,
-          value: row.value,
-          unit: row.unit,
-          source: "APPLE_HEALTH",
-          measuredAt: row.measuredAt,
-          externalId: row.externalId,
-          externalSourceVersion: row.externalSourceVersion,
-          sleepStage: row.sleepStage ?? null,
-          deviceType: row.deviceType,
-        },
-        update: {
-          value: row.value,
-          measuredAt: row.measuredAt,
-          externalSourceVersion: row.externalSourceVersion,
-          sleepStage: row.sleepStage ?? null,
-          deviceType: row.deviceType,
-        },
-      });
-      stat.durationMs += Date.now() - rowStart;
-      if (exists) stat.updated += 1;
-      else stat.inserted += 1;
-      rowsUpserted += 1;
+          update: {
+            value: row.value,
+            measuredAt: row.measuredAt,
+            externalSourceVersion: row.externalSourceVersion,
+            sleepStage: row.sleepStage ?? null,
+            deviceType: row.deviceType,
+          },
+        });
+        stat.durationMs += Date.now() - rowStart;
+        if (exists) stat.updated += 1;
+        else stat.inserted += 1;
+        rowsUpserted += 1;
+      } catch (err) {
+        // Natural-key rescue. `Measurement` carries a SECOND full unique
+        // index beyond the externalId one this upsert keys on — the natural
+        // key `(userId, type, measuredAt, source, sleepStage)` (migration
+        // 0055, NULLS NOT DISTINCT). When two Apple samples share that natural
+        // key but carry DIFFERENT externalIds — two source apps writing at the
+        // same instant, or hash-derived ids on a re-import of overlapping data
+        // — the externalId leg above MISSES (the incoming externalId isn't in
+        // the table) and the create leg then collides on the natural key →
+        // P2002. A blind throw aborts the whole import mid-run (issue #486).
+        // Instead, mirror the sync providers' natural-key rescue
+        // (`withings/sync-sleep.ts`, `google-health/sync.ts`, `fitbit/sync.ts`):
+        // probe the occupied row by its natural key — tombstone-inclusive, no
+        // `deletedAt` filter, since a soft-deleted row still holds the key and
+        // would otherwise wedge the night forever — and ADOPT it: re-key onto
+        // the incoming externalId, take the new value/unit, resurrect if it was
+        // tombstoned. A single sample must NEVER abort the run: this catch sits
+        // OUTSIDE any transaction (each upsert is its own statement), so the
+        // caught P2002 can't poison a surrounding tx.
+        if (
+          err instanceof Prisma.PrismaClientKnownRequestError &&
+          err.code === "P2002"
+        ) {
+          try {
+            const twin = await prisma.measurement.findFirst({
+              where: {
+                userId,
+                type: row.type,
+                source: "APPLE_HEALTH",
+                measuredAt: row.measuredAt,
+                sleepStage: row.sleepStage ?? null,
+              },
+              select: { id: true },
+            });
+            if (twin) {
+              await prisma.measurement.update({
+                where: { id: twin.id },
+                // `measuredAt` + `sleepStage` already match by construction
+                // (they ARE the natural key we probed on), so only value/unit,
+                // the externalId re-key, provenance, and the resurrect move.
+                data: {
+                  value: row.value,
+                  unit: row.unit,
+                  externalId: row.externalId,
+                  externalSourceVersion: row.externalSourceVersion,
+                  deviceType: row.deviceType,
+                  deletedAt: null,
+                },
+              });
+              // An adopted natural-key twin is an in-place UPDATE, not a fresh
+              // row — keep the inserted/updated split honest.
+              stat.updated += 1;
+              rowsUpserted += 1;
+            } else {
+              // P2002 with no findable natural-key twin (a race, or a collision
+              // we can't reconcile here). Skip this one sample and keep going —
+              // surfaced under `unknown` so operators can see the skip.
+              unknown[`${row.type}::natural_key_unresolved`] =
+                (unknown[`${row.type}::natural_key_unresolved`] ?? 0) + 1;
+            }
+          } catch {
+            unknown[`${row.type}::natural_key_rescue_failed`] =
+              (unknown[`${row.type}::natural_key_rescue_failed`] ?? 0) + 1;
+          }
+        } else {
+          // Any other single-row write error: skip the sample, never abort the
+          // import over one bad row.
+          unknown[`${row.type}::upsert_failed`] =
+            (unknown[`${row.type}::upsert_failed`] ?? 0) + 1;
+        }
+        stat.durationMs += Date.now() - rowStart;
+      }
     }
   };
 

tests/integration/apple-health-import-natural-key.test.ts

@@ -0,0 +1,175 @@
+/**
+ * Issue #486 — the Apple `export.zip` importer must survive two samples that
+ * share the `measurements` natural key `(userId, type, measuredAt, source,
+ * sleepStage)` (migration 0055, NULLS NOT DISTINCT) while carrying DIFFERENT
+ * externalIds. This is a real DB-constraint contract — the collision only
+ * surfaces against Postgres, not the in-memory unit fake — so it is pinned
+ * here against the testcontainer.
+ *
+ * Pre-fix: the externalId upsert misses on the second sample, its create leg
+ * collides on the natural key, Prisma throws P2002, and the whole import
+ * aborts after minutes of work. Post-fix: the natural-key rescue adopts the
+ * occupied row in place and the run completes.
+ */
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+
+import { beforeEach, describe, expect, it } from "vitest";
+
+import { getPrismaClient, truncateAllTables } from "./setup";
+import { streamParseExportXml } from "@/lib/measurements/import-apple-health-export";
+
+beforeEach(async () => {
+  await truncateAllTables(getPrismaClient());
+});
+
+async function createUser(username: string) {
+  return getPrismaClient().user.create({
+    data: { username, email: `${username}@example.test`, role: "USER" },
+  });
+}
+
+function writeXml(records: string): string {
+  const tmp = mkdtempSync(join(tmpdir(), "healthlog-import-nk-"));
+  const xmlPath = join(tmp, "export.xml");
+  writeFileSync(
+    xmlPath,
+    `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE HealthData [<!ELEMENT HealthData (Record)*>]>
+<HealthData locale="en_US">
+${records}
+</HealthData>`,
+  );
+  return xmlPath;
+}
+
+// Both records share `endDate` (→ same `measuredAt`, the natural key) but
+// differ in `startDate`/`value` (→ different `hashSampleKey` externalId).
+const COLLIDING_RECORDS = `  <Record type="HKQuantityTypeIdentifierBodyMass"
+          unit="kg"
+          startDate="2026-05-14 08:10:00 +0200"
+          endDate="2026-05-14 08:14:00 +0200"
+          value="78.4"
+          sourceName="Source App A"/>
+  <Record type="HKQuantityTypeIdentifierBodyMass"
+          unit="kg"
+          startDate="2026-05-14 08:12:00 +0200"
+          endDate="2026-05-14 08:14:00 +0200"
+          value="78.6"
+          sourceName="Source App B"/>`;
+
+describe("Apple Health import — natural-key collision (issue #486)", () => {
+  it("completes against real Postgres and lands exactly one adopted row", async () => {
+    const prisma = getPrismaClient();
+    const user = await createUser("import-nk-collide");
+
+    const result = await streamParseExportXml({
+      xmlPath: writeXml(COLLIDING_RECORDS),
+      userId: user.id,
+      userTimezone: "Europe/Berlin",
+      prisma,
+    });
+
+    expect(result.perType.WEIGHT?.read).toBe(2);
+    expect(result.perType.WEIGHT?.inserted).toBe(1);
+    expect(result.perType.WEIGHT?.updated).toBe(1);
+    expect(result.unknown["WEIGHT::natural_key_unresolved"]).toBeUndefined();
+
+    const rows = await prisma.measurement.findMany({
+      where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
+    });
+    expect(rows).toHaveLength(1);
+    // Last write wins — the second sample's value was adopted onto the row.
+    expect(rows[0]?.value).toBe(78.6);
+  });
+
+  it("re-import with a re-keyed externalId adopts the prior row, no duplicate", async () => {
+    const prisma = getPrismaClient();
+    const user = await createUser("import-nk-reimport");
+
+    const first = await streamParseExportXml({
+      xmlPath: writeXml(
+        `  <Record type="HKQuantityTypeIdentifierBodyMass"
+          unit="kg"
+          startDate="2026-05-14 08:10:00 +0200"
+          endDate="2026-05-14 08:14:00 +0200"
+          value="78.4"/>`,
+      ),
+      userId: user.id,
+      userTimezone: "Europe/Berlin",
+      prisma,
+    });
+    expect(first.perType.WEIGHT?.inserted).toBe(1);
+
+    // Re-export: same instant (endDate) but a different start window →
+    // re-keyed externalId. The externalId upsert misses; the natural key is
+    // still held by the first row.
+    const second = await streamParseExportXml({
+      xmlPath: writeXml(
+        `  <Record type="HKQuantityTypeIdentifierBodyMass"
+          unit="kg"
+          startDate="2026-05-14 08:11:30 +0200"
+          endDate="2026-05-14 08:14:00 +0200"
+          value="79.0"/>`,
+      ),
+      userId: user.id,
+      userTimezone: "Europe/Berlin",
+      prisma,
+    });
+
+    expect(second.perType.WEIGHT?.inserted).toBe(0);
+    expect(second.perType.WEIGHT?.updated).toBe(1);
+
+    const rows = await prisma.measurement.findMany({
+      where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
+    });
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.value).toBe(79.0);
+  });
+
+  it("resurrects a soft-deleted row that holds the natural key", async () => {
+    const prisma = getPrismaClient();
+    const user = await createUser("import-nk-tombstone");
+
+    // Seed a TOMBSTONED WEIGHT row that already occupies the natural key with
+    // some legacy externalId. A tombstone still holds the 0055 key, so a blind
+    // create would P2002 forever ("erased re-keyed sleep" wedge class).
+    const measuredAt = new Date("2026-05-14T06:14:00.000Z");
+    await prisma.measurement.create({
+      data: {
+        userId: user.id,
+        type: "WEIGHT",
+        value: 70.0,
+        unit: "kg",
+        source: "APPLE_HEALTH",
+        measuredAt,
+        externalId: "sample:legacy-tombstoned-id",
+        deletedAt: new Date(),
+      },
+    });
+
+    const result = await streamParseExportXml({
+      xmlPath: writeXml(
+        `  <Record type="HKQuantityTypeIdentifierBodyMass"
+          unit="kg"
+          startDate="2026-05-14 08:10:00 +0200"
+          endDate="2026-05-14 08:14:00 +0200"
+          value="80.0"/>`,
+      ),
+      userId: user.id,
+      userTimezone: "Europe/Berlin",
+      prisma,
+    });
+
+    expect(result.perType.WEIGHT?.updated).toBe(1);
+
+    const rows = await prisma.measurement.findMany({
+      where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
+    });
+    expect(rows).toHaveLength(1);
+    // Adopted: value updated, tombstone lifted (resurrected).
+    expect(rows[0]?.value).toBe(80.0);
+    expect(rows[0]?.deletedAt).toBeNull();
+  });
+});