diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a003e116..acd329079 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## [Unreleased] +## [1.28.49] — 2026-07-16 — Apple Health re-import no longer stops on a duplicate reading + +Re-importing a cumulative export.zip could run for several minutes and then fail +outright on a unique-constraint error when two readings shared the same time, +type, and source but carried different internal ids — a single such reading +aborted the whole import. The importer now recognises that case: it updates the +existing reading in place (and brings back one that had been removed) instead of +failing, and if it still can't place a single reading it skips just that one and +carries on. A re-import completes cleanly. + ## [1.28.48] — 2026-07-16 — Security follow-ups Two fixes from the campaign's security re-verification: diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index e0a47af35..ba918fbbd 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: HealthLog API - version: 1.28.48 + version: 1.28.49 description: >- Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest. diff --git a/package.json b/package.json index 8bd4cbbee..5d0224cd0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.28.48", + "version": "1.28.49", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "PolyForm-Noncommercial-1.0.0", "homepage": "https://healthlog.dev", diff --git a/public/sw.js b/public/sw.js index 61291c47b..0ceaa7399 100644 --- a/public/sw.js +++ b/public/sw.js @@ -36,7 +36,7 @@ try { // v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild. const CACHE_VERSION = (typeof self !== "undefined" && self.__APP_VERSION__) || - /* @sw-version-fallback */ "v1.28.48"; + /* @sw-version-fallback */ "v1.28.49"; const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`; const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`; // v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*` diff --git a/src/lib/measurements/__tests__/import-apple-health-export.test.ts b/src/lib/measurements/__tests__/import-apple-health-export.test.ts index 39469d21f..3cb28f5f3 100644 --- a/src/lib/measurements/__tests__/import-apple-health-export.test.ts +++ b/src/lib/measurements/__tests__/import-apple-health-export.test.ts @@ -3,6 +3,7 @@ import { writeFileSync, mkdtempSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; +import { Prisma } from "@/generated/prisma/client"; import { hashSampleKey, parseRecordValue, @@ -96,6 +97,27 @@ function makeFakePrisma() { type Row = Record; const measurements: Row[] = []; const workouts: Row[] = []; + let nextId = 1; + + // The real `measurements` table carries a SECOND full unique index — the + // natural key `(userId, type, measuredAt, source, sleepStage)` (migration + // 0055, NULLS NOT DISTINCT) — on top of the `(userId, type, source, + // externalId)` one the upsert keys on. Model BOTH so a create that would + // collide on the natural key raises the exact P2002 the production DB does, + // which is what the natural-key rescue must catch. `measuredAt` compares by + // epoch; NULL `sleepStage` collides with NULL (NULLS NOT DISTINCT). + const sameMeasuredAt = (a: unknown, b: unknown): boolean => + a instanceof Date && b instanceof Date + ? a.getTime() === b.getTime() + : a === b; + const sameStage = (a: unknown, b: unknown): boolean => + (a ?? null) === (b ?? null); + const naturalKeyMatch = (m: Row, r: Row): boolean => + m.userId === r.userId && + m.type === r.type && + m.source === r.source && + sameMeasuredAt(m.measuredAt, r.measuredAt) && + sameStage(m.sleepStage, r.sleepStage); return { _measurements: measurements, @@ -137,6 +159,24 @@ function makeFakePrisma() { ) ?? null ); }, + // Natural-key probe used by the rescue path. Tombstone-inclusive: no + // `deletedAt` filter is applied here or by the caller. + findFirst: async ({ where }: { where: Record }) => { + const match = measurements.find((m) => naturalKeyMatch(m, where)); + return match ? { id: match.id } : null; + }, + update: async ({ + where, + data, + }: { + where: { id: unknown }; + data: Row; + }) => { + const idx = measurements.findIndex((m) => m.id === where.id); + if (idx < 0) throw new Error(`no row with id ${String(where.id)}`); + measurements[idx] = { ...measurements[idx], ...data }; + return measurements[idx]; + }, upsert: async ({ where, create, @@ -158,8 +198,17 @@ function makeFakePrisma() { measurements[idx] = { ...measurements[idx], ...update }; return measurements[idx]; } - measurements.push(create); - return create; + // The externalId leg missed → this is a create. Enforce the natural-key + // unique before inserting, exactly as Postgres would. + if (measurements.some((m) => naturalKeyMatch(m, create))) { + throw new Prisma.PrismaClientKnownRequestError( + "Unique constraint failed on the fields: (`user_id`,`type`,`measured_at`,`source`,`sleep_stage`)", + { code: "P2002", clientVersion: "test" }, + ); + } + const row = { id: `m${nextId++}`, ...create }; + measurements.push(row); + return row; }, }, workout: { @@ -398,6 +447,134 @@ describe("streamParseExportXml — end-to-end", () => { }); }); +/** + * Issue #486 — two Apple samples that share the `measurements` natural key + * `(userId, type, measuredAt, source, sleepStage)` but carry DIFFERENT + * externalIds. The externalId upsert misses on the second → its create leg + * collides on the natural key → P2002. Before the natural-key rescue this + * aborted the whole import after minutes of work; after it, the run completes + * with the colliding row adopted in place. + */ +describe("streamParseExportXml — natural-key collision (issue #486)", () => { + // Both records share `endDate` (→ same `measuredAt`, the natural key) but + // differ in `startDate`/`value` (→ different `hashSampleKey` externalId). + function collidingExportXml(): string { + return ` +]> + + + +`; + } + + it("completes and adopts the colliding row instead of throwing P2002", async () => { + const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-")); + const xmlPath = join(tmp, "export.xml"); + writeFileSync(xmlPath, collidingExportXml()); + const prisma = makeFakePrisma(); + + // On pre-fix code this rejects with the P2002 the fake raises when the + // second create hits the natural key; on fixed code it resolves. + const result = await streamParseExportXml({ + xmlPath, + userId: "user-1", + userTimezone: "Europe/Berlin", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prisma: prisma as any, + }); + + // Both samples were read; one row physically landed, the twin adopted it. + expect(result.perType.WEIGHT?.read).toBe(2); + expect(result.perType.WEIGHT?.inserted).toBe(1); + expect(result.perType.WEIGHT?.updated).toBe(1); + + const weightRows = prisma._measurements.filter((m) => m.type === "WEIGHT"); + expect(weightRows).toHaveLength(1); + // Last write wins: value + externalId re-keyed onto the second sample. + expect(weightRows[0]?.value).toBe(78.6); + const secondExternalId = hashSampleKey( + "HKQuantityTypeIdentifierBodyMass", + "78.6", + "2026-05-14 08:12:00 +0200", + "2026-05-14 08:14:00 +0200", + ); + expect(weightRows[0]?.externalId).toBe(secondExternalId); + + // The collision was rescued, not skipped. + expect(result.unknown["WEIGHT::natural_key_unresolved"]).toBeUndefined(); + expect(result.unknown["WEIGHT::natural_key_rescue_failed"]).toBeUndefined(); + }); + + it("re-import of overlapping data with a re-keyed externalId adopts the prior row", async () => { + const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-")); + const prisma = makeFakePrisma(); + + // First import: single reading, externalId derived from its start/end. + const firstPath = join(tmp, "export-1.xml"); + writeFileSync( + firstPath, + ` +]> + + +`, + ); + const first = await streamParseExportXml({ + xmlPath: firstPath, + userId: "user-1", + userTimezone: "Europe/Berlin", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prisma: prisma as any, + }); + expect(first.perType.WEIGHT?.inserted).toBe(1); + expect(prisma._measurements).toHaveLength(1); + + // Re-export: SAME instant (endDate), but the sample now carries a + // different start window → a re-keyed externalId. The externalId upsert + // misses; the natural key is still held by the first row. + const secondPath = join(tmp, "export-2.xml"); + writeFileSync( + secondPath, + ` +]> + + +`, + ); + const second = await streamParseExportXml({ + xmlPath: secondPath, + userId: "user-1", + userTimezone: "Europe/Berlin", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prisma: prisma as any, + }); + + // No duplicate row; the prior row was adopted, counted as an update. + expect(second.perType.WEIGHT?.inserted).toBe(0); + expect(second.perType.WEIGHT?.updated).toBe(1); + expect(prisma._measurements).toHaveLength(1); + expect(prisma._measurements[0]?.value).toBe(79.0); + }); +}); + /** * Streaming-memory ceiling: synthesise a ~5 MB export and assert peak * heap stays bounded. Smaller than the §11 §5 fixture goal (50 MB) to diff --git a/src/lib/measurements/import-apple-health-export.ts b/src/lib/measurements/import-apple-health-export.ts index c1c508f17..54199cc01 100644 --- a/src/lib/measurements/import-apple-health-export.ts +++ b/src/lib/measurements/import-apple-health-export.ts @@ -374,39 +374,112 @@ export async function streamParseExportXml( const stat = bumpStat(row.type); const rowStart = Date.now(); const exists = existingKey.has(`${row.type}::${row.externalId}`); - await prisma.measurement.upsert({ - where: { - userId_type_source_externalId: { + try { + await prisma.measurement.upsert({ + where: { + userId_type_source_externalId: { + userId, + type: row.type, + source: "APPLE_HEALTH", + externalId: row.externalId, + }, + }, + create: { userId, type: row.type, + value: row.value, + unit: row.unit, source: "APPLE_HEALTH", + measuredAt: row.measuredAt, externalId: row.externalId, + externalSourceVersion: row.externalSourceVersion, + sleepStage: row.sleepStage ?? null, + deviceType: row.deviceType, }, - }, - create: { - userId, - type: row.type, - value: row.value, - unit: row.unit, - source: "APPLE_HEALTH", - measuredAt: row.measuredAt, - externalId: row.externalId, - externalSourceVersion: row.externalSourceVersion, - sleepStage: row.sleepStage ?? null, - deviceType: row.deviceType, - }, - update: { - value: row.value, - measuredAt: row.measuredAt, - externalSourceVersion: row.externalSourceVersion, - sleepStage: row.sleepStage ?? null, - deviceType: row.deviceType, - }, - }); - stat.durationMs += Date.now() - rowStart; - if (exists) stat.updated += 1; - else stat.inserted += 1; - rowsUpserted += 1; + update: { + value: row.value, + measuredAt: row.measuredAt, + externalSourceVersion: row.externalSourceVersion, + sleepStage: row.sleepStage ?? null, + deviceType: row.deviceType, + }, + }); + stat.durationMs += Date.now() - rowStart; + if (exists) stat.updated += 1; + else stat.inserted += 1; + rowsUpserted += 1; + } catch (err) { + // Natural-key rescue. `Measurement` carries a SECOND full unique + // index beyond the externalId one this upsert keys on — the natural + // key `(userId, type, measuredAt, source, sleepStage)` (migration + // 0055, NULLS NOT DISTINCT). When two Apple samples share that natural + // key but carry DIFFERENT externalIds — two source apps writing at the + // same instant, or hash-derived ids on a re-import of overlapping data + // — the externalId leg above MISSES (the incoming externalId isn't in + // the table) and the create leg then collides on the natural key → + // P2002. A blind throw aborts the whole import mid-run (issue #486). + // Instead, mirror the sync providers' natural-key rescue + // (`withings/sync-sleep.ts`, `google-health/sync.ts`, `fitbit/sync.ts`): + // probe the occupied row by its natural key — tombstone-inclusive, no + // `deletedAt` filter, since a soft-deleted row still holds the key and + // would otherwise wedge the night forever — and ADOPT it: re-key onto + // the incoming externalId, take the new value/unit, resurrect if it was + // tombstoned. A single sample must NEVER abort the run: this catch sits + // OUTSIDE any transaction (each upsert is its own statement), so the + // caught P2002 can't poison a surrounding tx. + if ( + err instanceof Prisma.PrismaClientKnownRequestError && + err.code === "P2002" + ) { + try { + const twin = await prisma.measurement.findFirst({ + where: { + userId, + type: row.type, + source: "APPLE_HEALTH", + measuredAt: row.measuredAt, + sleepStage: row.sleepStage ?? null, + }, + select: { id: true }, + }); + if (twin) { + await prisma.measurement.update({ + where: { id: twin.id }, + // `measuredAt` + `sleepStage` already match by construction + // (they ARE the natural key we probed on), so only value/unit, + // the externalId re-key, provenance, and the resurrect move. + data: { + value: row.value, + unit: row.unit, + externalId: row.externalId, + externalSourceVersion: row.externalSourceVersion, + deviceType: row.deviceType, + deletedAt: null, + }, + }); + // An adopted natural-key twin is an in-place UPDATE, not a fresh + // row — keep the inserted/updated split honest. + stat.updated += 1; + rowsUpserted += 1; + } else { + // P2002 with no findable natural-key twin (a race, or a collision + // we can't reconcile here). Skip this one sample and keep going — + // surfaced under `unknown` so operators can see the skip. + unknown[`${row.type}::natural_key_unresolved`] = + (unknown[`${row.type}::natural_key_unresolved`] ?? 0) + 1; + } + } catch { + unknown[`${row.type}::natural_key_rescue_failed`] = + (unknown[`${row.type}::natural_key_rescue_failed`] ?? 0) + 1; + } + } else { + // Any other single-row write error: skip the sample, never abort the + // import over one bad row. + unknown[`${row.type}::upsert_failed`] = + (unknown[`${row.type}::upsert_failed`] ?? 0) + 1; + } + stat.durationMs += Date.now() - rowStart; + } } }; diff --git a/tests/integration/apple-health-import-natural-key.test.ts b/tests/integration/apple-health-import-natural-key.test.ts new file mode 100644 index 000000000..2683ab4a4 --- /dev/null +++ b/tests/integration/apple-health-import-natural-key.test.ts @@ -0,0 +1,175 @@ +/** + * Issue #486 — the Apple `export.zip` importer must survive two samples that + * share the `measurements` natural key `(userId, type, measuredAt, source, + * sleepStage)` (migration 0055, NULLS NOT DISTINCT) while carrying DIFFERENT + * externalIds. This is a real DB-constraint contract — the collision only + * surfaces against Postgres, not the in-memory unit fake — so it is pinned + * here against the testcontainer. + * + * Pre-fix: the externalId upsert misses on the second sample, its create leg + * collides on the natural key, Prisma throws P2002, and the whole import + * aborts after minutes of work. Post-fix: the natural-key rescue adopts the + * occupied row in place and the run completes. + */ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +import { beforeEach, describe, expect, it } from "vitest"; + +import { getPrismaClient, truncateAllTables } from "./setup"; +import { streamParseExportXml } from "@/lib/measurements/import-apple-health-export"; + +beforeEach(async () => { + await truncateAllTables(getPrismaClient()); +}); + +async function createUser(username: string) { + return getPrismaClient().user.create({ + data: { username, email: `${username}@example.test`, role: "USER" }, + }); +} + +function writeXml(records: string): string { + const tmp = mkdtempSync(join(tmpdir(), "healthlog-import-nk-")); + const xmlPath = join(tmp, "export.xml"); + writeFileSync( + xmlPath, + ` +]> + +${records} +`, + ); + return xmlPath; +} + +// Both records share `endDate` (→ same `measuredAt`, the natural key) but +// differ in `startDate`/`value` (→ different `hashSampleKey` externalId). +const COLLIDING_RECORDS = ` + `; + +describe("Apple Health import — natural-key collision (issue #486)", () => { + it("completes against real Postgres and lands exactly one adopted row", async () => { + const prisma = getPrismaClient(); + const user = await createUser("import-nk-collide"); + + const result = await streamParseExportXml({ + xmlPath: writeXml(COLLIDING_RECORDS), + userId: user.id, + userTimezone: "Europe/Berlin", + prisma, + }); + + expect(result.perType.WEIGHT?.read).toBe(2); + expect(result.perType.WEIGHT?.inserted).toBe(1); + expect(result.perType.WEIGHT?.updated).toBe(1); + expect(result.unknown["WEIGHT::natural_key_unresolved"]).toBeUndefined(); + + const rows = await prisma.measurement.findMany({ + where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" }, + }); + expect(rows).toHaveLength(1); + // Last write wins — the second sample's value was adopted onto the row. + expect(rows[0]?.value).toBe(78.6); + }); + + it("re-import with a re-keyed externalId adopts the prior row, no duplicate", async () => { + const prisma = getPrismaClient(); + const user = await createUser("import-nk-reimport"); + + const first = await streamParseExportXml({ + xmlPath: writeXml( + ` `, + ), + userId: user.id, + userTimezone: "Europe/Berlin", + prisma, + }); + expect(first.perType.WEIGHT?.inserted).toBe(1); + + // Re-export: same instant (endDate) but a different start window → + // re-keyed externalId. The externalId upsert misses; the natural key is + // still held by the first row. + const second = await streamParseExportXml({ + xmlPath: writeXml( + ` `, + ), + userId: user.id, + userTimezone: "Europe/Berlin", + prisma, + }); + + expect(second.perType.WEIGHT?.inserted).toBe(0); + expect(second.perType.WEIGHT?.updated).toBe(1); + + const rows = await prisma.measurement.findMany({ + where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" }, + }); + expect(rows).toHaveLength(1); + expect(rows[0]?.value).toBe(79.0); + }); + + it("resurrects a soft-deleted row that holds the natural key", async () => { + const prisma = getPrismaClient(); + const user = await createUser("import-nk-tombstone"); + + // Seed a TOMBSTONED WEIGHT row that already occupies the natural key with + // some legacy externalId. A tombstone still holds the 0055 key, so a blind + // create would P2002 forever ("erased re-keyed sleep" wedge class). + const measuredAt = new Date("2026-05-14T06:14:00.000Z"); + await prisma.measurement.create({ + data: { + userId: user.id, + type: "WEIGHT", + value: 70.0, + unit: "kg", + source: "APPLE_HEALTH", + measuredAt, + externalId: "sample:legacy-tombstoned-id", + deletedAt: new Date(), + }, + }); + + const result = await streamParseExportXml({ + xmlPath: writeXml( + ` `, + ), + userId: user.id, + userTimezone: "Europe/Berlin", + prisma, + }); + + expect(result.perType.WEIGHT?.updated).toBe(1); + + const rows = await prisma.measurement.findMany({ + where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" }, + }); + expect(rows).toHaveLength(1); + // Adopted: value updated, tombstone lifted (resurrected). + expect(rows[0]?.value).toBe(80.0); + expect(rows[0]?.deletedAt).toBeNull(); + }); +});