chore(release): v1.28.49 — Apple Health re-import survives a natural-key collision

The Apple Health ZIP import upserted each spot reading on the
(userId, type, source, externalId) index only. When two readings shared the
natural key (userId, type, measuredAt, source, sleepStage) with different
externalIds — a re-import of overlapping data, or two same-instant Apple
samples from different source apps — the externalId upsert missed and the
create collided on the second unique index (migration 0055), raising P2002
and aborting the entire import (issue #486, after the v1.28.36 dedup fix let
the upload run).

flushSpotBatch now wraps the per-row upsert: on P2002 it probes the occupying
row by the natural key (tombstone-inclusive) and adopts it in place — re-keys
to the incoming externalId, takes the new value/unit, resurrects if
soft-deleted — mirroring the v1.28.39 provider natural-key rescues (closest to
withings/sync-sleep). If no twin resolves it, the one reading is skipped and
the run continues; the catch sits in the per-row loop outside any transaction,
so a collision can never abort the batch or the import. Adopted rows count as
updated, skipped ones surface honestly. Workout + stats: paths verified unable
to hit this (no second unique index / deterministic externalId). Repro +
real-Postgres integration tests added.
This commit is contained in:
Marc-André Bombeck committed 2026-07-16 08:48:41 +02:00
1 parent a523d2cc79
commit 0f4e0be1a6
7 files changed
+468 -33

No files matched your search

+10
View File
@@ -2,6 +2,16 @@
## [Unreleased]
## [1.28.49] — 2026-07-16 — Apple Health re-import no longer stops on a duplicate reading
Re-importing a cumulative export.zip could run for several minutes and then fail
outright on a unique-constraint error when two readings shared the same time,
type, and source but carried different internal ids — a single such reading
aborted the whole import. The importer now recognises that case: it updates the
existing reading in place (and brings back one that had been removed) instead of
failing, and if it still can't place a single reading it skips just that one and
carries on. A re-import completes cleanly.
## [1.28.48] — 2026-07-16 — Security follow-ups
Two fixes from the campaign's security re-verification:
+1 -1
View File
@@ -1,7 +1,7 @@
openapi: 3.1.0
info:
title: HealthLog API
version: 1.28.48
version: 1.28.49
description: >-
Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest.
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "healthlog",
"version": "1.28.48",
"version": "1.28.49",
"description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.",
"license": "PolyForm-Noncommercial-1.0.0",
"homepage": "https://healthlog.dev",
+1 -1
View File
@@ -36,7 +36,7 @@ try {
// v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild.
const CACHE_VERSION =
(typeof self !== "undefined" && self.__APP_VERSION__) ||
/* @sw-version-fallback */ "v1.28.48";
/* @sw-version-fallback */ "v1.28.49";
const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`;
const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`;
// v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*`
@@ -3,6 +3,7 @@ import { writeFileSync, mkdtempSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { Prisma } from "@/generated/prisma/client";
import {
hashSampleKey,
parseRecordValue,
@@ -96,6 +97,27 @@ function makeFakePrisma() {
type Row = Record<string, unknown>;
const measurements: Row[] = [];
const workouts: Row[] = [];
let nextId = 1;
// The real `measurements` table carries a SECOND full unique index — the
// natural key `(userId, type, measuredAt, source, sleepStage)` (migration
// 0055, NULLS NOT DISTINCT) — on top of the `(userId, type, source,
// externalId)` one the upsert keys on. Model BOTH so a create that would
// collide on the natural key raises the exact P2002 the production DB does,
// which is what the natural-key rescue must catch. `measuredAt` compares by
// epoch; NULL `sleepStage` collides with NULL (NULLS NOT DISTINCT).
const sameMeasuredAt = (a: unknown, b: unknown): boolean =>
a instanceof Date && b instanceof Date
? a.getTime() === b.getTime()
: a === b;
const sameStage = (a: unknown, b: unknown): boolean =>
(a ?? null) === (b ?? null);
const naturalKeyMatch = (m: Row, r: Row): boolean =>
m.userId === r.userId &&
m.type === r.type &&
m.source === r.source &&
sameMeasuredAt(m.measuredAt, r.measuredAt) &&
sameStage(m.sleepStage, r.sleepStage);
return {
_measurements: measurements,
@@ -137,6 +159,24 @@ function makeFakePrisma() {
) ?? null
);
},
// Natural-key probe used by the rescue path. Tombstone-inclusive: no
// `deletedAt` filter is applied here or by the caller.
findFirst: async ({ where }: { where: Record<string, unknown> }) => {
const match = measurements.find((m) => naturalKeyMatch(m, where));
return match ? { id: match.id } : null;
},
update: async ({
where,
data,
}: {
where: { id: unknown };
data: Row;
}) => {
const idx = measurements.findIndex((m) => m.id === where.id);
if (idx < 0) throw new Error(`no row with id ${String(where.id)}`);
measurements[idx] = { ...measurements[idx], ...data };
return measurements[idx];
},
upsert: async ({
where,
create,
@@ -158,8 +198,17 @@ function makeFakePrisma() {
measurements[idx] = { ...measurements[idx], ...update };
return measurements[idx];
}
measurements.push(create);
return create;
// The externalId leg missed → this is a create. Enforce the natural-key
// unique before inserting, exactly as Postgres would.
if (measurements.some((m) => naturalKeyMatch(m, create))) {
throw new Prisma.PrismaClientKnownRequestError(
"Unique constraint failed on the fields: (`user_id`,`type`,`measured_at`,`source`,`sleep_stage`)",
{ code: "P2002", clientVersion: "test" },
);
}
const row = { id: `m${nextId++}`, ...create };
measurements.push(row);
return row;
},
},
workout: {
@@ -398,6 +447,134 @@ describe("streamParseExportXml — end-to-end", () => {
});
});
/**
* Issue #486 — two Apple samples that share the `measurements` natural key
* `(userId, type, measuredAt, source, sleepStage)` but carry DIFFERENT
* externalIds. The externalId upsert misses on the second → its create leg
* collides on the natural key → P2002. Before the natural-key rescue this
* aborted the whole import after minutes of work; after it, the run completes
* with the colliding row adopted in place.
*/
describe("streamParseExportXml — natural-key collision (issue #486)", () => {
// Both records share `endDate` (→ same `measuredAt`, the natural key) but
// differ in `startDate`/`value` (→ different `hashSampleKey` externalId).
function collidingExportXml(): string {
return `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE HealthData [<!ELEMENT HealthData (Record)*>]>
<HealthData locale="en_US">
<Record type="HKQuantityTypeIdentifierBodyMass"
unit="kg"
startDate="2026-05-14 08:10:00 +0200"
endDate="2026-05-14 08:14:00 +0200"
value="78.4"
sourceName="Source App A"/>
<Record type="HKQuantityTypeIdentifierBodyMass"
unit="kg"
startDate="2026-05-14 08:12:00 +0200"
endDate="2026-05-14 08:14:00 +0200"
value="78.6"
sourceName="Source App B"/>
</HealthData>`;
}
it("completes and adopts the colliding row instead of throwing P2002", async () => {
const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
const xmlPath = join(tmp, "export.xml");
writeFileSync(xmlPath, collidingExportXml());
const prisma = makeFakePrisma();
// On pre-fix code this rejects with the P2002 the fake raises when the
// second create hits the natural key; on fixed code it resolves.
const result = await streamParseExportXml({
xmlPath,
userId: "user-1",
userTimezone: "Europe/Berlin",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
prisma: prisma as any,
});
// Both samples were read; one row physically landed, the twin adopted it.
expect(result.perType.WEIGHT?.read).toBe(2);
expect(result.perType.WEIGHT?.inserted).toBe(1);
expect(result.perType.WEIGHT?.updated).toBe(1);
const weightRows = prisma._measurements.filter((m) => m.type === "WEIGHT");
expect(weightRows).toHaveLength(1);
// Last write wins: value + externalId re-keyed onto the second sample.
expect(weightRows[0]?.value).toBe(78.6);
const secondExternalId = hashSampleKey(
"HKQuantityTypeIdentifierBodyMass",
"78.6",
"2026-05-14 08:12:00 +0200",
"2026-05-14 08:14:00 +0200",
);
expect(weightRows[0]?.externalId).toBe(secondExternalId);
// The collision was rescued, not skipped.
expect(result.unknown["WEIGHT::natural_key_unresolved"]).toBeUndefined();
expect(result.unknown["WEIGHT::natural_key_rescue_failed"]).toBeUndefined();
});
it("re-import of overlapping data with a re-keyed externalId adopts the prior row", async () => {
const tmp = mkdtempSync(join(tmpdir(), "healthlog-parser-test-"));
const prisma = makeFakePrisma();
// First import: single reading, externalId derived from its start/end.
const firstPath = join(tmp, "export-1.xml");
writeFileSync(
firstPath,
`<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE HealthData [<!ELEMENT HealthData (Record)*>]>
<HealthData locale="en_US">
<Record type="HKQuantityTypeIdentifierBodyMass"
unit="kg"
startDate="2026-05-14 08:10:00 +0200"
endDate="2026-05-14 08:14:00 +0200"
value="78.4"/>
</HealthData>`,
);
const first = await streamParseExportXml({
xmlPath: firstPath,
userId: "user-1",
userTimezone: "Europe/Berlin",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
prisma: prisma as any,
});
expect(first.perType.WEIGHT?.inserted).toBe(1);
expect(prisma._measurements).toHaveLength(1);
// Re-export: SAME instant (endDate), but the sample now carries a
// different start window → a re-keyed externalId. The externalId upsert
// misses; the natural key is still held by the first row.
const secondPath = join(tmp, "export-2.xml");
writeFileSync(
secondPath,
`<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE HealthData [<!ELEMENT HealthData (Record)*>]>
<HealthData locale="en_US">
<Record type="HKQuantityTypeIdentifierBodyMass"
unit="kg"
startDate="2026-05-14 08:11:30 +0200"
endDate="2026-05-14 08:14:00 +0200"
value="79.0"/>
</HealthData>`,
);
const second = await streamParseExportXml({
xmlPath: secondPath,
userId: "user-1",
userTimezone: "Europe/Berlin",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
prisma: prisma as any,
});
// No duplicate row; the prior row was adopted, counted as an update.
expect(second.perType.WEIGHT?.inserted).toBe(0);
expect(second.perType.WEIGHT?.updated).toBe(1);
expect(prisma._measurements).toHaveLength(1);
expect(prisma._measurements[0]?.value).toBe(79.0);
});
});
/**
* Streaming-memory ceiling: synthesise a ~5 MB export and assert peak
* heap stays bounded. Smaller than the §11 §5 fixture goal (50 MB) to
@@ -374,39 +374,112 @@ export async function streamParseExportXml(
const stat = bumpStat(row.type);
const rowStart = Date.now();
const exists = existingKey.has(`${row.type}::${row.externalId}`);
await prisma.measurement.upsert({
where: {
userId_type_source_externalId: {
try {
await prisma.measurement.upsert({
where: {
userId_type_source_externalId: {
userId,
type: row.type,
source: "APPLE_HEALTH",
externalId: row.externalId,
},
},
create: {
userId,
type: row.type,
value: row.value,
unit: row.unit,
source: "APPLE_HEALTH",
measuredAt: row.measuredAt,
externalId: row.externalId,
externalSourceVersion: row.externalSourceVersion,
sleepStage: row.sleepStage ?? null,
deviceType: row.deviceType,
},
},
create: {
userId,
type: row.type,
value: row.value,
unit: row.unit,
source: "APPLE_HEALTH",
measuredAt: row.measuredAt,
externalId: row.externalId,
externalSourceVersion: row.externalSourceVersion,
sleepStage: row.sleepStage ?? null,
deviceType: row.deviceType,
},
update: {
value: row.value,
measuredAt: row.measuredAt,
externalSourceVersion: row.externalSourceVersion,
sleepStage: row.sleepStage ?? null,
deviceType: row.deviceType,
},
});
stat.durationMs += Date.now() - rowStart;
if (exists) stat.updated += 1;
else stat.inserted += 1;
rowsUpserted += 1;
update: {
value: row.value,
measuredAt: row.measuredAt,
externalSourceVersion: row.externalSourceVersion,
sleepStage: row.sleepStage ?? null,
deviceType: row.deviceType,
},
});
stat.durationMs += Date.now() - rowStart;
if (exists) stat.updated += 1;
else stat.inserted += 1;
rowsUpserted += 1;
} catch (err) {
// Natural-key rescue. `Measurement` carries a SECOND full unique
// index beyond the externalId one this upsert keys on — the natural
// key `(userId, type, measuredAt, source, sleepStage)` (migration
// 0055, NULLS NOT DISTINCT). When two Apple samples share that natural
// key but carry DIFFERENT externalIds — two source apps writing at the
// same instant, or hash-derived ids on a re-import of overlapping data
// — the externalId leg above MISSES (the incoming externalId isn't in
// the table) and the create leg then collides on the natural key →
// P2002. A blind throw aborts the whole import mid-run (issue #486).
// Instead, mirror the sync providers' natural-key rescue
// (`withings/sync-sleep.ts`, `google-health/sync.ts`, `fitbit/sync.ts`):
// probe the occupied row by its natural key — tombstone-inclusive, no
// `deletedAt` filter, since a soft-deleted row still holds the key and
// would otherwise wedge the night forever — and ADOPT it: re-key onto
// the incoming externalId, take the new value/unit, resurrect if it was
// tombstoned. A single sample must NEVER abort the run: this catch sits
// OUTSIDE any transaction (each upsert is its own statement), so the
// caught P2002 can't poison a surrounding tx.
if (
err instanceof Prisma.PrismaClientKnownRequestError &&
err.code === "P2002"
) {
try {
const twin = await prisma.measurement.findFirst({
where: {
userId,
type: row.type,
source: "APPLE_HEALTH",
measuredAt: row.measuredAt,
sleepStage: row.sleepStage ?? null,
},
select: { id: true },
});
if (twin) {
await prisma.measurement.update({
where: { id: twin.id },
// `measuredAt` + `sleepStage` already match by construction
// (they ARE the natural key we probed on), so only value/unit,
// the externalId re-key, provenance, and the resurrect move.
data: {
value: row.value,
unit: row.unit,
externalId: row.externalId,
externalSourceVersion: row.externalSourceVersion,
deviceType: row.deviceType,
deletedAt: null,
},
});
// An adopted natural-key twin is an in-place UPDATE, not a fresh
// row — keep the inserted/updated split honest.
stat.updated += 1;
rowsUpserted += 1;
} else {
// P2002 with no findable natural-key twin (a race, or a collision
// we can't reconcile here). Skip this one sample and keep going —
// surfaced under `unknown` so operators can see the skip.
unknown[`${row.type}::natural_key_unresolved`] =
(unknown[`${row.type}::natural_key_unresolved`] ?? 0) + 1;
}
} catch {
unknown[`${row.type}::natural_key_rescue_failed`] =
(unknown[`${row.type}::natural_key_rescue_failed`] ?? 0) + 1;
}
} else {
// Any other single-row write error: skip the sample, never abort the
// import over one bad row.
unknown[`${row.type}::upsert_failed`] =
(unknown[`${row.type}::upsert_failed`] ?? 0) + 1;
}
stat.durationMs += Date.now() - rowStart;
}
}
};
@@ -0,0 +1,175 @@
/**
* Issue #486 — the Apple `export.zip` importer must survive two samples that
* share the `measurements` natural key `(userId, type, measuredAt, source,
* sleepStage)` (migration 0055, NULLS NOT DISTINCT) while carrying DIFFERENT
* externalIds. This is a real DB-constraint contract — the collision only
* surfaces against Postgres, not the in-memory unit fake — so it is pinned
* here against the testcontainer.
*
* Pre-fix: the externalId upsert misses on the second sample, its create leg
* collides on the natural key, Prisma throws P2002, and the whole import
* aborts after minutes of work. Post-fix: the natural-key rescue adopts the
* occupied row in place and the run completes.
*/
import { mkdtempSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { beforeEach, describe, expect, it } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import { streamParseExportXml } from "@/lib/measurements/import-apple-health-export";
beforeEach(async () => {
await truncateAllTables(getPrismaClient());
});
async function createUser(username: string) {
return getPrismaClient().user.create({
data: { username, email: `${username}@example.test`, role: "USER" },
});
}
function writeXml(records: string): string {
const tmp = mkdtempSync(join(tmpdir(), "healthlog-import-nk-"));
const xmlPath = join(tmp, "export.xml");
writeFileSync(
xmlPath,
`<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE HealthData [<!ELEMENT HealthData (Record)*>]>
<HealthData locale="en_US">
${records}
</HealthData>`,
);
return xmlPath;
}
// Both records share `endDate` (→ same `measuredAt`, the natural key) but
// differ in `startDate`/`value` (→ different `hashSampleKey` externalId).
const COLLIDING_RECORDS = ` <Record type="HKQuantityTypeIdentifierBodyMass"
unit="kg"
startDate="2026-05-14 08:10:00 +0200"
endDate="2026-05-14 08:14:00 +0200"
value="78.4"
sourceName="Source App A"/>
<Record type="HKQuantityTypeIdentifierBodyMass"
unit="kg"
startDate="2026-05-14 08:12:00 +0200"
endDate="2026-05-14 08:14:00 +0200"
value="78.6"
sourceName="Source App B"/>`;
describe("Apple Health import — natural-key collision (issue #486)", () => {
it("completes against real Postgres and lands exactly one adopted row", async () => {
const prisma = getPrismaClient();
const user = await createUser("import-nk-collide");
const result = await streamParseExportXml({
xmlPath: writeXml(COLLIDING_RECORDS),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
expect(result.perType.WEIGHT?.read).toBe(2);
expect(result.perType.WEIGHT?.inserted).toBe(1);
expect(result.perType.WEIGHT?.updated).toBe(1);
expect(result.unknown["WEIGHT::natural_key_unresolved"]).toBeUndefined();
const rows = await prisma.measurement.findMany({
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
});
expect(rows).toHaveLength(1);
// Last write wins — the second sample's value was adopted onto the row.
expect(rows[0]?.value).toBe(78.6);
});
it("re-import with a re-keyed externalId adopts the prior row, no duplicate", async () => {
const prisma = getPrismaClient();
const user = await createUser("import-nk-reimport");
const first = await streamParseExportXml({
xmlPath: writeXml(
` <Record type="HKQuantityTypeIdentifierBodyMass"
unit="kg"
startDate="2026-05-14 08:10:00 +0200"
endDate="2026-05-14 08:14:00 +0200"
value="78.4"/>`,
),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
expect(first.perType.WEIGHT?.inserted).toBe(1);
// Re-export: same instant (endDate) but a different start window →
// re-keyed externalId. The externalId upsert misses; the natural key is
// still held by the first row.
const second = await streamParseExportXml({
xmlPath: writeXml(
` <Record type="HKQuantityTypeIdentifierBodyMass"
unit="kg"
startDate="2026-05-14 08:11:30 +0200"
endDate="2026-05-14 08:14:00 +0200"
value="79.0"/>`,
),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
expect(second.perType.WEIGHT?.inserted).toBe(0);
expect(second.perType.WEIGHT?.updated).toBe(1);
const rows = await prisma.measurement.findMany({
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
});
expect(rows).toHaveLength(1);
expect(rows[0]?.value).toBe(79.0);
});
it("resurrects a soft-deleted row that holds the natural key", async () => {
const prisma = getPrismaClient();
const user = await createUser("import-nk-tombstone");
// Seed a TOMBSTONED WEIGHT row that already occupies the natural key with
// some legacy externalId. A tombstone still holds the 0055 key, so a blind
// create would P2002 forever ("erased re-keyed sleep" wedge class).
const measuredAt = new Date("2026-05-14T06:14:00.000Z");
await prisma.measurement.create({
data: {
userId: user.id,
type: "WEIGHT",
value: 70.0,
unit: "kg",
source: "APPLE_HEALTH",
measuredAt,
externalId: "sample:legacy-tombstoned-id",
deletedAt: new Date(),
},
});
const result = await streamParseExportXml({
xmlPath: writeXml(
` <Record type="HKQuantityTypeIdentifierBodyMass"
unit="kg"
startDate="2026-05-14 08:10:00 +0200"
endDate="2026-05-14 08:14:00 +0200"
value="80.0"/>`,
),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
expect(result.perType.WEIGHT?.updated).toBe(1);
const rows = await prisma.measurement.findMany({
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
});
expect(rows).toHaveLength(1);
// Adopted: value updated, tombstone lifted (resurrected).
expect(rows[0]?.value).toBe(80.0);
expect(rows[0]?.deletedAt).toBeNull();
});
});