fix(measurements): merge MANUAL and Apple Health rows for the same reading

A measurement logged by hand on the iOS app while offline is mirrored
into HealthKit. On pairing a server, adopt-on-pair uploads it as a
MANUAL row while HealthKit background sync independently re-ingests the
same sample as an APPLE_HEALTH row. The two carry different external ids
and sources, so the composite unique index keeps both — doubling the
whole hand-logged history on first pair.

Collapse the pair at ingest: an incoming MANUAL or APPLE_HEALTH row is
dropped as a duplicate when a same-reading row of the opposite source
already exists (same type and value, measuredAt within two seconds).
The match is scoped to that one cross-source pair, so same-source rows
and rows from server-owned sources keep their existing contracts, and
two genuinely distinct readings at the same minute never collapse.
stats:* cumulative rows stay on their per-day overwrite lane.

The collapsed copy returns status "duplicate" with reason
"cross_source_merge" so the client cursor advances unchanged; no
schema, migration, or OpenAPI change.
This commit is contained in:
Marc-André Bombeck committed 2026-06-04 16:44:39 +02:00
1 parent b54b95dbc7
commit accd7c8514
4 files changed
+765 -3

No files matched your search

@@ -0,0 +1,185 @@
/**
* v1.11.4 (iOS #2) — unit coverage for the MANUAL↔APPLE_HEALTH
* same-reading merge predicate. The route integration test exercises the
* end-to-end ingest path; this suite pins the pure matching rule:
* - cross-source same reading → match
* - same-source / server-source pair → no match (no over-merge)
* - different value or out-of-tolerance timestamp → no match
*/
import { describe, expect, it } from "vitest";
import {
MEASURED_AT_TOLERANCE_MS,
isMergeableSource,
isSameReadingAcrossSource,
measuredAtMatch,
oppositeMergeSource,
valuesMatch,
type MergeCandidate,
} from "@/lib/measurements/cross-source-merge";
import type { MeasurementType } from "@/generated/prisma/client";
const T0 = new Date("2026-06-04T07:30:00.000Z");
function candidate(over: Partial<MergeCandidate> = {}): MergeCandidate {
return {
type: "WEIGHT" as MeasurementType,
source: "MANUAL",
value: 81.4,
measuredAt: T0,
...over,
};
}
function incoming(
over: Partial<{
type: MeasurementType;
source: string;
value: number;
measuredAt: Date;
}> = {},
) {
return {
type: "WEIGHT" as MeasurementType,
source: "APPLE_HEALTH",
value: 81.4,
measuredAt: T0,
...over,
};
}
describe("isMergeableSource", () => {
it("accepts only the two client-facing mirror sources", () => {
expect(isMergeableSource("MANUAL")).toBe(true);
expect(isMergeableSource("APPLE_HEALTH")).toBe(true);
expect(isMergeableSource("WITHINGS")).toBe(false);
expect(isMergeableSource("IMPORT")).toBe(false);
expect(isMergeableSource("COMPUTED")).toBe(false);
expect(isMergeableSource("WHOOP")).toBe(false);
});
});
describe("oppositeMergeSource", () => {
it("flips MANUAL ↔ APPLE_HEALTH", () => {
expect(oppositeMergeSource("MANUAL")).toBe("APPLE_HEALTH");
expect(oppositeMergeSource("APPLE_HEALTH")).toBe("MANUAL");
});
});
describe("valuesMatch", () => {
it("matches identical and sub-epsilon values", () => {
expect(valuesMatch(81.4, 81.4)).toBe(true);
expect(valuesMatch(81.4, 81.4 + 1e-12)).toBe(true);
expect(valuesMatch(0.97, 0.97)).toBe(true);
});
it("rejects genuinely different values", () => {
expect(valuesMatch(81.4, 81.5)).toBe(false);
expect(valuesMatch(120, 121)).toBe(false);
});
});
describe("measuredAtMatch", () => {
it("matches inside the ±tolerance window", () => {
expect(measuredAtMatch(T0, T0)).toBe(true);
expect(
measuredAtMatch(T0, new Date(T0.getTime() + MEASURED_AT_TOLERANCE_MS)),
).toBe(true);
expect(
measuredAtMatch(T0, new Date(T0.getTime() - MEASURED_AT_TOLERANCE_MS)),
).toBe(true);
});
it("rejects timestamps beyond the tolerance window", () => {
expect(
measuredAtMatch(
T0,
new Date(T0.getTime() + MEASURED_AT_TOLERANCE_MS + 1),
),
).toBe(false);
// A whole minute apart — two distinct readings, never the mirror pair.
expect(measuredAtMatch(T0, new Date(T0.getTime() + 60_000))).toBe(false);
});
});
describe("isSameReadingAcrossSource", () => {
it("matches a MANUAL candidate against an incoming APPLE_HEALTH mirror", () => {
expect(isSameReadingAcrossSource(incoming(), candidate())).toBe(true);
});
it("matches an APPLE_HEALTH candidate against an incoming MANUAL mirror", () => {
expect(
isSameReadingAcrossSource(
incoming({ source: "MANUAL" }),
candidate({ source: "APPLE_HEALTH" }),
),
).toBe(true);
});
it("matches within the timestamp tolerance", () => {
expect(
isSameReadingAcrossSource(
incoming({ measuredAt: new Date(T0.getTime() + 1_500) }),
candidate(),
),
).toBe(true);
});
it("does NOT match a same-source pair (no over-merge)", () => {
// Two APPLE_HEALTH readings — keep the existing per-source contract.
expect(
isSameReadingAcrossSource(
incoming({ source: "APPLE_HEALTH" }),
candidate({ source: "APPLE_HEALTH" }),
),
).toBe(false);
// Two MANUAL readings.
expect(
isSameReadingAcrossSource(
incoming({ source: "MANUAL" }),
candidate({ source: "MANUAL" }),
),
).toBe(false);
});
it("does NOT match when a server-owned source is involved", () => {
expect(
isSameReadingAcrossSource(
incoming({ source: "APPLE_HEALTH" }),
candidate({ source: "WITHINGS" }),
),
).toBe(false);
// Incoming WITHINGS could never reach this route, but the predicate
// must still refuse it.
expect(
isSameReadingAcrossSource(
incoming({ source: "WITHINGS" }),
candidate({ source: "APPLE_HEALTH" }),
),
).toBe(false);
});
it("does NOT match a different measurement type", () => {
expect(
isSameReadingAcrossSource(
incoming({ type: "PULSE" as MeasurementType }),
candidate({ type: "WEIGHT" as MeasurementType }),
),
).toBe(false);
});
it("does NOT match a different value (two real readings, same minute)", () => {
expect(
isSameReadingAcrossSource(incoming({ value: 81.4 }), candidate({ value: 82.0 })),
).toBe(false);
});
it("does NOT match a timestamp beyond the tolerance window", () => {
expect(
isSameReadingAcrossSource(
incoming({ measuredAt: new Date(T0.getTime() + 60_000) }),
candidate(),
),
).toBe(false);
});
});
+151
View File
@@ -0,0 +1,151 @@
/**
* v1.11.4 (iOS #2) — same-reading merge for the MANUAL ↔ APPLE_HEALTH
* mirror duplicate.
*
* ## The problem
* When the iOS app runs standalone/offline and the user logs a MANUAL
* measurement, iOS mirrors that reading into HealthKit too. On pairing a
* server, two independent upload paths then reach
* `POST /api/measurements/batch` for the SAME physical reading:
* 1. adopt-on-pair backfill uploads it as `source = MANUAL`
* (externalId = the app's local id), and
* 2. HealthKit background sync re-ingests the mirrored sample and
* uploads it as `source = APPLE_HEALTH` (externalId = HKSample.uuid).
*
* The two carry different `externalId`s and different `source`s, so the
* `(userId, type, source, externalId)` unique index treats them as two
* distinct rows — duplicating the whole manually-logged history on first
* pair.
*
* ## The rule
* Treat a MANUAL row and an APPLE_HEALTH row as the SAME physical
* reading when they share:
* - the same `userId` (multi-tenant scoping — always narrowed), AND
* - the same `type`, AND
* - the same `value` (within a tiny float epsilon — both rows are the
* same hand-entered number run through the identical
* `mapAppleHealthEntry` conversion, so equality is exact in practice;
* the epsilon only guards against sub-ulp drift), AND
* - a `measuredAt` within `MEASURED_AT_TOLERANCE_MS` of each other.
*
* The merge is deliberately ONE-SIDED in scope: it only fires for the
* MANUAL ↔ APPLE_HEALTH cross-source pair. MANUAL ↔ MANUAL,
* APPLE_HEALTH ↔ APPLE_HEALTH, and any pair involving a server-owned
* source (WITHINGS / IMPORT / COMPUTED / WHOOP) keep the existing
* first-write-wins / per-source contracts untouched. Two genuinely
* distinct readings at the same minute from two devices therefore never
* collapse — they are same-source or involve a server source, both of
* which fall outside this rule.
*
* ## Ingest-time, first-physical-reading-wins
* The collapse runs at INGEST time, not read time: read-time dedup would
* have to live in the rollup tier and every analytics path forever.
* Ingest-time keeps the storage clean and the rule contained.
*
* Because the two uploads can arrive in either order and minutes apart
* in SEPARATE batches, the collapse compares each incoming row against
* the rows already PERSISTED for the user (cross-batch) as well as the
* rows already chosen for insert earlier in the SAME batch.
*
* The winner is whichever physical reading landed first. The `value` is
* identical between the two rows by construction, so the only difference
* the surviving row carries is its `source` label — the numeric series
* the user sees is the same either way. A source-priority-aware REPLACE
* (overwrite the loser's source/row with the higher-ranked source) was
* considered and rejected for a patch: it would require mutating or
* re-sourcing an existing row (rollup re-source + sync-version churn) for
* zero change to the displayed value. First-physical-reading-wins is the
* natural extension of the endpoint's existing first-write-wins
* philosophy across the MANUAL↔APPLE_HEALTH boundary.
*
* `stats:*` cumulative externalIds are explicitly OUT of scope — those
* are per-day aggregate rows the iOS observer overwrites in place
* (issue #213), never hand-entered point readings, so the cross-source
* mirror duplicate cannot arise for them.
*/
import type { MeasurementType } from "@/generated/prisma/client";
/**
* Timestamp tolerance for the same-reading match. A reading the user
* hand-enters and the app mirrors to HealthKit shares the same instant;
* the HealthKit re-ingest may round to the second or carry a sub-second
* drift, so a tight ±2 s window absorbs that without any chance of
* catching a separate genuine reading — a user does not hand-enter two
* distinct readings of the same type within two seconds.
*/
export const MEASURED_AT_TOLERANCE_MS = 2_000;
/**
* Relative epsilon for the value match. Both rows run the same
* `convertToDbUnit` over the same source number, so the stored floats
* are bit-identical in practice; the epsilon is a defensive guard
* against any future conversion that introduces sub-ulp rounding. Scaled
* to the magnitude of the value so it stays meaningful for both a
* 0.97-fraction SpO2 and a 12 000-step count.
*/
const VALUE_RELATIVE_EPSILON = 1e-9;
/** The two client-facing sources that can mirror the same reading. */
type MergeableSource = "MANUAL" | "APPLE_HEALTH";
/** The opposite member of the MANUAL ↔ APPLE_HEALTH pair. */
export function oppositeMergeSource(source: MergeableSource): MergeableSource {
return source === "MANUAL" ? "APPLE_HEALTH" : "MANUAL";
}
/**
* True when `source` participates in the cross-source merge. Only the
* two client-facing mirror sources qualify; server-owned sources are
* untouched.
*/
export function isMergeableSource(source: string): source is MergeableSource {
return source === "MANUAL" || source === "APPLE_HEALTH";
}
/** Are two values the same reading within the relative epsilon? */
export function valuesMatch(a: number, b: number): boolean {
if (a === b) return true;
const scale = Math.max(Math.abs(a), Math.abs(b), 1);
return Math.abs(a - b) <= VALUE_RELATIVE_EPSILON * scale;
}
/** Are two instants within the same-reading tolerance window? */
export function measuredAtMatch(a: Date, b: Date): boolean {
return Math.abs(a.getTime() - b.getTime()) <= MEASURED_AT_TOLERANCE_MS;
}
/**
* A candidate same-reading row — either already persisted for the user
* or already chosen for insert earlier in the same batch. Carries only
* the fields the match needs.
*/
export interface MergeCandidate {
type: MeasurementType;
source: string;
value: number;
measuredAt: Date;
}
/**
* Does `candidate` represent the same physical reading as an incoming
* row of `(type, incomingSource, value, measuredAt)`, under the
* cross-source MANUAL↔APPLE_HEALTH rule? Requires:
* - the incoming row's source is mergeable,
* - the candidate's source is the OPPOSITE mergeable source,
* - same type, value (±epsilon), measuredAt (±tolerance).
*/
export function isSameReadingAcrossSource(
incoming: {
type: MeasurementType;
source: string;
value: number;
measuredAt: Date;
},
candidate: MergeCandidate,
): boolean {
if (!isMergeableSource(incoming.source)) return false;
if (candidate.source !== oppositeMergeSource(incoming.source)) return false;
if (candidate.type !== incoming.type) return false;
if (!valuesMatch(candidate.value, incoming.value)) return false;
return measuredAtMatch(candidate.measuredAt, incoming.measuredAt);
}