mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 12:32:43 +00:00
feat(sources): pickCanonicalSource — two-axis lookup with single-axis fallback
After picking the canonical source for a day, the picker now consults a device-type ladder and keeps ONLY the rows from the top-ranked device- type present in the day's bucket. Rationale: Apple Watch + iPhone + Withings scale all stream steps into HealthKit; summing the three triple-counts the daily total. The watch is the most reliable signal, so the picker keeps the watch rows and drops phone/scale. Lookup order (high to low): deviceTypePriority[metricType] (per-metric override) > deviceTypePriority.default (user default) > DEFAULT_DEVICE_TYPE_PRIORITY (constant). When no device-type info is present (legacy NULL rows / pre-W8c data), the second axis is a no-op — every row in the bucket passes through. Custom ladders that don't enumerate the bucket's device-types also fall through unfiltered, so the picker never silently drops data. SourcePickerRow gains two optional fields (deviceType, type). Both default-null so existing call sites continue to compile. Tests: 18 cases pass (8 single-axis legacy + 10 new W8c two-axis). Covers per-metric override, custom ladder, legacy NULL fallback, preference of known over unknown, and the W8c-over-W5e shape merge.
This commit is contained in:
Marc-André Bombeck committed
1 parent
f05c55f638
commit
42d88fa5a8
2 files changed
+384
-19
No files matched your search
@@ -4,6 +4,7 @@ import { pickCanonicalSourceRows } from "../source-priority";
|
||||
|
||||
/**
|
||||
* v1.4.25 W5e — cross-source canonical-row picker tests.
|
||||
* v1.4.25 W8c — two-axis picker (source + device-type) tests.
|
||||
*
|
||||
* Coverage focuses on the cumulative-metric use case where two
|
||||
* sources record the same day's value and one must win.
|
||||
@@ -12,7 +13,7 @@ function isoDayKey(d: Date): string {
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
describe("pickCanonicalSourceRows — cumulative-metric picker", () => {
|
||||
describe("pickCanonicalSourceRows — single-axis cumulative-metric picker", () => {
|
||||
it("returns empty for empty input", () => {
|
||||
const out = pickCanonicalSourceRows([], "steps", null, isoDayKey);
|
||||
expect(out.canonicalRows).toEqual([]);
|
||||
@@ -86,6 +87,7 @@ describe("pickCanonicalSourceRows — cumulative-metric picker", () => {
|
||||
value: 82.0,
|
||||
},
|
||||
];
|
||||
// W5e flat shape still accepted (back-compat).
|
||||
const userPriority = {
|
||||
weight: ["MANUAL", "WITHINGS", "APPLE_HEALTH"],
|
||||
};
|
||||
@@ -162,3 +164,250 @@ describe("pickCanonicalSourceRows — cumulative-metric picker", () => {
|
||||
expect(out.canonicalRows[0].source).toBe("APPLE_HEALTH");
|
||||
});
|
||||
});
|
||||
|
||||
describe("pickCanonicalSourceRows — v1.4.25 W8c two-axis device-type picker", () => {
|
||||
it("is a no-op when every row has the same source and no device-type", () => {
|
||||
// The pre-W8c reality: rows arrive without `deviceType` set, so
|
||||
// the second axis can't differentiate anything — every row in the
|
||||
// bucket passes through. Locks in the back-compat invariant.
|
||||
const rows = [
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T08:00:00Z"),
|
||||
source: "WITHINGS" as const,
|
||||
value: 4000,
|
||||
},
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T20:00:00Z"),
|
||||
source: "WITHINGS" as const,
|
||||
value: 2000,
|
||||
},
|
||||
];
|
||||
const out = pickCanonicalSourceRows(rows, "steps", null, isoDayKey);
|
||||
expect(out.canonicalRows).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("keeps only `watch` rows when same source contributed watch + phone (default device ladder)", () => {
|
||||
// Apple Watch + iPhone both stream steps via HealthKit. Summing
|
||||
// both triple-counts. Default ladder is watch > ring > band >
|
||||
// phone > scale > other > unknown, so watch wins.
|
||||
const rows = [
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T09:00:00Z"),
|
||||
source: "APPLE_HEALTH" as const,
|
||||
deviceType: "watch",
|
||||
type: "ACTIVITY_STEPS" as const,
|
||||
value: 1500,
|
||||
},
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T10:00:00Z"),
|
||||
source: "APPLE_HEALTH" as const,
|
||||
deviceType: "watch",
|
||||
type: "ACTIVITY_STEPS" as const,
|
||||
value: 2400,
|
||||
},
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T09:00:00Z"),
|
||||
source: "APPLE_HEALTH" as const,
|
||||
deviceType: "phone",
|
||||
type: "ACTIVITY_STEPS" as const,
|
||||
value: 4100,
|
||||
},
|
||||
];
|
||||
const out = pickCanonicalSourceRows(rows, "steps", null, isoDayKey);
|
||||
expect(out.canonicalRows).toHaveLength(2);
|
||||
expect(out.canonicalRows.every((r) => r.deviceType === "watch")).toBe(true);
|
||||
expect(out.pickedByDay.get("2026-05-12")).toBe("APPLE_HEALTH");
|
||||
});
|
||||
|
||||
it("respects a user-level deviceTypePriority default ladder", () => {
|
||||
// User explicitly demoted watch in favour of phone — maybe their
|
||||
// ScanWatch is currently broken and they trust iPhone.
|
||||
const rows = [
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T09:00:00Z"),
|
||||
source: "APPLE_HEALTH" as const,
|
||||
deviceType: "watch",
|
||||
type: "ACTIVITY_STEPS" as const,
|
||||
value: 1500,
|
||||
},
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T09:00:00Z"),
|
||||
source: "APPLE_HEALTH" as const,
|
||||
deviceType: "phone",
|
||||
type: "ACTIVITY_STEPS" as const,
|
||||
value: 4100,
|
||||
},
|
||||
];
|
||||
const userPriority = {
|
||||
deviceTypePriority: {
|
||||
default: ["phone", "watch"],
|
||||
},
|
||||
};
|
||||
const out = pickCanonicalSourceRows(
|
||||
rows,
|
||||
"steps",
|
||||
userPriority,
|
||||
isoDayKey,
|
||||
);
|
||||
expect(out.canonicalRows).toHaveLength(1);
|
||||
expect(out.canonicalRows[0].deviceType).toBe("phone");
|
||||
expect(out.canonicalRows[0].value).toBe(4100);
|
||||
});
|
||||
|
||||
it("respects a per-MeasurementType deviceTypePriority override", () => {
|
||||
// For ACTIVITY_STEPS the user prefers phone (gym headphones);
|
||||
// every other metric stays default. Verify the override fires.
|
||||
const rows = [
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T09:00:00Z"),
|
||||
source: "APPLE_HEALTH" as const,
|
||||
deviceType: "watch",
|
||||
type: "ACTIVITY_STEPS" as const,
|
||||
value: 1500,
|
||||
},
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T09:00:00Z"),
|
||||
source: "APPLE_HEALTH" as const,
|
||||
deviceType: "phone",
|
||||
type: "ACTIVITY_STEPS" as const,
|
||||
value: 4100,
|
||||
},
|
||||
];
|
||||
const userPriority = {
|
||||
deviceTypePriority: {
|
||||
default: ["watch", "phone"],
|
||||
// Override: phone wins for steps specifically.
|
||||
ACTIVITY_STEPS: ["phone", "watch"],
|
||||
},
|
||||
};
|
||||
const out = pickCanonicalSourceRows(
|
||||
rows,
|
||||
"steps",
|
||||
userPriority,
|
||||
isoDayKey,
|
||||
);
|
||||
expect(out.canonicalRows).toHaveLength(1);
|
||||
expect(out.canonicalRows[0].deviceType).toBe("phone");
|
||||
});
|
||||
|
||||
it("keeps unknown/legacy NULL rows when no ranked device-type coexists", () => {
|
||||
// A pre-v1.4.25 Withings row has `deviceType: null` →
|
||||
// normalizeDeviceType maps it to "unknown". With no other ranked
|
||||
// device-type in the bucket, the picker must keep the row so
|
||||
// legacy data continues to drive analytics.
|
||||
const rows = [
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T07:00:00Z"),
|
||||
source: "WITHINGS" as const,
|
||||
deviceType: null,
|
||||
type: "WEIGHT" as const,
|
||||
value: 82.4,
|
||||
},
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T19:00:00Z"),
|
||||
source: "WITHINGS" as const,
|
||||
deviceType: null,
|
||||
type: "WEIGHT" as const,
|
||||
value: 82.5,
|
||||
},
|
||||
];
|
||||
const out = pickCanonicalSourceRows(rows, "weight", null, isoDayKey);
|
||||
expect(out.canonicalRows).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("prefers a known device-type over `unknown` legacy rows", () => {
|
||||
// Mixed bucket — a new `scale`-tagged row coexists with two
|
||||
// legacy NULL rows. The picker must drop the legacy rows in
|
||||
// favour of the well-tagged one.
|
||||
const rows = [
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T07:00:00Z"),
|
||||
source: "WITHINGS" as const,
|
||||
deviceType: null,
|
||||
type: "WEIGHT" as const,
|
||||
value: 82.4,
|
||||
},
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T19:00:00Z"),
|
||||
source: "WITHINGS" as const,
|
||||
deviceType: "scale",
|
||||
type: "WEIGHT" as const,
|
||||
value: 82.5,
|
||||
},
|
||||
];
|
||||
const out = pickCanonicalSourceRows(rows, "weight", null, isoDayKey);
|
||||
expect(out.canonicalRows).toHaveLength(1);
|
||||
expect(out.canonicalRows[0].deviceType).toBe("scale");
|
||||
});
|
||||
|
||||
it("falls through (keeps every row) when the user ladder doesn't list any present device-type", () => {
|
||||
// User typed `["ring"]` as their custom ladder; bucket has only
|
||||
// `["watch","phone"]`. The picker can't pick anything from the
|
||||
// ladder, so it preserves data rather than silently dropping it.
|
||||
const rows = [
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T09:00:00Z"),
|
||||
source: "APPLE_HEALTH" as const,
|
||||
deviceType: "watch",
|
||||
type: "ACTIVITY_STEPS" as const,
|
||||
value: 1500,
|
||||
},
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T09:00:00Z"),
|
||||
source: "APPLE_HEALTH" as const,
|
||||
deviceType: "phone",
|
||||
type: "ACTIVITY_STEPS" as const,
|
||||
value: 4100,
|
||||
},
|
||||
];
|
||||
const userPriority = {
|
||||
deviceTypePriority: {
|
||||
default: ["ring"],
|
||||
},
|
||||
};
|
||||
const out = pickCanonicalSourceRows(
|
||||
rows,
|
||||
"steps",
|
||||
userPriority,
|
||||
isoDayKey,
|
||||
);
|
||||
expect(out.canonicalRows).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("two-axis lookup order: per-metric override > metricPriority > flat default", () => {
|
||||
// Full-stack test: nested W8c metricPriority pins WITHINGS first
|
||||
// for weight even though the constant default ladder has WITHINGS
|
||||
// first anyway — verify the W8c shape is consulted. Mixed-source
|
||||
// bucket; the picker must reach the nested key.
|
||||
const rows = [
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T07:00:00Z"),
|
||||
source: "WITHINGS" as const,
|
||||
deviceType: "scale",
|
||||
type: "WEIGHT" as const,
|
||||
value: 82.4,
|
||||
},
|
||||
{
|
||||
measuredAt: new Date("2026-05-12T07:30:00Z"),
|
||||
source: "MANUAL" as const,
|
||||
deviceType: null,
|
||||
type: "WEIGHT" as const,
|
||||
value: 82.0,
|
||||
},
|
||||
];
|
||||
const userPriority = {
|
||||
// Flat shape says MANUAL wins.
|
||||
weight: ["MANUAL", "WITHINGS"],
|
||||
// Nested W8c shape says WITHINGS wins — should beat the flat shape.
|
||||
metricPriority: { weight: ["WITHINGS", "MANUAL"] },
|
||||
};
|
||||
const out = pickCanonicalSourceRows(
|
||||
rows,
|
||||
"weight",
|
||||
userPriority,
|
||||
isoDayKey,
|
||||
);
|
||||
expect(out.canonicalRows).toHaveLength(1);
|
||||
expect(out.canonicalRows[0].source).toBe("WITHINGS");
|
||||
});
|
||||
});
|
||||
@@ -14,15 +14,30 @@
|
||||
* audit trail — the user can flip the priority and the analytics
|
||||
* re-pick instantly without a re-sync.
|
||||
*
|
||||
* Today (v1.4.25) only WITHINGS + MANUAL coexist. The function
|
||||
* effectively no-ops for every user because there's only one ingest
|
||||
* source per metric. The architecture lands here so v1.5's iOS
|
||||
* passthrough drops onto a known foundation.
|
||||
* v1.4.25 W8c — two-axis pick. After narrowing to the winning source,
|
||||
* the picker now consults a device-type ladder ("watch beats phone
|
||||
* beats scale") and keeps ONLY the rows from the top-ranked device-
|
||||
* type present in that day's bucket. Rationale: a user wearing an
|
||||
* Apple Watch + carrying an iPhone + standing on a Withings scale
|
||||
* pushes three different hourly step streams into HealthKit; summing
|
||||
* them triple-counts. The watch's stream is the most reliable
|
||||
* "moving body" signal, so we keep the watch rows and drop the phone
|
||||
* + scale rows. Rows tagged with `deviceType = null` resolve to
|
||||
* `"unknown"` (lowest rank) and are kept only if NO ranked device-type
|
||||
* coexists in the same day — never silently dropped.
|
||||
*
|
||||
* Picker stays O(n) overall with O(1) device-type rank lookups.
|
||||
*/
|
||||
import type { MeasurementSource } from "@/generated/prisma/client";
|
||||
import type {
|
||||
MeasurementSource,
|
||||
MeasurementType,
|
||||
} from "@/generated/prisma/client";
|
||||
|
||||
import {
|
||||
DEFAULT_SOURCE_PRIORITY,
|
||||
DEFAULT_DEVICE_TYPE_PRIORITY,
|
||||
type DeviceType,
|
||||
getSourceLadder,
|
||||
normalizeDeviceType,
|
||||
parseSourcePriority,
|
||||
type SourcePriorityMetricKey,
|
||||
} from "@/lib/validations/source-priority";
|
||||
@@ -31,21 +46,45 @@ import {
|
||||
* Minimum row shape the helper consults — anything else on the row is
|
||||
* fine, the helper just narrows the type so it composes with the
|
||||
* existing Measurement reads without a transform step.
|
||||
*
|
||||
* `deviceType` is the v1.4.25 W8c nullable column. Legacy rows + every
|
||||
* pre-W8c Measurement leaves it `null`; the picker maps null → unknown
|
||||
* via `normalizeDeviceType()` so the picker never branches on
|
||||
* presence.
|
||||
*
|
||||
* `type` is the `MeasurementType` enum on each Measurement row and
|
||||
* lets the picker reach into the per-metric device-type override
|
||||
* without re-deriving it from the caller's `metricKey`. Optional so
|
||||
* callers who only carry an aggregation-bucket key (e.g. legacy paths)
|
||||
* still compile; the picker falls back to the user-default ladder
|
||||
* when absent.
|
||||
*/
|
||||
export interface SourcePickerRow {
|
||||
measuredAt: Date;
|
||||
source: MeasurementSource;
|
||||
deviceType?: string | null;
|
||||
type?: MeasurementType | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the canonical-source rows for a per-day cumulative metric.
|
||||
*
|
||||
* Algorithm:
|
||||
* Algorithm (v1.4.25 W8c two-axis):
|
||||
* 1. Bucket rows by `dayKey(measuredAt)`.
|
||||
* 2. For each day, walk the priority list in order and pick the
|
||||
* FIRST source that has any row in that bucket.
|
||||
* 3. Return every row from the picked source — drop rows from the
|
||||
* other sources in the same bucket.
|
||||
* 2. For each day, walk the SOURCE priority list in order and pick
|
||||
* the FIRST source that has any row in that bucket.
|
||||
* 3. Among the picked source's rows in that bucket, walk the
|
||||
* DEVICE-TYPE priority list and keep ONLY rows from the FIRST
|
||||
* device-type that has any row in that bucket. Cumulative-metric
|
||||
* callers sum these without double-counting; point-metric
|
||||
* callers pick the latest downstream.
|
||||
* 4. If `deviceTypePriority` has a per-metric override keyed by the
|
||||
* row's `MeasurementType`, that wins over the user's default
|
||||
* ladder which wins over `DEFAULT_DEVICE_TYPE_PRIORITY`.
|
||||
* 5. Rows whose device-type isn't in the ladder fall through as
|
||||
* `"unknown"`; they only survive when no ranked device-type
|
||||
* coexists in the same day, so legacy NULL-tagged rows still
|
||||
* contribute when they're the only signal.
|
||||
*
|
||||
* Day-key strategy: the caller supplies the function so the analytics
|
||||
* paths (timezone-aware) and tests (deterministic ISO date) can share
|
||||
@@ -68,9 +107,8 @@ export function pickCanonicalSourceRows<T extends SourcePickerRow>(
|
||||
return { canonicalRows: [], pickedByDay: new Map() };
|
||||
}
|
||||
|
||||
const priority =
|
||||
parseSourcePriority(userPriorityJson)[metricKey] ??
|
||||
DEFAULT_SOURCE_PRIORITY[metricKey];
|
||||
const resolved = parseSourcePriority(userPriorityJson);
|
||||
const sourceLadder = getSourceLadder(resolved, metricKey);
|
||||
|
||||
// Bucket per day, track which sources contributed rows in each bucket.
|
||||
const buckets = new Map<
|
||||
@@ -88,12 +126,35 @@ export function pickCanonicalSourceRows<T extends SourcePickerRow>(
|
||||
buckets.set(key, slot);
|
||||
}
|
||||
|
||||
// Cache device-type ladders by (MeasurementType | "__default__") so a
|
||||
// bucket with mixed-type rows resolves the ladder once per type
|
||||
// rather than once per row.
|
||||
const ladderCache = new Map<string, readonly DeviceType[]>();
|
||||
function resolveLadder(rowType: MeasurementType | null | undefined): readonly DeviceType[] {
|
||||
const cacheKey = rowType ?? "__default__";
|
||||
let ladder = ladderCache.get(cacheKey);
|
||||
if (!ladder) {
|
||||
const override = rowType
|
||||
? resolved.deviceTypePriority[rowType]
|
||||
: undefined;
|
||||
ladder =
|
||||
override && override.length > 0
|
||||
? override
|
||||
: resolved.deviceTypePriority.default &&
|
||||
resolved.deviceTypePriority.default.length > 0
|
||||
? resolved.deviceTypePriority.default
|
||||
: DEFAULT_DEVICE_TYPE_PRIORITY;
|
||||
ladderCache.set(cacheKey, ladder);
|
||||
}
|
||||
return ladder;
|
||||
}
|
||||
|
||||
const canonicalRows: T[] = [];
|
||||
const pickedByDay = new Map<string, MeasurementSource>();
|
||||
for (const [key, slot] of buckets) {
|
||||
// Walk the priority list — first match wins.
|
||||
// ── Axis 1: walk the source priority list — first match wins. ──
|
||||
let picked: MeasurementSource | undefined;
|
||||
for (const source of priority) {
|
||||
for (const source of sourceLadder) {
|
||||
if (slot.sources.has(source)) {
|
||||
picked = source;
|
||||
break;
|
||||
@@ -108,8 +169,63 @@ export function pickCanonicalSourceRows<T extends SourcePickerRow>(
|
||||
continue;
|
||||
}
|
||||
pickedByDay.set(key, picked);
|
||||
for (const row of slot.rows) {
|
||||
if (row.source === picked) canonicalRows.push(row);
|
||||
|
||||
// Narrow to the winning source's rows for this day.
|
||||
const pickedRows = slot.rows.filter((row) => row.source === picked);
|
||||
|
||||
// ── Axis 2: keep only rows from the top-ranked device-type. ──
|
||||
// Single-row fast path — no device-type discrimination needed.
|
||||
if (pickedRows.length === 1) {
|
||||
canonicalRows.push(pickedRows[0]);
|
||||
continue;
|
||||
}
|
||||
|
||||
// If every row in this bucket is unknown/null (no device-type
|
||||
// info), the second axis can't differentiate anything — keep
|
||||
// every row. Same intent as the source-axis fallback above:
|
||||
// never silently drop data when the picker has no signal to
|
||||
// filter on. This is the v1.4.25-today behaviour for Withings
|
||||
// rows whose webhook didn't carry a device hint.
|
||||
const hasAnyKnownDeviceType = pickedRows.some(
|
||||
(row) => normalizeDeviceType(row.deviceType) !== "unknown",
|
||||
);
|
||||
if (!hasAnyKnownDeviceType) {
|
||||
canonicalRows.push(...pickedRows);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Resolve the ladder from the first row's type — buckets are
|
||||
// typically same-type so this only resolves once per bucket. If
|
||||
// the bucket carries mixed types, `resolveLadder` is cached and
|
||||
// walked per row inside the device-type walk below.
|
||||
const sampleLadder = resolveLadder(pickedRows[0].type);
|
||||
const presentDeviceTypes = new Set<DeviceType>();
|
||||
for (const row of pickedRows)
|
||||
presentDeviceTypes.add(normalizeDeviceType(row.deviceType));
|
||||
|
||||
let pickedDeviceType: DeviceType | undefined;
|
||||
for (const dt of sampleLadder) {
|
||||
if (presentDeviceTypes.has(dt)) {
|
||||
pickedDeviceType = dt;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Edge case: the user's custom ladder doesn't enumerate the
|
||||
// device-types we have in the bucket (e.g. user typed
|
||||
// `["watch","phone"]`, the bucket has `["scale"]` rows). Fall
|
||||
// through to keep every row — same intent as the source-axis
|
||||
// fallback. The `DEFAULT_DEVICE_TYPE_PRIORITY` constant covers
|
||||
// every enum slot so this only fires when the user typed a custom
|
||||
// ladder that's missing one of the present types.
|
||||
if (!pickedDeviceType) {
|
||||
canonicalRows.push(...pickedRows);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const row of pickedRows) {
|
||||
if (normalizeDeviceType(row.deviceType) === pickedDeviceType) {
|
||||
canonicalRows.push(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user