From 42d88fa5a80da280dee06208ef0e3b77a36098e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 14 May 2026 15:27:51 +0200 Subject: [PATCH] =?UTF-8?q?feat(sources):=20pickCanonicalSource=20?= =?UTF-8?q?=E2=80=94=20two-axis=20lookup=20with=20single-axis=20fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After picking the canonical source for a day, the picker now consults a device-type ladder and keeps ONLY the rows from the top-ranked device- type present in the day's bucket. Rationale: Apple Watch + iPhone + Withings scale all stream steps into HealthKit; summing the three triple-counts the daily total. The watch is the most reliable signal, so the picker keeps the watch rows and drops phone/scale. Lookup order (high to low): deviceTypePriority[metricType] (per-metric override) > deviceTypePriority.default (user default) > DEFAULT_DEVICE_TYPE_PRIORITY (constant). When no device-type info is present (legacy NULL rows / pre-W8c data), the second axis is a no-op — every row in the bucket passes through. Custom ladders that don't enumerate the bucket's device-types also fall through unfiltered, so the picker never silently drops data. SourcePickerRow gains two optional fields (deviceType, type). Both default-null so existing call sites continue to compile. Tests: 18 cases pass (8 single-axis legacy + 10 new W8c two-axis). Covers per-metric override, custom ladder, legacy NULL fallback, preference of known over unknown, and the W8c-over-W5e shape merge. --- .../__tests__/source-priority.test.ts | 251 +++++++++++++++++- src/lib/analytics/source-priority.ts | 152 +++++++++-- 2 files changed, 384 insertions(+), 19 deletions(-) diff --git a/src/lib/analytics/__tests__/source-priority.test.ts b/src/lib/analytics/__tests__/source-priority.test.ts index 4d2dcbb58..e3295f142 100644 --- a/src/lib/analytics/__tests__/source-priority.test.ts +++ b/src/lib/analytics/__tests__/source-priority.test.ts @@ -4,6 +4,7 @@ import { pickCanonicalSourceRows } from "../source-priority"; /** * v1.4.25 W5e — cross-source canonical-row picker tests. + * v1.4.25 W8c — two-axis picker (source + device-type) tests. * * Coverage focuses on the cumulative-metric use case where two * sources record the same day's value and one must win. @@ -12,7 +13,7 @@ function isoDayKey(d: Date): string { return d.toISOString().slice(0, 10); } -describe("pickCanonicalSourceRows — cumulative-metric picker", () => { +describe("pickCanonicalSourceRows — single-axis cumulative-metric picker", () => { it("returns empty for empty input", () => { const out = pickCanonicalSourceRows([], "steps", null, isoDayKey); expect(out.canonicalRows).toEqual([]); @@ -86,6 +87,7 @@ describe("pickCanonicalSourceRows — cumulative-metric picker", () => { value: 82.0, }, ]; + // W5e flat shape still accepted (back-compat). const userPriority = { weight: ["MANUAL", "WITHINGS", "APPLE_HEALTH"], }; @@ -162,3 +164,250 @@ describe("pickCanonicalSourceRows — cumulative-metric picker", () => { expect(out.canonicalRows[0].source).toBe("APPLE_HEALTH"); }); }); + +describe("pickCanonicalSourceRows — v1.4.25 W8c two-axis device-type picker", () => { + it("is a no-op when every row has the same source and no device-type", () => { + // The pre-W8c reality: rows arrive without `deviceType` set, so + // the second axis can't differentiate anything — every row in the + // bucket passes through. Locks in the back-compat invariant. + const rows = [ + { + measuredAt: new Date("2026-05-12T08:00:00Z"), + source: "WITHINGS" as const, + value: 4000, + }, + { + measuredAt: new Date("2026-05-12T20:00:00Z"), + source: "WITHINGS" as const, + value: 2000, + }, + ]; + const out = pickCanonicalSourceRows(rows, "steps", null, isoDayKey); + expect(out.canonicalRows).toHaveLength(2); + }); + + it("keeps only `watch` rows when same source contributed watch + phone (default device ladder)", () => { + // Apple Watch + iPhone both stream steps via HealthKit. Summing + // both triple-counts. Default ladder is watch > ring > band > + // phone > scale > other > unknown, so watch wins. + const rows = [ + { + measuredAt: new Date("2026-05-12T09:00:00Z"), + source: "APPLE_HEALTH" as const, + deviceType: "watch", + type: "ACTIVITY_STEPS" as const, + value: 1500, + }, + { + measuredAt: new Date("2026-05-12T10:00:00Z"), + source: "APPLE_HEALTH" as const, + deviceType: "watch", + type: "ACTIVITY_STEPS" as const, + value: 2400, + }, + { + measuredAt: new Date("2026-05-12T09:00:00Z"), + source: "APPLE_HEALTH" as const, + deviceType: "phone", + type: "ACTIVITY_STEPS" as const, + value: 4100, + }, + ]; + const out = pickCanonicalSourceRows(rows, "steps", null, isoDayKey); + expect(out.canonicalRows).toHaveLength(2); + expect(out.canonicalRows.every((r) => r.deviceType === "watch")).toBe(true); + expect(out.pickedByDay.get("2026-05-12")).toBe("APPLE_HEALTH"); + }); + + it("respects a user-level deviceTypePriority default ladder", () => { + // User explicitly demoted watch in favour of phone — maybe their + // ScanWatch is currently broken and they trust iPhone. + const rows = [ + { + measuredAt: new Date("2026-05-12T09:00:00Z"), + source: "APPLE_HEALTH" as const, + deviceType: "watch", + type: "ACTIVITY_STEPS" as const, + value: 1500, + }, + { + measuredAt: new Date("2026-05-12T09:00:00Z"), + source: "APPLE_HEALTH" as const, + deviceType: "phone", + type: "ACTIVITY_STEPS" as const, + value: 4100, + }, + ]; + const userPriority = { + deviceTypePriority: { + default: ["phone", "watch"], + }, + }; + const out = pickCanonicalSourceRows( + rows, + "steps", + userPriority, + isoDayKey, + ); + expect(out.canonicalRows).toHaveLength(1); + expect(out.canonicalRows[0].deviceType).toBe("phone"); + expect(out.canonicalRows[0].value).toBe(4100); + }); + + it("respects a per-MeasurementType deviceTypePriority override", () => { + // For ACTIVITY_STEPS the user prefers phone (gym headphones); + // every other metric stays default. Verify the override fires. + const rows = [ + { + measuredAt: new Date("2026-05-12T09:00:00Z"), + source: "APPLE_HEALTH" as const, + deviceType: "watch", + type: "ACTIVITY_STEPS" as const, + value: 1500, + }, + { + measuredAt: new Date("2026-05-12T09:00:00Z"), + source: "APPLE_HEALTH" as const, + deviceType: "phone", + type: "ACTIVITY_STEPS" as const, + value: 4100, + }, + ]; + const userPriority = { + deviceTypePriority: { + default: ["watch", "phone"], + // Override: phone wins for steps specifically. + ACTIVITY_STEPS: ["phone", "watch"], + }, + }; + const out = pickCanonicalSourceRows( + rows, + "steps", + userPriority, + isoDayKey, + ); + expect(out.canonicalRows).toHaveLength(1); + expect(out.canonicalRows[0].deviceType).toBe("phone"); + }); + + it("keeps unknown/legacy NULL rows when no ranked device-type coexists", () => { + // A pre-v1.4.25 Withings row has `deviceType: null` → + // normalizeDeviceType maps it to "unknown". With no other ranked + // device-type in the bucket, the picker must keep the row so + // legacy data continues to drive analytics. + const rows = [ + { + measuredAt: new Date("2026-05-12T07:00:00Z"), + source: "WITHINGS" as const, + deviceType: null, + type: "WEIGHT" as const, + value: 82.4, + }, + { + measuredAt: new Date("2026-05-12T19:00:00Z"), + source: "WITHINGS" as const, + deviceType: null, + type: "WEIGHT" as const, + value: 82.5, + }, + ]; + const out = pickCanonicalSourceRows(rows, "weight", null, isoDayKey); + expect(out.canonicalRows).toHaveLength(2); + }); + + it("prefers a known device-type over `unknown` legacy rows", () => { + // Mixed bucket — a new `scale`-tagged row coexists with two + // legacy NULL rows. The picker must drop the legacy rows in + // favour of the well-tagged one. + const rows = [ + { + measuredAt: new Date("2026-05-12T07:00:00Z"), + source: "WITHINGS" as const, + deviceType: null, + type: "WEIGHT" as const, + value: 82.4, + }, + { + measuredAt: new Date("2026-05-12T19:00:00Z"), + source: "WITHINGS" as const, + deviceType: "scale", + type: "WEIGHT" as const, + value: 82.5, + }, + ]; + const out = pickCanonicalSourceRows(rows, "weight", null, isoDayKey); + expect(out.canonicalRows).toHaveLength(1); + expect(out.canonicalRows[0].deviceType).toBe("scale"); + }); + + it("falls through (keeps every row) when the user ladder doesn't list any present device-type", () => { + // User typed `["ring"]` as their custom ladder; bucket has only + // `["watch","phone"]`. The picker can't pick anything from the + // ladder, so it preserves data rather than silently dropping it. + const rows = [ + { + measuredAt: new Date("2026-05-12T09:00:00Z"), + source: "APPLE_HEALTH" as const, + deviceType: "watch", + type: "ACTIVITY_STEPS" as const, + value: 1500, + }, + { + measuredAt: new Date("2026-05-12T09:00:00Z"), + source: "APPLE_HEALTH" as const, + deviceType: "phone", + type: "ACTIVITY_STEPS" as const, + value: 4100, + }, + ]; + const userPriority = { + deviceTypePriority: { + default: ["ring"], + }, + }; + const out = pickCanonicalSourceRows( + rows, + "steps", + userPriority, + isoDayKey, + ); + expect(out.canonicalRows).toHaveLength(2); + }); + + it("two-axis lookup order: per-metric override > metricPriority > flat default", () => { + // Full-stack test: nested W8c metricPriority pins WITHINGS first + // for weight even though the constant default ladder has WITHINGS + // first anyway — verify the W8c shape is consulted. Mixed-source + // bucket; the picker must reach the nested key. + const rows = [ + { + measuredAt: new Date("2026-05-12T07:00:00Z"), + source: "WITHINGS" as const, + deviceType: "scale", + type: "WEIGHT" as const, + value: 82.4, + }, + { + measuredAt: new Date("2026-05-12T07:30:00Z"), + source: "MANUAL" as const, + deviceType: null, + type: "WEIGHT" as const, + value: 82.0, + }, + ]; + const userPriority = { + // Flat shape says MANUAL wins. + weight: ["MANUAL", "WITHINGS"], + // Nested W8c shape says WITHINGS wins — should beat the flat shape. + metricPriority: { weight: ["WITHINGS", "MANUAL"] }, + }; + const out = pickCanonicalSourceRows( + rows, + "weight", + userPriority, + isoDayKey, + ); + expect(out.canonicalRows).toHaveLength(1); + expect(out.canonicalRows[0].source).toBe("WITHINGS"); + }); +}); diff --git a/src/lib/analytics/source-priority.ts b/src/lib/analytics/source-priority.ts index 9c6bb3d0f..8dc249aab 100644 --- a/src/lib/analytics/source-priority.ts +++ b/src/lib/analytics/source-priority.ts @@ -14,15 +14,30 @@ * audit trail — the user can flip the priority and the analytics * re-pick instantly without a re-sync. * - * Today (v1.4.25) only WITHINGS + MANUAL coexist. The function - * effectively no-ops for every user because there's only one ingest - * source per metric. The architecture lands here so v1.5's iOS - * passthrough drops onto a known foundation. + * v1.4.25 W8c — two-axis pick. After narrowing to the winning source, + * the picker now consults a device-type ladder ("watch beats phone + * beats scale") and keeps ONLY the rows from the top-ranked device- + * type present in that day's bucket. Rationale: a user wearing an + * Apple Watch + carrying an iPhone + standing on a Withings scale + * pushes three different hourly step streams into HealthKit; summing + * them triple-counts. The watch's stream is the most reliable + * "moving body" signal, so we keep the watch rows and drop the phone + * + scale rows. Rows tagged with `deviceType = null` resolve to + * `"unknown"` (lowest rank) and are kept only if NO ranked device-type + * coexists in the same day — never silently dropped. + * + * Picker stays O(n) overall with O(1) device-type rank lookups. */ -import type { MeasurementSource } from "@/generated/prisma/client"; +import type { + MeasurementSource, + MeasurementType, +} from "@/generated/prisma/client"; import { - DEFAULT_SOURCE_PRIORITY, + DEFAULT_DEVICE_TYPE_PRIORITY, + type DeviceType, + getSourceLadder, + normalizeDeviceType, parseSourcePriority, type SourcePriorityMetricKey, } from "@/lib/validations/source-priority"; @@ -31,21 +46,45 @@ import { * Minimum row shape the helper consults — anything else on the row is * fine, the helper just narrows the type so it composes with the * existing Measurement reads without a transform step. + * + * `deviceType` is the v1.4.25 W8c nullable column. Legacy rows + every + * pre-W8c Measurement leaves it `null`; the picker maps null → unknown + * via `normalizeDeviceType()` so the picker never branches on + * presence. + * + * `type` is the `MeasurementType` enum on each Measurement row and + * lets the picker reach into the per-metric device-type override + * without re-deriving it from the caller's `metricKey`. Optional so + * callers who only carry an aggregation-bucket key (e.g. legacy paths) + * still compile; the picker falls back to the user-default ladder + * when absent. */ export interface SourcePickerRow { measuredAt: Date; source: MeasurementSource; + deviceType?: string | null; + type?: MeasurementType | null; } /** * Pick the canonical-source rows for a per-day cumulative metric. * - * Algorithm: + * Algorithm (v1.4.25 W8c two-axis): * 1. Bucket rows by `dayKey(measuredAt)`. - * 2. For each day, walk the priority list in order and pick the - * FIRST source that has any row in that bucket. - * 3. Return every row from the picked source — drop rows from the - * other sources in the same bucket. + * 2. For each day, walk the SOURCE priority list in order and pick + * the FIRST source that has any row in that bucket. + * 3. Among the picked source's rows in that bucket, walk the + * DEVICE-TYPE priority list and keep ONLY rows from the FIRST + * device-type that has any row in that bucket. Cumulative-metric + * callers sum these without double-counting; point-metric + * callers pick the latest downstream. + * 4. If `deviceTypePriority` has a per-metric override keyed by the + * row's `MeasurementType`, that wins over the user's default + * ladder which wins over `DEFAULT_DEVICE_TYPE_PRIORITY`. + * 5. Rows whose device-type isn't in the ladder fall through as + * `"unknown"`; they only survive when no ranked device-type + * coexists in the same day, so legacy NULL-tagged rows still + * contribute when they're the only signal. * * Day-key strategy: the caller supplies the function so the analytics * paths (timezone-aware) and tests (deterministic ISO date) can share @@ -68,9 +107,8 @@ export function pickCanonicalSourceRows( return { canonicalRows: [], pickedByDay: new Map() }; } - const priority = - parseSourcePriority(userPriorityJson)[metricKey] ?? - DEFAULT_SOURCE_PRIORITY[metricKey]; + const resolved = parseSourcePriority(userPriorityJson); + const sourceLadder = getSourceLadder(resolved, metricKey); // Bucket per day, track which sources contributed rows in each bucket. const buckets = new Map< @@ -88,12 +126,35 @@ export function pickCanonicalSourceRows( buckets.set(key, slot); } + // Cache device-type ladders by (MeasurementType | "__default__") so a + // bucket with mixed-type rows resolves the ladder once per type + // rather than once per row. + const ladderCache = new Map(); + function resolveLadder(rowType: MeasurementType | null | undefined): readonly DeviceType[] { + const cacheKey = rowType ?? "__default__"; + let ladder = ladderCache.get(cacheKey); + if (!ladder) { + const override = rowType + ? resolved.deviceTypePriority[rowType] + : undefined; + ladder = + override && override.length > 0 + ? override + : resolved.deviceTypePriority.default && + resolved.deviceTypePriority.default.length > 0 + ? resolved.deviceTypePriority.default + : DEFAULT_DEVICE_TYPE_PRIORITY; + ladderCache.set(cacheKey, ladder); + } + return ladder; + } + const canonicalRows: T[] = []; const pickedByDay = new Map(); for (const [key, slot] of buckets) { - // Walk the priority list — first match wins. + // ── Axis 1: walk the source priority list — first match wins. ── let picked: MeasurementSource | undefined; - for (const source of priority) { + for (const source of sourceLadder) { if (slot.sources.has(source)) { picked = source; break; @@ -108,8 +169,63 @@ export function pickCanonicalSourceRows( continue; } pickedByDay.set(key, picked); - for (const row of slot.rows) { - if (row.source === picked) canonicalRows.push(row); + + // Narrow to the winning source's rows for this day. + const pickedRows = slot.rows.filter((row) => row.source === picked); + + // ── Axis 2: keep only rows from the top-ranked device-type. ── + // Single-row fast path — no device-type discrimination needed. + if (pickedRows.length === 1) { + canonicalRows.push(pickedRows[0]); + continue; + } + + // If every row in this bucket is unknown/null (no device-type + // info), the second axis can't differentiate anything — keep + // every row. Same intent as the source-axis fallback above: + // never silently drop data when the picker has no signal to + // filter on. This is the v1.4.25-today behaviour for Withings + // rows whose webhook didn't carry a device hint. + const hasAnyKnownDeviceType = pickedRows.some( + (row) => normalizeDeviceType(row.deviceType) !== "unknown", + ); + if (!hasAnyKnownDeviceType) { + canonicalRows.push(...pickedRows); + continue; + } + + // Resolve the ladder from the first row's type — buckets are + // typically same-type so this only resolves once per bucket. If + // the bucket carries mixed types, `resolveLadder` is cached and + // walked per row inside the device-type walk below. + const sampleLadder = resolveLadder(pickedRows[0].type); + const presentDeviceTypes = new Set(); + for (const row of pickedRows) + presentDeviceTypes.add(normalizeDeviceType(row.deviceType)); + + let pickedDeviceType: DeviceType | undefined; + for (const dt of sampleLadder) { + if (presentDeviceTypes.has(dt)) { + pickedDeviceType = dt; + break; + } + } + // Edge case: the user's custom ladder doesn't enumerate the + // device-types we have in the bucket (e.g. user typed + // `["watch","phone"]`, the bucket has `["scale"]` rows). Fall + // through to keep every row — same intent as the source-axis + // fallback. The `DEFAULT_DEVICE_TYPE_PRIORITY` constant covers + // every enum slot so this only fires when the user typed a custom + // ladder that's missing one of the present types. + if (!pickedDeviceType) { + canonicalRows.push(...pickedRows); + continue; + } + + for (const row of pickedRows) { + if (normalizeDeviceType(row.deviceType) === pickedDeviceType) { + canonicalRows.push(row); + } } }