From 0f6d668df1bb04e1b1addf250d2cad4ffb27998a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 22 Oct 2026 06:16:54 +0200 Subject: [PATCH 01/19] fix(workouts): match cross-source duplicates by a start window, not a clock slot Two copies of one session were collapsed only when both starts fell in the same fixed five-minute clock slot, so a pair thirty seconds apart that straddled a slot boundary (08:04:50 and 08:05:20) stayed as two workouts in the list, the batch write, the calendar, strain and the Coach. Workouts of one user and sport are now clustered by start time: a workout joins the open cluster while it starts less than five minutes after the cluster's first start. Wherever no pair within five minutes straddles a slot boundary the clusters are exactly the old slots, which a parity test checks against the previous picker on thousands of generated inputs, strain impulses included. A property test shows any offset under five minutes now collapses, five minutes or more never does, and a uniform shift of every start keeps the picked set. --- src/lib/insights/strain-score.ts | 12 +- .../pick-canonical-workout-rows.legacy.ts | 329 ++++++++++++++++++ ...ck-canonical-workout-rows.property.test.ts | 300 ++++++++++++++++ .../pick-canonical-workout-rows.test.ts | 8 +- .../pick-canonical-workout-rows.ts | 88 +++-- 5 files changed, 701 insertions(+), 36 deletions(-) create mode 100644 src/lib/measurements/__tests__/pick-canonical-workout-rows.legacy.ts create mode 100644 src/lib/measurements/__tests__/pick-canonical-workout-rows.property.test.ts diff --git a/src/lib/insights/strain-score.ts b/src/lib/insights/strain-score.ts index ee87a461b..863776d17 100644 --- a/src/lib/insights/strain-score.ts +++ b/src/lib/insights/strain-score.ts @@ -74,7 +74,7 @@ import { import { shiftDateKey } from "@/lib/tz/format"; import { pickCanonicalWorkoutRows, - workoutDuplicateSlotKey, + workoutDuplicateClusterKeys, workoutSourceLadder, type WorkoutPickerRow, } from "@/lib/measurements/pick-canonical-workout-rows"; @@ -591,7 +591,7 @@ export function canonicalDayActiveEnergy< * one source's rows count: the best-ranked on the ladder, a source off the * ladder ranking below every listed one, and among sources off the ladder * the one with the largest impulse. Rows of that one source still add up - * (two distinct sessions of the same sport in one five-minute slot). + * (two distinct sessions of the same sport in one duplicate cluster). * Returns the positive impulses, one per session. Pure; exported for tests. */ export function canonicalWorkoutTrimps< @@ -615,9 +615,9 @@ export function canonicalWorkoutTrimps< string, { picked: Map; dropped: number } >(); - for (const row of rows) { - const key = workoutDuplicateSlotKey(row); - const slot = slots.get(key) ?? { picked: new Map(), dropped: 0 }; + const keys = workoutDuplicateClusterKeys(rows); + for (const [index, row] of rows.entries()) { + const slot = slots.get(keys[index]!) ?? { picked: new Map(), dropped: 0 }; const trimp = trimpOf(row); if (trimp > 0) { if (picked.has(row.id)) { @@ -626,7 +626,7 @@ export function canonicalWorkoutTrimps< slot.dropped = Math.max(slot.dropped, trimp); } } - slots.set(key, slot); + slots.set(keys[index]!, slot); } const out: number[] = []; for (const slot of slots.values()) { diff --git a/src/lib/measurements/__tests__/pick-canonical-workout-rows.legacy.ts b/src/lib/measurements/__tests__/pick-canonical-workout-rows.legacy.ts new file mode 100644 index 000000000..69c27af81 --- /dev/null +++ b/src/lib/measurements/__tests__/pick-canonical-workout-rows.legacy.ts @@ -0,0 +1,329 @@ +/** + * The workout picker as it stood while duplicates were matched by fixed + * five-minute clock slots. Kept only as the reference the parity tests in + * `pick-canonical-workout-rows.property.test.ts` compare the duplicate window + * against: on every input where no two workouts of one sport within five + * minutes straddle a slot boundary, both must return the same rows. + */ +import type { MeasurementSource } from "@/generated/prisma/client"; + +import { + getSourceLadder, + parseSourcePriority, +} from "@/lib/validations/source-priority"; + +/** + * Minimum row shape — same posture as `SourcePickerRow`. The helper + * only consults the three fields; downstream callers can read every + * other column they fetched alongside. + */ +export interface WorkoutPickerRow { + userId?: string; + startedAt: Date; + /** + * Session length. Together with `sportType` + `startedAt` this is the + * exact-session identity used to collapse repeated same-source sync rows. + * Optional only so fixtures can omit it; every production call site MUST + * select it, and `workout-canonical-identity-guard.test.ts` freezes that set. + */ + durationSec?: number; + sportType: string; + source: MeasurementSource; + avgHeartRate?: number | null; + maxHeartRate?: number | null; + metadata?: unknown; +} + +/** + * HealthKit can resend the same session with a new external UUID. The source + * ladder correctly chooses Apple over WHOOP, but previously kept every Apple + * row in that winning bucket, which made the web list show the same workout + * two or three times. + * + * The identity is `(sportType, startedAt, durationSec)`. `endedAt` was part of + * this key when the collapse first shipped and has been dropped deliberately: + * + * - It is not a free extra check. On a workout with pauses, `endedAt` is the + * wall-clock end while `durationSec` counts only moving time, so the two + * legitimately disagree, and a stored row can carry either convention. Two + * re-sends of one session can therefore differ in `endedAt` alone, and the + * stricter key let that duplicate through. + * - It bought nothing in exchange. Nobody starts a second workout of the + * same sport in the same instant with the same duration, so start plus + * duration already identifies the session uniquely. + * - Requiring it split the callers. Four of the five call sites select + * `durationSec` but not `endedAt`, so the collapse ran on the workout list + * and silently did nothing for the Coach narration, the per-sport baseline + * and the deep-link cluster. One list, one number, two answers. + * + * A row without `durationSec` cannot be identified and passes through + * uncollapsed. That is a fixture-only path: every production call site selects + * the field and `workout-canonical-identity-guard.test.ts` keeps it that way. + */ +function exactSessionKey(row: WorkoutPickerRow): string | null { + if (row.durationSec == null) return null; + return [row.sportType, row.startedAt.getTime(), row.durationSec].join(":"); +} + +function rowRichness(row: WorkoutPickerRow): number { + return ( + (row.avgHeartRate != null ? 2 : 0) + + (row.maxHeartRate != null ? 2 : 0) + + (row.metadata != null ? 1 : 0) + ); +} + +function deduplicateWinningRows(rows: T[]): T[] { + const bySession = new Map(); + const passthrough: T[] = []; + for (const row of rows) { + const key = exactSessionKey(row); + if (key == null) { + passthrough.push(row); + continue; + } + const existing = bySession.get(key); + if (!existing || rowRichness(row) > rowRichness(existing)) { + bySession.set(key, row); + } + } + return [...passthrough, ...bySession.values()]; +} + +/** + * Bucket-slot length. Two workouts whose `startedAt` falls in the + * same 5-minute slot AND share the same sportType are treated as the + * "same" workout from different sources. Tightens further than the + * 1-minute clock skew between HK + Withings would suggest because + * iOS posts HK `startDate` rounded to the second but Withings stamps + * its `startdate` independently — five minutes gives both heads + * enough slack on a heavy-traffic morning sync without crossing the + * "two back-to-back runs" boundary (a user does NOT do two ten-minute + * runs five minutes apart in practice). + */ +const BUCKET_SLOT_MS = 5 * 60 * 1000; + +/** + * The cross-source duplicate slot a workout falls in. Exported for the one + * reader that has to look past the picked row: the strain engine, whose winner + * may carry no heart-rate series while its dropped twin does. + */ +export function legacyWorkoutDuplicateSlotKey(row: WorkoutPickerRow): string { + return bucketKey(row); +} + +function bucketKey(row: WorkoutPickerRow): string { + const slot = Math.floor(row.startedAt.getTime() / BUCKET_SLOT_MS); + return `${row.userId ?? ""}:${slot}:${row.sportType}`; +} + +function jsonRecord(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +const WHOOP_ZONE_DURATION_KEYS = [ + "zone_zero_milli", + "zone_one_milli", + "zone_two_milli", + "zone_three_milli", + "zone_four_milli", + "zone_five_milli", +] as const; + +function sanitizedWhoopZoneDurations( + metadata: unknown, +): Record | null { + const raw = jsonRecord(jsonRecord(metadata)?.zoneDurations); + if (!raw) return null; + + const sanitized: Record = {}; + for (const key of WHOOP_ZONE_DURATION_KEYS) { + const value = raw[key]; + if (value === undefined) continue; + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + return null; + } + sanitized[key] = value; + } + return Object.keys(sanitized).length > 0 ? sanitized : null; +} + +function enrichFromWhoopTwin( + winner: T, + donor: T, +): T { + const winnerMetadata = jsonRecord(winner.metadata); + const donorZones = sanitizedWhoopZoneDurations(donor.metadata); + const needsZones = + donorZones != null && winnerMetadata?.zoneDurations == null; + const needsAvg = winner.avgHeartRate == null && donor.avgHeartRate != null; + const needsMax = winner.maxHeartRate == null && donor.maxHeartRate != null; + + if (!needsAvg && !needsMax && !needsZones) return winner; + + return { + ...winner, + ...(needsAvg ? { avgHeartRate: donor.avgHeartRate } : {}), + ...(needsMax ? { maxHeartRate: donor.maxHeartRate } : {}), + ...(needsZones + ? { + metadata: { + ...(winnerMetadata ?? {}), + zoneDurations: donorZones, + }, + } + : {}), + }; +} + +/** + * Pair winners and WHOOP donors one-to-one by nearest start time. A single + * donor can never enrich two same-source occurrences that happen to share a + * fixed slot. Stable input indexes break equal-distance ties. + */ +function enrichWinnersFromWhoop( + winners: readonly T[], + donors: readonly T[], +): T[] { + if (winners.length === 0 || donors.length === 0) return [...winners]; + + const candidates: Array<{ + winnerIndex: number; + donorIndex: number; + deltaMs: number; + }> = []; + winners.forEach((winner, winnerIndex) => { + donors.forEach((donor, donorIndex) => { + candidates.push({ + winnerIndex, + donorIndex, + deltaMs: Math.abs( + winner.startedAt.getTime() - donor.startedAt.getTime(), + ), + }); + }); + }); + candidates.sort( + (left, right) => + left.deltaMs - right.deltaMs || + left.winnerIndex - right.winnerIndex || + left.donorIndex - right.donorIndex, + ); + + const enriched = [...winners]; + const matchedWinners = new Set(); + const matchedDonors = new Set(); + for (const candidate of candidates) { + if ( + matchedWinners.has(candidate.winnerIndex) || + matchedDonors.has(candidate.donorIndex) + ) { + continue; + } + enriched[candidate.winnerIndex] = enrichFromWhoopTwin( + winners[candidate.winnerIndex], + donors[candidate.donorIndex], + ); + matchedWinners.add(candidate.winnerIndex); + matchedDonors.add(candidate.donorIndex); + } + return enriched; +} + +/** + * The source order for workouts: the `steps` metric's ladder, the + * canonical-activity bucket the Sources settings configure, falling back to + * its default when the user saved none or it does not parse. + */ +function workoutSourceLadder( + userPriorityJson: unknown, +): readonly MeasurementSource[] { + return getSourceLadder(parseSourcePriority(userPriorityJson), "steps"); +} + +/** + * Pick canonical rows across sources for a user's workouts. + * + * @param rows input rows (deterministic order required) + * @param userPriorityJson the user's `sourcePriorityJson` blob — same + * shape the measurement picker consults. + * Falls back to the canonical default ladder + * when null / unparseable. + * @returns the subset of input rows that survives the picker. Insertion + * order preserved. + */ +export function legacyPickCanonicalWorkoutRows( + rows: readonly T[], + userPriorityJson: unknown = null, +): T[] { + if (rows.length === 0) return []; + + const sourceLadder = workoutSourceLadder(userPriorityJson); + + const buckets = new Map< + string, + { + rows: T[]; + sources: Set; + } + >(); + for (const row of rows) { + const key = bucketKey(row); + const slot = buckets.get(key) ?? { rows: [], sources: new Set() }; + slot.rows.push(row); + slot.sources.add(row.source); + buckets.set(key, slot); + } + + const canonical: T[] = []; + for (const [, slot] of buckets) { + if (slot.rows.length === 1) { + // No cross-source overlap → keep the single row regardless of + // ladder membership. + canonical.push(slot.rows[0]); + continue; + } + + let picked: MeasurementSource | undefined; + for (const source of sourceLadder) { + if (slot.sources.has(source)) { + picked = source; + break; + } + } + + if (!picked) { + // None of the bucket's sources are listed in the ladder — never + // silently drop. Keep every row in the bucket; downstream + // callers can choose to surface a "duplicate from unrecognised + // sources" diagnostic. + canonical.push(...slot.rows); + continue; + } + + const winners = deduplicateWinningRows( + slot.rows.filter((row) => row.source === picked), + ); + const whoopDonors = + picked === "WHOOP" + ? [] + : slot.rows.filter((row) => row.source === "WHOOP"); + canonical.push(...enrichWinnersFromWhoop(winners, whoopDonors)); + } + + // Re-sort to preserve input order — the bucket map iterates in + // insertion order but the per-bucket fan-out can interleave rows + // from later buckets ahead of earlier-bucket survivors. Caller + // wants the same `startedAt ASC` order it passed in. + canonical.sort((a, b) => { + const delta = a.startedAt.getTime() - b.startedAt.getTime(); + if (delta !== 0) return delta; + // Stable-tiebreak on sportType so the output is deterministic + // even when two surviving rows share startedAt (rare; manual + // entries on the same second). + return a.sportType.localeCompare(b.sportType); + }); + return canonical; +} diff --git a/src/lib/measurements/__tests__/pick-canonical-workout-rows.property.test.ts b/src/lib/measurements/__tests__/pick-canonical-workout-rows.property.test.ts new file mode 100644 index 000000000..009643764 --- /dev/null +++ b/src/lib/measurements/__tests__/pick-canonical-workout-rows.property.test.ts @@ -0,0 +1,300 @@ +import { describe, expect, it } from "vitest"; + +import { canonicalWorkoutTrimps } from "@/lib/insights/strain-score"; + +import { + pickCanonicalWorkoutRows, + workoutDuplicateClusterKeys, + workoutSourceLadder, +} from "../pick-canonical-workout-rows"; +import { + legacyPickCanonicalWorkoutRows, + legacyWorkoutDuplicateSlotKey, +} from "./pick-canonical-workout-rows.legacy"; + +type Source = "APPLE_HEALTH" | "WHOOP" | "WITHINGS" | "MANUAL" | "STRAVA"; + +interface Row { + id: string; + userId?: string; + startedAt: Date; + durationSec?: number; + sportType: string; + source: Source; + avgHeartRate: number | null; + maxHeartRate: number | null; + metadata: Record | null; +} + +const MIN = 60_000; +const WINDOW = 5 * MIN; +const SOURCES: Source[] = [ + "APPLE_HEALTH", + "WHOOP", + "WITHINGS", + "MANUAL", + "STRAVA", +]; +const SPORTS = ["running", "cycling", "walking"]; + +/** Small seeded generator, so a failure names a reproducible case. */ +function rng(seed: number) { + let a = seed >>> 0; + const next = () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + return { + next, + int: (lo: number, hi: number) => lo + Math.floor(next() * (hi - lo + 1)), + pick: (list: readonly T[]) => list[Math.floor(next() * list.length)]!, + }; +} + +function randomRows(seed: number): Row[] { + const r = rng(seed); + const base = Date.UTC(2026, 4, 16, 6, 0, 0) + r.int(0, 86_400_000); + const count = r.int(1, 14); + const rows: Row[] = []; + for (let i = 0; i < count; i++) { + // Mostly near each other, so clusters form; sometimes a re-send of an + // earlier row (same start and duration) under another id. + const twin = i > 0 && r.next() < 0.2 ? rows[r.int(0, i - 1)]! : null; + const startedAt = twin + ? new Date( + twin.startedAt.getTime() + (r.next() < 0.5 ? 0 : r.int(1, 90) * 1000), + ) + : new Date(base + r.int(0, 40 * MIN)); + rows.push({ + id: `w-${seed}-${i}`, + userId: r.next() < 0.15 ? "other" : "u1", + startedAt, + durationSec: + r.next() < 0.1 ? undefined : (twin?.durationSec ?? r.int(10, 90) * 60), + sportType: twin && r.next() < 0.8 ? twin.sportType : r.pick(SPORTS), + source: r.pick(SOURCES), + avgHeartRate: r.next() < 0.5 ? r.int(100, 160) : null, + maxHeartRate: r.next() < 0.5 ? r.int(160, 190) : null, + metadata: + r.next() < 0.3 + ? { zoneDurations: { zone_two_milli: r.int(1, 60) * MIN } } + : null, + }); + } + // Callers pass `startedAt ASC, id ASC`. + rows.sort( + (a, b) => + a.startedAt.getTime() - b.startedAt.getTime() || a.id.localeCompare(b.id), + ); + return rows; +} + +/** + * True when some pair of one (user, sport) lies within the window but in two + * different fixed slots: the only inputs on which the old picker and the + * duplicate window may disagree. + */ +function straddlesASlot(rows: readonly Row[]): boolean { + for (let i = 0; i < rows.length; i++) { + for (let j = i + 1; j < rows.length; j++) { + const a = rows[i]!; + const b = rows[j]!; + if (a.userId !== b.userId || a.sportType !== b.sportType) continue; + if (Math.abs(a.startedAt.getTime() - b.startedAt.getTime()) >= WINDOW) + continue; + if (legacyWorkoutDuplicateSlotKey(a) !== legacyWorkoutDuplicateSlotKey(b)) + return true; + } + } + return false; +} + +const LADDERS: unknown[] = [ + null, + { steps: ["WITHINGS", "APPLE_HEALTH"] }, + { steps: ["WHOOP"] }, +]; + +/** The strain fold as it stood on fixed slots, for the parity check. */ +function legacyTrimps( + rows: readonly Row[], + ladderJson: unknown, + trimpOf: (r: Row) => number, +) { + const picked = new Set( + legacyPickCanonicalWorkoutRows(rows, ladderJson).map((r) => r.id), + ); + const ladder = workoutSourceLadder(ladderJson); + const rankOf = (source: string) => { + const index = ladder.indexOf(source as (typeof ladder)[number]); + return index === -1 ? ladder.length : index; + }; + const slots = new Map< + string, + { picked: Map; dropped: number } + >(); + for (const row of rows) { + const key = legacyWorkoutDuplicateSlotKey(row); + const slot = slots.get(key) ?? { picked: new Map(), dropped: 0 }; + const t = trimpOf(row); + if (t > 0) { + if (picked.has(row.id)) + slot.picked.set(row.source, (slot.picked.get(row.source) ?? 0) + t); + else slot.dropped = Math.max(slot.dropped, t); + } + slots.set(key, slot); + } + const out: number[] = []; + for (const slot of slots.values()) { + let best: { rank: number; trimp: number } | null = null; + for (const [source, trimp] of slot.picked) { + const rank = rankOf(source); + if ( + best === null || + rank < best.rank || + (rank === best.rank && trimp > best.trimp) + ) { + best = { rank, trimp }; + } + } + const trimp = best !== null ? best.trimp : slot.dropped; + if (trimp > 0) out.push(trimp); + } + return out; +} + +describe("workout duplicate window", () => { + it("returns exactly the old rows wherever no pair straddles a slot boundary", () => { + let compared = 0; + for (let seed = 1; seed <= 4000; seed++) { + const rows = randomRows(seed); + if (straddlesASlot(rows)) continue; + compared++; + for (const ladder of LADDERS) { + expect(pickCanonicalWorkoutRows(rows, ladder), `seed ${seed}`).toEqual( + legacyPickCanonicalWorkoutRows(rows, ladder), + ); + } + } + // The generator must actually produce comparable inputs, clusters + // included, or the loop above proves nothing. + expect(compared).toBeGreaterThan(1000); + }); + + it("gives the strain fold the same impulses wherever no pair straddles a slot", () => { + const trimpOf = (row: Row) => row.avgHeartRate ?? 0; + let compared = 0; + for (let seed = 1; seed <= 2000; seed++) { + const rows = randomRows(seed); + if (straddlesASlot(rows)) continue; + compared++; + for (const ladder of LADDERS) { + expect( + canonicalWorkoutTrimps(rows, ladder, trimpOf), + `seed ${seed}`, + ).toEqual(legacyTrimps(rows, ladder, trimpOf)); + } + } + expect(compared).toBeGreaterThan(500); + }); + + it("collapses two sources' copies of one session at any offset under five minutes", () => { + const r = rng(7); + let legacyMissed = 0; + for (let i = 0; i < 3000; i++) { + const start = Date.UTC(2026, 4, 16) + r.int(0, 86_400_000); + const offset = r.int(0, WINDOW - 1); + const rows: Row[] = [ + { + id: "apple", + userId: "u1", + startedAt: new Date(start), + durationSec: 1800, + sportType: "running", + source: "APPLE_HEALTH", + avgHeartRate: null, + maxHeartRate: null, + metadata: null, + }, + { + id: "withings", + userId: "u1", + startedAt: new Date(start + offset), + durationSec: 1790, + sportType: "running", + source: "WITHINGS", + avgHeartRate: null, + maxHeartRate: null, + metadata: null, + }, + ]; + expect(pickCanonicalWorkoutRows(rows).map((row) => row.id)).toEqual([ + "apple", + ]); + if (legacyPickCanonicalWorkoutRows(rows).length === 2) legacyMissed++; + } + // The fixed slots missed a share of these; the window misses none. + expect(legacyMissed).toBeGreaterThan(0); + }); + + it("never collapses two workouts five minutes or more apart", () => { + const r = rng(11); + for (let i = 0; i < 2000; i++) { + const start = Date.UTC(2026, 4, 16) + r.int(0, 86_400_000); + const gap = WINDOW + r.int(0, 30 * MIN); + const rows: Row[] = ["APPLE_HEALTH", "WITHINGS"].map((source, k) => ({ + id: `w${k}`, + userId: "u1", + startedAt: new Date(start + k * gap), + durationSec: 1200, + sportType: "running", + source: source as Source, + avgHeartRate: null, + maxHeartRate: null, + metadata: null, + })); + expect(pickCanonicalWorkoutRows(rows)).toHaveLength(2); + } + }); + + it("does not depend on where the clock's five-minute marks fall", () => { + // Shifting every start by the same amount keeps every gap, so it must + // keep the picked set. The fixed slots broke this property. + for (let seed = 1; seed <= 1500; seed++) { + const rows = randomRows(seed); + const shift = rng(seed * 31).int(1, WINDOW - 1); + const shifted = rows.map((row) => ({ + ...row, + startedAt: new Date(row.startedAt.getTime() + shift), + })); + const ids = (list: Row[]) => list.map((row) => row.id).sort(); + expect(ids(pickCanonicalWorkoutRows(shifted)), `seed ${seed}`).toEqual( + ids(pickCanonicalWorkoutRows(rows)), + ); + } + }); + + it("anchors on the first start, so a row of workouts does not chain", () => { + const at = (m: number) => new Date(Date.UTC(2026, 4, 16, 8, m, 0)); + const rows: Row[] = [0, 4, 8, 12].map((m, k) => ({ + id: `w${m}`, + userId: "u1", + startedAt: at(m), + durationSec: 600, + sportType: "running", + source: (k % 2 === 0 ? "APPLE_HEALTH" : "WITHINGS") as Source, + avgHeartRate: null, + maxHeartRate: null, + metadata: null, + })); + const keys = workoutDuplicateClusterKeys(rows); + expect(new Set(keys).size).toBe(2); + expect(pickCanonicalWorkoutRows(rows).map((row) => row.id)).toEqual([ + "w0", + "w8", + ]); + }); +}); diff --git a/src/lib/measurements/__tests__/pick-canonical-workout-rows.test.ts b/src/lib/measurements/__tests__/pick-canonical-workout-rows.test.ts index 29e43e414..ff4411167 100644 --- a/src/lib/measurements/__tests__/pick-canonical-workout-rows.test.ts +++ b/src/lib/measurements/__tests__/pick-canonical-workout-rows.test.ts @@ -450,7 +450,12 @@ describe("pickCanonicalWorkoutRows", () => { ]); }); - it("does not chain neighbouring workouts across fixed canonical slots", () => { + it("does not chain neighbouring workouts past the first start's window", () => { + // `middle` starts four minutes after `first`: inside the duplicate + // window, so it is the same run from a lower-ranked source. `last` is + // eight minutes after `first`, outside the window measured from the + // cluster's first start, so it stays even though it is only four minutes + // after `middle`. const rows: RowFixture[] = [ { id: "first", @@ -474,7 +479,6 @@ describe("pickCanonicalWorkoutRows", () => { expect(pickCanonicalWorkoutRows(rows).map((row) => row.id)).toEqual([ "first", - "middle", "last", ]); }); diff --git a/src/lib/measurements/pick-canonical-workout-rows.ts b/src/lib/measurements/pick-canonical-workout-rows.ts index d426cb1c0..de26a85ef 100644 --- a/src/lib/measurements/pick-canonical-workout-rows.ts +++ b/src/lib/measurements/pick-canonical-workout-rows.ts @@ -9,11 +9,14 @@ * * Algorithm (v1.4.30 — matches the measurement ladder per R-F open * question #4): - * 1. Bucket workouts by the rounded start timestamp + sport-type - * (`<5-minute slot>:`). Two HKWorkout entries for the same - * logical run start within a minute of each other on both - * sources; the rounding gives the cross-source matcher slack - * without collapsing distinct back-to-back workouts. + * 1. Cluster workouts per user and sport-type by start time: sorted by + * start, a workout joins the open cluster when it starts less than + * five minutes after the cluster's first workout, and opens a new + * one otherwise. Two HKWorkout entries for the same logical run + * start within a minute of each other on both sources; the window + * gives the cross-source matcher slack without collapsing distinct + * back-to-back workouts, and anchoring on the first start keeps a + * row of workouts a few minutes apart from chaining into one. * 2. For each bucket, walk the source ladder (`workoutSourceLadder`) * and pick the FIRST source that has any row in that bucket. Keep * every row from the winning source; drop the rest. @@ -115,30 +118,58 @@ function deduplicateWinningRows(rows: T[]): T[] { } /** - * Bucket-slot length. Two workouts whose `startedAt` falls in the - * same 5-minute slot AND share the same sportType are treated as the - * "same" workout from different sources. Tightens further than the - * 1-minute clock skew between HK + Withings would suggest because - * iOS posts HK `startDate` rounded to the second but Withings stamps - * its `startdate` independently — five minutes gives both heads - * enough slack on a heavy-traffic morning sync without crossing the - * "two back-to-back runs" boundary (a user does NOT do two ten-minute - * runs five minutes apart in practice). + * The cross-source duplicate window. Two workouts of the same sport whose + * starts lie less than five minutes apart are treated as the "same" workout + * from different sources. iOS posts the HealthKit `startDate` rounded to the + * second while Withings stamps its `startdate` independently; five minutes + * gives both heads enough slack on a heavy-traffic morning sync without + * crossing the "two back-to-back runs" boundary (a user does NOT do two + * ten-minute runs five minutes apart in practice). + * + * This used to be a fixed five-minute clock slot (`floor(start / 5 min)`), + * so two copies of one session thirty seconds apart that straddled a slot + * boundary (08:04:50 and 08:05:20) were never compared. The window is now + * measured from the cluster's first start. Where no two workouts of one sport + * within the window straddle a slot boundary, the clusters are exactly the + * old slots: every pair inside one slot is less than five minutes apart, and + * a cluster cannot reach past its slot without such a pair. */ -const BUCKET_SLOT_MS = 5 * 60 * 1000; +const DUPLICATE_WINDOW_MS = 5 * 60 * 1000; /** - * The cross-source duplicate slot a workout falls in. Exported for the one - * reader that has to look past the picked row: the strain engine, whose winner - * may carry no heart-rate series while its dropped twin does. + * The duplicate cluster of every row, aligned with `rows`. Rows of different + * users or sports never share a cluster. Within one (user, sport), rows are + * taken in start order (input order on equal starts) and a row joins the open + * cluster while it starts less than {@link DUPLICATE_WINDOW_MS} after the + * cluster's first row. The key names the cluster by that first row's start, + * so it is stable for a given set of rows. Exported for the one reader that + * has to look past the picked row: the strain engine, whose winner may carry + * no heart-rate series while its dropped twin does. */ -export function workoutDuplicateSlotKey(row: WorkoutPickerRow): string { - return bucketKey(row); -} - -function bucketKey(row: WorkoutPickerRow): string { - const slot = Math.floor(row.startedAt.getTime() / BUCKET_SLOT_MS); - return `${row.userId ?? ""}:${slot}:${row.sportType}`; +export function workoutDuplicateClusterKeys( + rows: readonly WorkoutPickerRow[], +): string[] { + const groups = new Map(); + rows.forEach((row, index) => { + const group = `${row.userId ?? ""}:${row.sportType}`; + const list = groups.get(group); + if (list) list.push(index); + else groups.set(group, [index]); + }); + const keys = new Array(rows.length); + for (const [group, indexes] of groups) { + indexes.sort( + (a, b) => + rows[a]!.startedAt.getTime() - rows[b]!.startedAt.getTime() || a - b, + ); + let anchor = Number.NaN; + for (const index of indexes) { + const start = rows[index]!.startedAt.getTime(); + if (!(start - anchor < DUPLICATE_WINDOW_MS)) anchor = start; + keys[index] = `${group}:${anchor}`; + } + } + return keys; } function jsonRecord(value: unknown): Record | null { @@ -205,7 +236,7 @@ function enrichFromWhoopTwin( /** * Pair winners and WHOOP donors one-to-one by nearest start time. A single * donor can never enrich two same-source occurrences that happen to share a - * fixed slot. Stable input indexes break equal-distance ties. + * duplicate cluster. Stable input indexes break equal-distance ties. */ function enrichWinnersFromWhoop( winners: readonly T[], @@ -293,8 +324,9 @@ export function pickCanonicalWorkoutRows( sources: Set; } >(); - for (const row of rows) { - const key = bucketKey(row); + const clusterKeys = workoutDuplicateClusterKeys(rows); + for (const [index, row] of rows.entries()) { + const key = clusterKeys[index]!; const slot = buckets.get(key) ?? { rows: [], sources: new Set() }; slot.rows.push(row); slot.sources.add(row.source); From 63e8630fcb5cebe55e65b8870ae3c5781d1282ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 22 Oct 2026 06:41:15 +0200 Subject: [PATCH 02/19] perf(bench): profile a production boot and the idle heap scripts/bench/boot-profile.ts starts the production build with every role under the production heap ceiling against the seeded benchmark database, and records the time to a healthy probe and to a ready worker, every event-loop stall the process logs, the heap lines, the full collections from --trace-gc, the main thread's busy stretches during the boot with their self time by source module (resolved through the chunk source maps), and one heap snapshot after an idle wait, summarised by node type, constructor, largest strings and largest retained subtrees. The baseline on the current runtime: a 753 ms busy stretch at 0.7 s, most of it the worker graph, and a live heap of 164 to 172 MB after five idle minutes, of which the query compiler holds about 50 MB and the eight locale catalogs about 33 MB (32 parsed copies and two 9.7 MB chunk sources). --- scripts/bench/boot-profile.ts | 398 +++++ scripts/bench/profile-analyze.ts | 599 ++++++++ .../bench/results/v1.45.0-before-boot.json | 1351 +++++++++++++++++ scripts/bench/results/v1.45.0-before-boot.md | 221 +++ 4 files changed, 2569 insertions(+) create mode 100644 scripts/bench/boot-profile.ts create mode 100644 scripts/bench/profile-analyze.ts create mode 100644 scripts/bench/results/v1.45.0-before-boot.json create mode 100644 scripts/bench/results/v1.45.0-before-boot.md diff --git a/scripts/bench/boot-profile.ts b/scripts/bench/boot-profile.ts new file mode 100644 index 000000000..e2e499bfe --- /dev/null +++ b/scripts/bench/boot-profile.ts @@ -0,0 +1,398 @@ +/** + * Boot and idle-memory profile of the production build. + * + * DATABASE_URL=postgresql://… API_TOKEN_HMAC_KEY=… ENCRYPTION_KEYS=… \ + * ENCRYPTION_ACTIVE_KEY_ID=… pnpm dlx tsx scripts/bench/boot-profile.ts \ + * [--idle-seconds 300] [--heap-mb 792] [--port 3101] \ + * [--no-snapshot] [--no-cpu-prof] [--out scripts/bench/results/