/** * The full-history extremes read the DAY rollup tier and give the figures the * raw-row fold gave. * * `readAllTimeExtremes` used to fold every raw reading of the four types on * each call. It now folds the DAY tier for the whole days inside the fold * window and the raw rows for the rest. The reference below is that earlier * query, verbatim, so the comparison is against what the function used to * answer, not against a second derivation of the same idea. * * The seed has what the two paths could disagree on: two sources on the same * days (weight by scale and by hand, pulse from a watch's hourly statistics * with their own range and a strap's dense samples), readings older than the * fold window, readings either side of a UTC midnight, soft-deleted rows, and * a reading deleted after the tier was built (through the write path's own * rollup recompute). * * Means are compared to nine significant digits: the tier sums the same day * values in a different order. Extremes are compared exactly. */ import { beforeAll, describe, expect, it } from "vitest"; import { getPrismaClient, truncateAllTables } from "./setup"; import { Prisma, type MeasurementType } from "@/generated/prisma/client"; import { readAllTimeExtremes } from "@/lib/insights/feature-blocks"; import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql"; import { dayWeightedRowsSql, foldMeanSql, isHourlyMeanTypeSql, } from "@/lib/measurements/day-mean"; import { loadUserSourcePriority } from "@/lib/rollups/measurement-read"; import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups"; import { afterMeasurementMutation } from "@/lib/rollups/after-measurement-mutation"; const prisma = getPrismaClient(); const USER = "all-time-tier-parity"; const TYPES: MeasurementType[] = [ "WEIGHT", "BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA", "PULSE", ]; const HOUR = 3_600_000; const DAY = 24 * HOUR; const today = Date.UTC( new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate(), ); /** The query `readAllTimeExtremes` ran before it read the DAY tier. */ async function referenceExtremes( userId: string, types: readonly MeasurementType[], floor: Date | null, ) { const out = new Map(); const since = floor === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${floor}`; const priorityJson = await loadUserSourcePriority(userId); const rank = Prisma.raw( buildSourceRankCase(priorityJson, 'p."type"', 'p."source"'), ); const typeList = Prisma.join( types.map((t) => Prisma.sql`${t}::measurement_type`), ); const mean = Prisma.raw( foldMeanSql({ typeColumn: 'c."type"', total: "c.total", count: "c.cnt", weightedSum: "c.wsum", weightSum: "c.wdays", }), ); const rows = await prisma.$queryRaw< Array<{ type: string; mean: number; min: number; max: number }> >` WITH hourly_src AS ( SELECT m.* FROM measurements m WHERE m."user_id" = ${userId} AND m."deleted_at" IS NULL AND m."type" IN (${typeList}) ${since} AND ${Prisma.raw(isHourlyMeanTypeSql('m."type"'))} ), weighted AS ( SELECT w."type", w."source", date_trunc('day', w."measured_at") AS day, SUM(w."value" * w.day_weight)::double precision AS wsum, SUM(w.day_weight)::double precision AS wdays FROM ${dayWeightedRowsSql("hourly_src", null, { bySource: true })} w GROUP BY w."type", w."source", 3 ), per_source AS ( SELECT m."type", m."source", date_trunc('day', m."measured_at") AS day, COUNT(*)::int AS cnt, SUM(m."value")::double precision AS total, MIN(m."value")::double precision AS min_value, MAX(m."value")::double precision AS max_value FROM measurements m WHERE m."user_id" = ${userId} AND m."deleted_at" IS NULL AND m."type" IN (${typeList}) ${since} GROUP BY m."type", m."source", 3 ), canon AS ( SELECT DISTINCT ON (p."type", p.day) p."type", p.cnt, p.total, p.min_value, p.max_value, w.wsum, w.wdays FROM per_source p LEFT JOIN weighted w ON w."type" = p."type" AND w."source" = p."source" AND w.day = p.day ORDER BY p."type", p.day, (${rank}), p."source"::text ) SELECT c."type"::text AS type, ${mean}::double precision AS mean, MIN(c.min_value)::double precision AS min, MAX(c.max_value)::double precision AS max FROM canon c GROUP BY c."type" `; for (const r of rows) { out.set(r.type, { mean: r.mean, min: r.min, max: r.max }); } return out; } /** A small deterministic generator, so a failure reproduces. */ function lcg(seed: number): () => number { let s = seed; return () => { s = (s * 1_103_515_245 + 12_345) % 2_147_483_648; return s / 2_147_483_648; }; } function seed(): Prisma.MeasurementCreateManyInput[] { const rnd = lcg(7); const rows: Prisma.MeasurementCreateManyInput[] = []; let n = 0; const add = ( type: MeasurementType, unit: string, source: Prisma.MeasurementCreateManyInput["source"], at: number, value: number, extra: Partial = {}, ) => { n += 1; rows.push({ id: `${USER}-${n}`, userId: USER, type, unit, source, value, measuredAt: new Date(at), ...extra, }); }; // Weight and blood pressure across six years: older than the fold window // too. Every fifth day a hand entry beside the scale's, every 23rd day a // reading a minute either side of UTC midnight, and a few tombstones. for (let d = 6 * 365; d >= 0; d -= 1) { const day = today - d * DAY; add("WEIGHT", "kg", "WITHINGS", day + 6 * HOUR, 70 + 10 * rnd()); if (d % 5 === 0) add("WEIGHT", "kg", "MANUAL", day + 20 * HOUR, 69 + 12 * rnd()); for (const [type, base] of [ ["BLOOD_PRESSURE_SYS", 110], ["BLOOD_PRESSURE_DIA", 65], ] as const) { add(type, "mmHg", "WITHINGS", day + 7 * HOUR, base + 40 * rnd()); if (d % 23 === 0) { add(type, "mmHg", "WITHINGS", day - 60_000, base + 45 * rnd()); add(type, "mmHg", "WITHINGS", day + 60_000, base - 5 + 50 * rnd()); } if (d % 31 === 0) { add(type, "mmHg", "WITHINGS", day + 9 * HOUR, base + 80, { deletedAt: new Date(), }); } } } // Pulse over the last 400 days: a watch's hourly statistics carrying the // hour's own min / max, and a strap's dense samples on every other day. for (let d = 400; d >= 0; d -= 1) { const day = today - d * DAY; for (let h = 0; h < 24; h += 3) { const avg = 55 + 30 * rnd(); add("PULSE", "bpm", "APPLE_HEALTH", day + h * HOUR, avg, { valueMin: avg - 15, valueMax: avg + 40, externalId: `stats:HKQuantityTypeIdentifierHeartRate:${day + h * HOUR}`, }); } if (d % 2 === 0) { for (let i = 0; i < 30; i += 1) { add( "PULSE", "bpm", "WHOOP", day + 10 * HOUR + i * 120_000, 50 + 70 * rnd(), ); } } } return rows; } async function expectParity(floor: Date | null) { const [actual, reference] = await Promise.all([ readAllTimeExtremes(USER, TYPES, floor), referenceExtremes(USER, TYPES, floor), ]); expect([...actual.keys()].sort()).toEqual([...reference.keys()].sort()); expect(reference.size).toBe(4); for (const [type, ref] of reference) { const got = actual.get(type as MeasurementType)!; expect(got.min, type).toBe(ref.min); expect(got.max, type).toBe(ref.max); expect(got.mean! / ref.mean, type).toBeCloseTo(1, 9); } } beforeAll(async () => { await truncateAllTables(prisma); await prisma.user.create({ data: { id: USER, username: USER, timezone: "Pacific/Auckland" }, }); const rows = seed(); for (let i = 0; i < rows.length; i += 5000) { await prisma.measurement.createMany({ data: rows.slice(i, i + 5000) }); } await recomputeUserRollups(USER, { granularities: ["DAY"] }); }, 300_000); describe("readAllTimeExtremes over the DAY tier", () => { it("reads the tier for every type", async () => { const covered = await prisma.measurementRollup.groupBy({ by: ["type"], where: { userId: USER, granularity: "DAY", readingMinValue: { not: null }, }, }); expect(covered.map((c) => c.type).sort()).toEqual([...TYPES].sort()); }); it("gives the raw fold's figures over the whole history", async () => { await expectParity(null); }); it("gives the raw fold's figures from a floor inside a day", async () => { await expectParity(new Date(today - 200 * DAY + 13 * HOUR + 17 * 60_000)); }); it("follows a deletion through the write path's recompute", async () => { const hottest = await prisma.measurement.findFirst({ where: { userId: USER, type: "PULSE", source: "WHOOP", deletedAt: null }, orderBy: { value: "desc" }, }); await prisma.measurement.update({ where: { id: hottest!.id }, data: { deletedAt: new Date() }, }); await afterMeasurementMutation( USER, [{ type: "PULSE", measuredAt: hottest!.measuredAt }], { source: "manual" }, ); await expectParity(null); }); it("is the tier that answers, not the raw rows", async () => { // Write it, then break it: a tier row raised past every reading must // move the answer, or the parity above proved nothing about the tier. const day = new Date(today - 3 * DAY); await prisma.measurementRollup.updateMany({ where: { userId: USER, type: "WEIGHT", granularity: "DAY", bucketStart: day, }, data: { readingMaxValue: 999 }, }); const out = await readAllTimeExtremes(USER, ["WEIGHT"]); expect(out.get("WEIGHT")?.max).toBe(999); }); it("reads the raw rows for a type with a tier row lacking the extremes", async () => { const day = new Date(today - 3 * DAY); await prisma.measurementRollup.updateMany({ where: { userId: USER, type: "WEIGHT", granularity: "DAY", bucketStart: day, }, data: { readingMinValue: null }, }); const [actual, reference] = await Promise.all([ readAllTimeExtremes(USER, ["WEIGHT"]), referenceExtremes(USER, ["WEIGHT"], null), ]); expect(actual.get("WEIGHT")?.max).toBe(reference.get("WEIGHT")?.max); expect(actual.get("WEIGHT")?.min).toBe(reference.get("WEIGHT")?.min); }); it("reads the raw rows for a type whose tier outlived a hard delete", async () => { // Rebuild a clean tier, then delete the hottest pulse reading with no // rollup recompute: its DAY row now counts a reading that is gone. await recomputeUserRollups(USER, { granularities: ["DAY"] }); const hottest = await prisma.measurement.findFirst({ where: { userId: USER, type: "PULSE", deletedAt: null }, orderBy: { value: "desc" }, }); await prisma.measurement.delete({ where: { id: hottest!.id } }); const stale = await prisma.measurementRollup.findMany({ where: { userId: USER, type: "PULSE", granularity: "DAY", readingMaxValue: hottest!.value, }, }); expect(stale.length).toBeGreaterThan(0); await expectParity(null); const out = await readAllTimeExtremes(USER, ["PULSE"]); expect(out.get("PULSE")?.max).toBeLessThan(hottest!.value); }); });