Merge branch 'fix/v1437-review' into release/v1.43.7

# Conflicts:
#	src/app/api/analytics/route.ts
This commit is contained in:
Marc-André Bombeck committed 2026-10-20 06:20:42 +02:00
commit 45c027aee2
217 files changed
+3636 -2127

No files matched your search

@@ -250,8 +250,9 @@ describe("readCoachCorrelations", () => {
// Next-day sleep falls as today's burden rises (8h baseline − burden).
measurements.push({
type: "SLEEP_DURATION",
value: 8 - burden,
measuredAt: new Date(base + (i + 1) * 86_400_000),
// A night of hours waking at 07:00, not a midday minute-long nap.
value: (8 - burden) * 60,
measuredAt: new Date(base + (i + 1) * 86_400_000 - 5 * 3_600_000),
});
}
@@ -346,8 +347,9 @@ describe("readCoachCorrelations", () => {
// Higher pressure today → shorter sleep tomorrow (a clean negative r).
measurements.push({
type: "SLEEP_DURATION",
value: 8 - swing * 0.5,
measuredAt: new Date(base + (i + 1) * 86_400_000),
// A night of hours waking at 07:00, not a midday minute-long nap.
value: (8 - swing * 0.5) * 60,
measuredAt: new Date(base + (i + 1) * 86_400_000 - 5 * 3_600_000),
});
}
@@ -4,6 +4,8 @@ import {
reconstructSleepNights,
reconstructSleepSessions,
pickMainNightAndNaps,
isDaytimeNap,
separateNightsAndNaps,
summarizeSleepNights,
type SleepStageRow,
} from "@/lib/analytics/sleep-night";
@@ -594,6 +596,80 @@ describe("pickMainNightAndNaps", () => {
});
});
describe("the one nap rule", () => {
it("tells a nap from a night by length and time of day together", () => {
// Short and in the afternoon: a nap.
expect(isDaytimeNap(60, 14 * 60)).toBe(true);
// Short but at night: the night.
expect(isDaytimeNap(150, 2 * 60)).toBe(false);
expect(isDaytimeNap(150, 22 * 60)).toBe(false);
// A late sleeper's short night, centred at 09:00.
expect(isDaytimeNap(150, 9 * 60)).toBe(false);
// Long in the daytime (after a night shift): a main sleep.
expect(isDaytimeNap(420, 13 * 60)).toBe(false);
});
it("gives a day of naps alone no main night", () => {
// 14:00 to 15:00 Berlin, nothing overnight (the watch was charging).
const rows: SleepStageRow[] = [row("2026-06-04T13:00:00.000Z", "CORE", 60)];
const sessions = reconstructSleepSessions(rows, "Europe/Berlin");
const { main, naps } = pickMainNightAndNaps(sessions);
expect(main).toBeNull();
expect(naps).toHaveLength(1);
expect(naps[0].daytimeNap).toBe(true);
const nights = reconstructSleepNights(rows, "Europe/Berlin");
expect(nights[0].asleepMinutes).toBe(60);
expect(nights[0].mainSession).toBeNull();
expect(separateNightsAndNaps(sessions)).toEqual([
{ night: "2026-06-04", main: null, naps },
]);
});
it("keeps a short night as the main sleep over a longer afternoon nap", () => {
const rows: SleepStageRow[] = [
// 02:00 to 03:30 Berlin: a broken night, 90 minutes.
row("2026-06-04T01:30:00.000Z", "CORE", 90),
// 13:00 to 15:00 Berlin: a two-hour nap.
row("2026-06-04T13:00:00.000Z", "CORE", 120),
];
const sessions = reconstructSleepSessions(rows, "Europe/Berlin");
const { main, naps } = pickMainNightAndNaps(sessions);
expect(main?.asleepMinutes).toBe(90);
expect(naps.map((n) => n.asleepMinutes)).toEqual([120]);
// The merged night places the same session on the clock.
const [night] = reconstructSleepNights(rows, "Europe/Berlin");
expect(night.mainSession?.asleepMinutes).toBe(90);
expect(night.mainSession?.end.getTime()).toBe(main?.end.getTime());
});
it("files a session under the day its last row ends on, over every writer", () => {
// The watch stops at 23:50 UTC; the phone's in-bed span runs to 00:10.
const rows: SleepStageRow[] = [
srcRow("2026-06-03T23:50:00.000Z", "CORE", 170, "WHOOP"),
srcRow("2026-06-04T00:10:00.000Z", "IN_BED", 200, "APPLE_HEALTH"),
];
const sessions = reconstructSleepSessions(rows, "UTC");
const nights = reconstructSleepNights(rows, "UTC");
expect(sessions.map((s) => s.night)).toEqual(["2026-06-04"]);
expect(nights.map((n) => n.night)).toEqual(["2026-06-04"]);
});
});
describe("a zero-minute granular row", () => {
it("does not strip the bare aggregate that carries the night", () => {
// A writer's bare 6-hour night beside a REM row of no length: the REM
// row partitions nothing, so the night is the aggregate's six hours.
const rows: SleepStageRow[] = [
row("2026-06-04T06:00:00.000Z", "ASLEEP", 360),
row("2026-06-04T06:00:00.000Z", "REM", 0),
];
const [session] = reconstructSleepSessions(rows, "UTC");
expect(session.asleepMinutes).toBe(360);
expect(session.stages.ASLEEP).toBe(360);
expect(reconstructSleepNights(rows, "UTC")[0].asleepMinutes).toBe(360);
});
});
describe("summarizeSleepNights", () => {
it("summarises per-night totals, not per-stage rows", () => {
const rows: SleepStageRow[] = [
@@ -373,6 +373,38 @@ describe("reference-score pillars", () => {
}
});
it("holds regularity back until 21 nights carry a midpoint, not 21 nights", () => {
const nights = days(24, 480).map((point, index) => ({
night: point.day,
asleepMinutes: point.value,
// Four nap-only days count toward the level but have no place on the
// clock, which leaves 20 midpoints.
midpoint: index % 6 === 0 ? null : 180,
}));
const result = computeSleepPillar({
...live,
asOf: NOW,
ageYears: 40,
nights,
sources: ["APPLE_HEALTH"],
});
expect(result.status).toBe("ok");
if (result.status === "ok") {
expect(result.value.observed.label).not.toContain("regularity");
}
const enough = computeSleepPillar({
...live,
asOf: NOW,
ageYears: 40,
nights: nights.map((n) => ({ ...n, midpoint: 180 })),
sources: ["APPLE_HEALTH"],
});
if (enough.status === "ok") {
expect(enough.value.observed.label).toContain("regularity");
}
expect(enough.status).toBe("ok");
});
it("scores exactly 28 sleep calendar keys", () => {
const nights = days(29, 600).map((point) => ({
night: point.day,
+4 -1
View File
@@ -62,8 +62,11 @@ export function computeSleepPillar(
const midpoints = nights
.map((night) => night.midpoint)
.filter((midpoint): midpoint is number => midpoint != null);
// The floor is on the nights that have a place on the clock: a day whose
// only sleep was a nap counts toward the level but carries no midpoint, so
// 21 nights could hold far fewer midpoints than the floor asks for.
const regularity =
nights.length >= SLEEP_REGULARITY_MIN_NIGHTS
midpoints.length >= SLEEP_REGULARITY_MIN_NIGHTS
? scoreConsistency(midpoints)
: null;
// These are the existing Sleep Score's own declared sub-score weights,
+153 -46
View File
@@ -48,7 +48,7 @@
* measured on the absolute UTC instant, so it is immune to DST shifts.
*
* Each session is keyed by the LOCAL WAKE DAY — `userDayKey` of the session's
* LAST segment (its end instant) in the user's timezone. This matches the
* LAST row end over every writer, in the user's timezone. This matches the
* Apple Health convention (a sleep session is attributed to the date you wake
* up) and keeps the keying tz/DST-correct because it runs on the real instant.
*
@@ -58,9 +58,12 @@
* day under one key so the headline "time asleep" for a day is the day's
* total. `reconstructSleepSessions` keeps each session SEPARATE, and
* `pickMainNightAndNaps` applies the convention the UI needs: the MAIN night
* is the session with the most asleep minutes (normally the overnight block),
* and every other session on the same wake day is a NAP, surfaced separately
* and never folded into the main night's headline.
* is the main sleep with the most asleep minutes (normally the overnight
* block), and every other session on the same wake day is a NAP, surfaced
* separately and never folded into the main night's headline. A session is
* a nap rather than a main sleep when it is short and centred in the daytime
* (`isDaytimeNap`), so a day with an afternoon nap and no recorded night has
* no main night at all, rather than the nap standing in for one.
*
* Writer de-dup
* -------------
@@ -127,6 +130,7 @@
*/
import type { MeasurementSource, SleepStage } from "@/generated/prisma/client";
import { userDayKey } from "@/lib/tz/format";
import { wallClockInTz } from "@/lib/tz/wall-clock";
import { summarize, type DataSummary } from "@/lib/analytics/trends";
import {
getDeviceTypeLadder,
@@ -151,6 +155,17 @@ const GRANULAR_ASLEEP_STAGES: ReadonlySet<SleepStage> = new Set<SleepStage>([
"DEEP",
]);
/**
* Every stage that counts as asleep: the granular partition plus the bare
* `ASLEEP` aggregate. A stage-less row counts as asleep too, but it has no
* stage to list. The sleep list's SQL session cut splices this list, so the
* two decide "does this session hold sleep?" from one definition.
*/
export const ASLEEP_STAGES: readonly SleepStage[] = [
"ASLEEP",
...GRANULAR_ASLEEP_STAGES,
];
/**
* True when any row in the set carries a GRANULAR asleep stage
* (CORE / DEEP / REM). When this holds, the bare `ASLEEP` aggregate (and any
@@ -162,7 +177,10 @@ const GRANULAR_ASLEEP_STAGES: ReadonlySet<SleepStage> = new Set<SleepStage>([
function sawGranularStage(rows: readonly SleepStageRow[]): boolean {
for (const r of rows) {
const stage = r.sleepStage;
if (stage != null && GRANULAR_ASLEEP_STAGES.has(stage)) return true;
// A zero-minute granular row partitions nothing: it must not strip the
// bare aggregate that does carry the night.
if (stage != null && GRANULAR_ASLEEP_STAGES.has(stage) && r.value > 0)
return true;
}
return false;
}
@@ -199,7 +217,7 @@ function asleepMinutesOf(rows: readonly SleepStageRow[]): number {
const stage = seg.stage;
if (stage != null && GRANULAR_ASLEEP_STAGES.has(stage)) {
granular += seg.minutes;
sawGranular = true;
if (seg.minutes > 0) sawGranular = true;
} else if (stage === "ASLEEP" || stage == null) {
// Bare unspecified aggregate (`ASLEEP`) or a stage-less legacy/manual
// row — only used when no granular stage carries the night.
@@ -258,6 +276,70 @@ function inBedEnvelopeMinutes(rows: readonly SleepStageRow[]): number | null {
*/
export const SESSION_GAP_MS = 3 * 60 * 60 * 1000;
/**
* Below this much asleep time a session can be a nap. Three hours is where
* wearables draw the line between a nap and a main sleep (Oura types a
* period `long_sleep` from three hours on).
*/
export const MAIN_SLEEP_MIN_MINUTES = 180;
/**
* The night window, in local minutes-of-day: a session whose midpoint falls
* from 20:00 to 10:00 is night sleep however short it is. Wide enough for a
* late chronotype (asleep 04:00 to 12:00 has its midpoint at 08:00) and for
* an early one, and clear of the afternoon where naps sit.
*/
export const NIGHT_WINDOW_START_MINUTES = 20 * 60;
export const NIGHT_WINDOW_END_MINUTES = 10 * 60;
/**
* True for a session that is a nap rather than a main sleep: short (under
* `MAIN_SLEEP_MIN_MINUTES` asleep) AND centred in the daytime. A short night
* is still the night (insomnia, a newborn, split sleep); a long daytime sleep
* (after a night shift) is still a main sleep. The one nap rule: the nap
* convention, the sleep score's midpoint and every list read it. Pure.
*/
export function isDaytimeNap(
asleepMinutes: number,
midpointMinutes: number,
): boolean {
if (asleepMinutes >= MAIN_SLEEP_MIN_MINUTES) return false;
const atNight =
midpointMinutes >= NIGHT_WINDOW_START_MINUTES ||
midpointMinutes < NIGHT_WINDOW_END_MINUTES;
return !atNight;
}
/**
* The centre of a session's asleep span as local minutes-of-day: its wake
* instant minus half its asleep minutes, read on the sleeper's wall clock.
*/
export function asleepMidpointMinutes(
end: Date,
asleepMinutes: number,
tz: string,
): number {
const mid = new Date(end.getTime() - (asleepMinutes / 2) * 60_000);
if (tz === "UTC") return mid.getUTCHours() * 60 + mid.getUTCMinutes();
const { hour, minute } = wallClockInTz(mid, tz);
return hour * 60 + minute;
}
/**
* The wake day a session is filed under: the local day its last row ends on,
* over every writer in the session. One rule for every reader, the SQL page
* cut of the sleep list included, which sees every writer and cannot know the
* winning one. Filing by the winner alone named a different day whenever a
* losing writer's in-bed span ran past midnight after the winner stopped, and
* the list and the night view then disagreed about which day the night was.
*/
function wakeDayOf(rows: readonly SleepStageRow[], tz: string): string {
let end = rows[0].measuredAt;
for (const r of rows)
if (r.measuredAt.getTime() > end.getTime()) end = r.measuredAt;
return userDayKey(end, tz);
}
export interface SleepStageRow {
value: number;
measuredAt: Date;
@@ -271,12 +353,12 @@ export interface SleepStageRow {
*/
source?: MeasurementSource | null;
/**
* Device-type tag of the stage row (`Measurement.deviceType`, v1.4.25
* W8c). Distinguishes WRITERS behind one source: several HealthKit apps
* Device-type tag of the stage row (`Measurement.deviceType`, v1.4.25).
* Distinguishes WRITERS behind one source: several HealthKit apps
* write sleep under `source = APPLE_HEALTH` (watch granular stages, phone
* coarse in-bed, vendor-app re-exports), and collapsing on source alone
* blends them. Optional — callers that don't select the column (and every
* row stored before W8c) collapse per source exactly as before.
* row stored before v1.4.25) collapse per source exactly as before.
*/
deviceType?: string | null;
}
@@ -309,9 +391,9 @@ export interface SleepNight {
stages: Partial<Record<SleepStage, number>>;
/**
* Non-null when two writer buckets reported clearly different asleep
* totals for the night's MAIN session — the same-wake-day session with
* totals for the night's MAIN session — the same-wake-day main sleep with
* the most asleep minutes, normally the overnight block (see
* `sessionSourceDiscrepancy`). Sessions never cross-compare: a nap's
* `pickMainNightAndNaps` and `sessionSourceDiscrepancy`). Sessions never cross-compare: a nap's
* writers are not measured against the overnight's. Observational only —
* the served `asleepMinutes` stays the winning writer's total; the
* annotation lets the UI mark the number with a discreet
@@ -319,12 +401,13 @@ export interface SleepNight {
*/
sourceDiscrepancy: SleepSourceDiscrepancy | null;
/**
* The night's MAIN session alone: the same-wake-day session with the most
* asleep minutes (the one `sourceDiscrepancy` reads), its wake instant and
* its own asleep minutes. A nap on the same wake day is in the night's
* totals but not here, so a reader that places the night on the clock (the
* sleep midpoint) is not dragged into the afternoon by it. Null when no
* session carries asleep minutes.
* The night's MAIN session alone, chosen by the nap convention
* (`pickMainNightAndNaps`: the main sleep with the most asleep minutes, the
* one `sourceDiscrepancy` reads), its wake instant and its own asleep
* minutes. A nap on the same wake day is in the night's totals but not
* here, so a reader that places the night on the clock (the sleep midpoint)
* is not dragged into the afternoon by it. Null when no session carries
* asleep minutes, and on a day whose only sleep is daytime naps.
*/
mainSession: { end: Date; asleepMinutes: number } | null;
}
@@ -643,13 +726,9 @@ export function reconstructSleepNights(
const canonical = pickSessionWriter(session, sleepLadder, deviceLadder);
const kept = session.filter((r) => writerKeyOf(r) === canonical);
const pool = kept.length > 0 ? kept : session;
// The wake day is the LATEST segment END in the session (Apple Health
// attributes a session to the morning you wake up).
const wakeInstant = pool.reduce(
(max, r) => (r.measuredAt.getTime() > max.getTime() ? r.measuredAt : max),
pool[0].measuredAt,
);
const key = userDayKey(wakeInstant, tz);
// The wake day is the session's latest row end (Apple Health attributes
// a session to the morning you wake up), over every writer.
const key = wakeDayOf(session, tz);
const list = byNight.get(key) ?? [];
list.push({ pool, all: session });
byNight.set(key, list);
@@ -671,9 +750,9 @@ export function reconstructSleepNights(
// edits.
let latest = nightSessions[0].pool[0].measuredAt;
const stages: Partial<Record<SleepStage, number>> = {};
// MAIN session of the night = most asleep minutes (same per-session
// asleep total the headline sums; tie → later wake instant, matching
// `pickMainNightAndNaps`). Its FULL raw row set (every writer,
// MAIN session of the night = the main sleep (not a daytime nap) with
// the most asleep minutes (same per-session asleep total the headline
// sums; tie → later wake instant), exactly as `pickMainNightAndNaps`. Its FULL raw row set (every writer,
// pre-collapse) feeds the night's source-discrepancy annotation —
// per session, so a nap's writers never cross-compare against the
// overnight block's.
@@ -731,6 +810,10 @@ export function reconstructSleepNights(
}
if (
sessionAsleep > 0 &&
!isDaytimeNap(
sessionAsleep,
asleepMidpointMinutes(new Date(sessionEnd), sessionAsleep, tz),
) &&
(sessionAsleep > mainSessionAsleep ||
(sessionAsleep === mainSessionAsleep && sessionEnd > mainSessionEnd))
) {
@@ -844,6 +927,12 @@ export interface SleepSession {
* "sources disagree" hint.
*/
sourceDiscrepancy: SleepSourceDiscrepancy | null;
/**
* `true` when the session is a daytime nap rather than a main sleep
* (`isDaytimeNap`). Decided here, once, so the nap convention, the sleep
* score and every list read the same answer.
*/
daytimeNap: boolean;
}
/** Resolve a stage row to its absolute span (start = end − duration). */
@@ -1079,7 +1168,7 @@ export function reconstructSleepSessions(
const asleep = asleepMinutesOf(pool);
const canonicalSource = sourceOfWriterKey(canonical);
sessions.push({
night: userDayKey(latest, tz),
night: wakeDayOf(session, tz),
source:
canonicalSource === NO_SOURCE
? null
@@ -1101,21 +1190,28 @@ export function reconstructSleepSessions(
// losing bucket's disagreement is visible; the served totals above
// stay winner-only.
sourceDiscrepancy: sessionSourceDiscrepancy(session),
daytimeNap: isDaytimeNap(
asleep,
asleepMidpointMinutes(latest, asleep, tz),
),
});
}
return sessions.sort((a, b) => a.start.getTime() - b.start.getTime());
}
/**
* Apply the NAP convention to a wake-day's sessions: the MAIN night is the
* session with the most asleep minutes (normally the overnight block); every
* other session on the same wake day is a nap, surfaced separately and never
* folded into the main night's headline.
* Apply the NAP convention to a wake-day's sessions. The MAIN night is the
* main-sleep session (see `isDaytimeNap`) with the most asleep minutes,
* normally the overnight block; every other scorable session on the same wake
* day is a nap, surfaced separately and never folded into the main night's
* headline. A day whose only sleep is daytime naps has no main night: `main`
* is null and the naps are all there is.
*
* Returns the main session plus the naps (sorted by start). When the input
* holds sessions across multiple wake days, only the sessions matching the
* main session's wake day are considered naps — call once per wake day, or
* pass a single wake day's sessions.
* main session's wake day (or, without one, the latest session's) are
* considered naps — call once per wake day, or pass a single wake day's
* sessions.
*/
export function pickMainNightAndNaps(sessions: readonly SleepSession[]): {
main: SleepSession | null;
@@ -1123,14 +1219,22 @@ export function pickMainNightAndNaps(sessions: readonly SleepSession[]): {
} {
const scorable = sessions.filter((s) => s.asleepMinutes > 0);
if (scorable.length === 0) return { main: null, naps: [] };
// Main = most asleep minutes; tie-break on the later end (overnight wins).
const main = [...scorable].sort((a, b) => {
if (b.asleepMinutes !== a.asleepMinutes)
return b.asleepMinutes - a.asleepMinutes;
return b.end.getTime() - a.end.getTime();
})[0];
// Main = most asleep minutes among the main sleeps; tie-break on the later
// end (overnight wins).
const main =
scorable
.filter((s) => !s.daytimeNap)
.sort((a, b) => {
if (b.asleepMinutes !== a.asleepMinutes)
return b.asleepMinutes - a.asleepMinutes;
return b.end.getTime() - a.end.getTime();
})[0] ?? null;
const day =
main?.night ??
scorable.reduce((a, b) => (b.end.getTime() > a.end.getTime() ? b : a))
.night;
const naps = scorable
.filter((s) => s !== main && s.night === main.night)
.filter((s) => s !== main && s.night === day)
.sort((a, b) => a.start.getTime() - b.start.getTime());
return { main, naps };
}
@@ -1139,8 +1243,11 @@ export function pickMainNightAndNaps(sessions: readonly SleepSession[]): {
export interface SleepDaySplit {
/** Wake-day key (YYYY-MM-DD) in the user's timezone. */
night: string;
/** The overnight block — the session with the most asleep minutes. */
main: SleepSession;
/**
* The main sleep (see `pickMainNightAndNaps`), or null on a day whose only
* sleep is daytime naps.
*/
main: SleepSession | null;
/** Every other scorable session on the same wake day, sorted by start. */
naps: SleepSession[];
}
@@ -1154,8 +1261,8 @@ export interface SleepDaySplit {
* than growing its own idea of a night. There is exactly one nap classifier in
* this codebase and it is `pickMainNightAndNaps`; this is its multi-day form.
*
* Days whose sessions carry no asleep minutes are dropped (no main night to
* report). The result is sorted ascending by wake day.
* Days whose sessions carry no asleep minutes are dropped; a day of naps
* alone is kept with `main` null. The result is sorted ascending by wake day.
*/
export function separateNightsAndNaps(
sessions: readonly SleepSession[],
@@ -1169,7 +1276,7 @@ export function separateNightsAndNaps(
const days: SleepDaySplit[] = [];
for (const [night, daySessions] of byDay) {
const { main, naps } = pickMainNightAndNaps(daySessions);
if (!main) continue;
if (!main && naps.length === 0) continue;
days.push({ night, main, naps });
}
return days.sort((a, b) => a.night.localeCompare(b.night));
@@ -73,6 +73,8 @@ export function buildSleepStageComposition(
const perNight: SleepStageCompositionNight[] = [];
for (const { night, main, naps } of days) {
// A day of naps alone has no night to compose.
if (!main) continue;
const nightStages: Record<string, number> = {};
for (const [stage, mins] of Object.entries(main.stages)) {
if (mins == null) continue;
+33
View File
@@ -0,0 +1,33 @@
import { ApiError } from "./api-fetch";
/**
* What a failed read means for the surface that made it.
*
* - `unauthenticated`: 401. The session has ended. The surface says so and
* offers the way back to sign-in; drawing nothing would leave a page that
* looks empty for a reason the reader cannot see.
* - `refused`: 403. The viewer may not read this, typically a shared or
* delegated record whose grant leaves the section out. Nothing failed that
* a retry could fix, so the surface draws its refused state, or nothing.
* - `throttled`: 429. The read budget is spent for now; an immediate retry
* would only be refused again.
* - `failed`: everything else, a real read failure worth a retry.
*/
export type ReadFailureKind =
"unauthenticated" | "refused" | "throttled" | "failed";
export function readFailureKind(error: unknown): ReadFailureKind {
if (error instanceof ApiError) {
if (error.status === 401) return "unauthenticated";
if (error.status === 403) return "refused";
if (error.status === 429) return "throttled";
}
return "failed";
}
/** The sign-in page, returning to where the reader is now. */
export function signInHref(): string {
if (typeof window === "undefined") return "/auth/login";
const here = `${window.location.pathname}${window.location.search}`;
return `/auth/login?next=${encodeURIComponent(here)}`;
}
@@ -53,7 +53,10 @@ describe("stripImageMetadata — passthrough", () => {
describe("stripImageMetadata — JPEG", () => {
// SOI + APP0(JFIF) + APP1(Exif w/ GPS) + APP2(ICC) + COM + DQT + SOS + scan.
const soi = Buffer.from([0xff, 0xd8]);
const app0 = jpegSegment(0xe0, Buffer.from("JFIF\0\0\0\0\0\0"));
const app0 = jpegSegment(
0xe0,
Buffer.from("JFIF\0\x01\x01\0\0\x01\0\x01\0\0"),
);
const exifPayload = Buffer.concat([
Buffer.from("Exif\0\0"),
// A recognisable GPS marker string we can assert is gone.
+2 -2
View File
@@ -31,8 +31,8 @@ import {
MAXMIND_DOWNLOAD_BASE,
} from "@/lib/geo/geolite2-fetch";
const CITY_BYTES = Buffer.from("CITY-MMDB-CONTENT-");
const ASN_BYTES = Buffer.from("ASN-MMDB-CONTENT-");
const CITY_BYTES = Buffer.from("CITY-MMDB-CONTENT-\x01\x02\x03");
const ASN_BYTES = Buffer.from("ASN-MMDB-CONTENT-\x04\x05\x06");
/** Build a MaxMind-shaped tar.gz: `<Edition>_<date>/<Edition>.mmdb`. */
function buildEditionTarball(
@@ -9,9 +9,10 @@
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const { prismaMock, safeFetchMock } = vi.hoisted(() => ({
const { prismaMock, safeFetchMock, annotateMock } = vi.hoisted(() => ({
prismaMock: { googleHealthConnection: { update: vi.fn() } },
safeFetchMock: vi.fn(),
annotateMock: vi.fn(),
}));
vi.mock("@/lib/db", () => ({ prisma: prismaMock }));
vi.mock("@/lib/safe-fetch", async (importOriginal) => ({
@@ -19,7 +20,7 @@ vi.mock("@/lib/safe-fetch", async (importOriginal) => ({
safeFetch: safeFetchMock,
}));
vi.mock("@/lib/logging/context", () => ({
annotate: () => {},
annotate: annotateMock,
getEvent: () => null,
}));
@@ -32,14 +33,16 @@ import {
type GoogleHealthBackfillEnd,
} from "../backfill-bounds";
import {
googleHealthBackfillEndedList,
googleHealthBackfillProgressAfterCompletion,
googleHealthBackfillRetryPending,
readGoogleHealthBackfillEnded,
recordGoogleHealthBackfillEnd,
runGoogleHealthCollection,
runWithGoogleHealthBackfillResume,
} from "../sync-core";
import {
googleHealthBackfillEndedList,
googleHealthBackfillProgressAfterCompletion,
googleHealthBackfillRetryPending,
readGoogleHealthBackfillProgress,
} from "../backfill-progress";
import {
forEachDataPointPage,
GOOGLE_HEALTH_DATA_TYPES,
@@ -312,13 +315,31 @@ describe("a collection ended short", () => {
-1,
)![0].data.backfillProgress as unknown;
expect(
readGoogleHealthBackfillEnded(written).get("fetchHeartRate"),
readGoogleHealthBackfillProgress(written).ended.get("fetchHeartRate"),
).toEqual(note);
// Waiting for another try is not where the history ends.
expect(googleHealthBackfillEndedList(written)).toEqual([]);
expect(googleHealthBackfillProgressAfterCompletion(ended)).toBeNull();
});
it("annotates an end in one fixed shape", async () => {
await slice(stalls, new Map([["fetchWeight", HORIZON_END]]));
const notes = annotateMock.mock.calls
.map(([fields]) => fields.meta?.["googleHealth.backfill.ended"])
.filter(Boolean);
expect(notes).toEqual([
{
collection: "fetchHeartRate",
type: "PULSE",
reason: "no_progress",
final: false,
endedCount: 2,
finalCount: 1,
types: ["PULSE", "WEIGHT"],
},
]);
});
it("is not walked again before its day is up", async () => {
const walk = vi.fn(async () => 1);
const { skipped } = await slice(
@@ -351,6 +372,31 @@ describe("a collection ended short", () => {
).toMatchObject({ ...END, attempts: 2 });
});
it("starts the count over when the retry got further back before it stalled again", async () => {
const further = { ...END, at: "2023-01-02T00:00:00.000Z" };
const done = new Set<string>();
const { ended, collections } = await slice(
async () => {
noteGoogleHealthDeferred();
await recordGoogleHealthBackfillEnd("fetchHeartRate", further);
return 4;
},
new Map<string, GoogleHealthBackfillEnd>([
["fetchHeartRate", parked(1, 25)],
]),
done,
);
// Progress, then a new stall: parked again, not final.
expect(ended.get("fetchHeartRate")).toMatchObject({
...further,
attempts: 1,
});
expect(done.has("fetchHeartRate")).toBe(false);
expect(collections).toEqual([
expect.objectContaining({ key: "fetchHeartRate", status: "stopped" }),
]);
});
it("goes back to an ordinary walk when the second attempt gets somewhere", async () => {
const { ended, collections } = await slice(
async () => {
@@ -20,12 +20,14 @@ vi.mock("@/lib/logging/context", () => ({
}));
import {
readGoogleHealthBackfillCursors,
recordGoogleHealthBackfillPage,
runGoogleHealthCollection,
runWithGoogleHealthBackfillResume,
type GoogleHealthBackfillCursor,
} from "../sync-core";
import {
readGoogleHealthBackfillProgress,
type GoogleHealthBackfillCursor,
} from "../backfill-progress";
import {
googleHealthSliceBudget,
runWithGoogleHealthClientOutcome,
@@ -124,7 +126,7 @@ describe("the resume edge of a page-by-page collection", () => {
});
// And it reads back the way it was written.
expect(
readGoogleHealthBackfillCursors(last.data.backfillProgress).get(
readGoogleHealthBackfillProgress(last.data.backfillProgress).cursors.get(
"fetchHeartRate",
),
).toEqual(cursors.get("fetchHeartRate"));
@@ -46,13 +46,14 @@ import { GOOGLE_HEALTH_NO_PROGRESS_PAGES } from "../backfill-bounds";
import {
googleHealthBackfillCursor,
noteGoogleHealthEdgeFilterUnsupported,
readGoogleHealthBackfillCursors,
readGoogleHealthBackfillEdgeFilterUnsupported,
recordGoogleHealthBackfillPage,
recordGoogleHealthBackfillResumePoint,
runWithGoogleHealthBackfillResume,
type GoogleHealthBackfillCursor,
} from "../sync-core";
import {
readGoogleHealthBackfillProgress,
type GoogleHealthBackfillCursor,
} from "../backfill-progress";
const MINUTE = 60_000;
const NOW = Date.parse("2026-01-01T00:00:00Z");
@@ -566,7 +567,8 @@ describe("the note that a collection refuses the edge filter", () => {
// Slice two: starts from what slice one left on disk.
server.requests = [];
const unsupported = readGoogleHealthBackfillEdgeFilterUnsupported(progress);
const unsupported =
readGoogleHealthBackfillProgress(progress).edgeFilterUnsupported;
await runWithGoogleHealthBackfillResume(
"u1",
new Set(),
@@ -576,7 +578,7 @@ describe("the note that a collection refuses the edge filter", () => {
const { rows } = await walk({ resumable: true, edge: cursor.edge });
expect(rows).toHaveLength(TOTAL - 19);
},
readGoogleHealthBackfillCursors(progress),
readGoogleHealthBackfillProgress(progress).cursors,
unsupported,
);
expect(server.requests.some((f) => f.includes("<="))).toBe(false);
@@ -616,7 +618,7 @@ describe("the note that a collection refuses the edge filter", () => {
page: 4,
});
expect(first.advanced.has("fetchHeartRate")).toBe(true);
const saved = readGoogleHealthBackfillCursors(lastProgress());
const saved = readGoogleHealthBackfillProgress(lastProgress()).cursors;
expect(saved.get("fetchHeartRate")).toEqual({
...cursor,
resume: { token: "4", page: 4, reached: 4 },
@@ -628,8 +630,9 @@ describe("the note that a collection refuses the edge filter", () => {
const back = await run(new Map(saved), { token: "2", page: 2 });
expect(back.advanced.size).toBe(0);
expect(
readGoogleHealthBackfillCursors(lastProgress()).get("fetchHeartRate")
?.resume,
readGoogleHealthBackfillProgress(lastProgress()).cursors.get(
"fetchHeartRate",
)?.resume,
).toEqual({ token: "2", page: 2, reached: 4 });
// A page written keeps the place; the next slice is handed it.
@@ -644,7 +647,7 @@ describe("the note that a collection refuses the edge filter", () => {
new Map(saved),
new Set(["fetchHeartRate"]),
);
const next = readGoogleHealthBackfillCursors(lastProgress());
const next = readGoogleHealthBackfillProgress(lastProgress()).cursors;
expect(next.get("fetchHeartRate")?.resume).toEqual(
saved.get("fetchHeartRate")?.resume,
);
@@ -663,10 +666,12 @@ describe("the note that a collection refuses the edge filter", () => {
it("reads only well-formed keys", () => {
expect([
...readGoogleHealthBackfillEdgeFilterUnsupported({
...readGoogleHealthBackfillProgress({
edgeFilterUnsupported: ["fetchHeartRate", 3, "x".repeat(65)],
}),
}).edgeFilterUnsupported,
]).toEqual(["fetchHeartRate"]);
expect(readGoogleHealthBackfillEdgeFilterUnsupported(null).size).toBe(0);
expect(
readGoogleHealthBackfillProgress(null).edgeFilterUnsupported.size,
).toBe(0);
});
});
+279
View File
@@ -0,0 +1,279 @@
/**
* The `backfillProgress` column of a Google Health connection: how far a
* full-history backfill has got, carried from one attempt to the next, and
* where a data type's history ended short of its beginning.
*
* One schema reads the column and one function writes it. The column is
* JSON the code itself wrote, but it outlives releases, so the read is
* lenient per entry: a malformed cursor or end is dropped on its own and the
* rest of the note still counts. A collection key longer than 64 characters
* is never one of ours and is dropped as well.
*/
import { z } from "zod";
import type { MeasurementType } from "@/generated/prisma/client";
import { MeasurementType as MeasurementTypes } from "@/generated/prisma/enums";
import {
googleHealthBackfillEndIsFinal,
googleHealthBackfillEndRetryAt,
type GoogleHealthBackfillEnd,
} from "./backfill-bounds";
/**
* How far a full-history backfill has got with one collection.
*
* Two grains. A collection walked to the end is noted in `done`, and the next
* attempt skips it. A collection read page by page (the spot readings: heart
* rate, weight, glucose, ...) also notes, after every page it writes, the
* edge it has reached: the oldest sample written so far when the walk runs
* newest first, as Google's list does. The next attempt asks Google only for
* the samples at or beyond that edge, so a collection that does not fit into
* one attempt is worked through over several instead of being read from its
* first page every time.
*
* The edge is a sample time, not Google's page token: page tokens are not
* documented to outlive the walk that received them, a sample time is a
* filter Google documents. The one sample on the edge itself is read twice
* and overwrites its own row.
*
* Before the edge existed only `done` was kept, and a heart-rate history of
* years that one attempt could not finish (its budget, a rate limit, one
* failed page among thousands) started again from the newest page on every
* attempt and never reached the end (#1220).
*
* The upper bound (`<=`) a newest-first walk resumes with is valid filter
* grammar but has not been seen working against the live API. A collection
* whose resumed request Google refuses with a 400 is noted in
* `edgeFilterUnsupported`, and its walks from then on read without the bound
* and pass over the samples beyond the edge themselves (see
* `forEachDataPointPage`).
*/
export interface GoogleHealthBackfillCursor {
/** The edge: the sample instant (ISO 8601) the walk has written up to. */
at: string;
/** The order the walk returned samples in: newest first, or oldest first. */
order: "desc" | "asc";
/**
* Pages this collection has written across every attempt so far. Kept for
* the record only: it bounds nothing. It once capped the walk at ten
* thousand pages, which a dense heart-rate history passes in under three
* years, and since it never reset the backfill stopped there for good
* (#1220). The walk is bounded by its progress and the backfill horizon
* instead (`backfill-bounds.ts`).
*/
pages: number;
/**
* Only for a collection read without the edge bound: where its last walk
* stopped in the unbounded list (see `GoogleHealthResumePoint`), and the
* furthest position any walk of it has saved. A saved place counts as the
* slice's progress only when it is further than every one before it, so a
* chain of slices that keep passing over the same pages ends.
*/
resume?: { token: string; page: number; reached: number };
}
/** Everything the column holds, read. */
export interface GoogleHealthBackfillProgress {
/** Collections walked to the end. */
done: Set<string>;
/** Per-collection edges. */
cursors: Map<string, GoogleHealthBackfillCursor>;
/** Collections Google refused the edge filter on. */
edgeFilterUnsupported: Set<string>;
/**
* Collections whose backfill ended short of the start of their history
* (the horizon, or a walk that stopped getting anywhere), each with where
* it ended and why. Survives the completion stamp, so the integration card
* can say where history ends.
*/
ended: Map<string, GoogleHealthBackfillEnd>;
}
const MAX_KEY_LENGTH = 64;
/** An instant Date can read, normalised to ISO 8601. */
const instant = z
.string()
.refine((s) => !Number.isNaN(Date.parse(s)))
.transform((s) => new Date(s).toISOString());
/** A whole, positive count; anything else reads as zero. */
const count = z
.unknown()
.transform((v) =>
typeof v === "number" && Number.isFinite(v) && v > 0 ? Math.trunc(v) : 0,
);
const resumeSchema = z
.object({ token: z.string().min(1).max(4096), page: count, reached: count })
.transform((r) =>
r.page > 0
? { token: r.token, page: r.page, reached: Math.max(r.reached, r.page) }
: undefined,
);
const cursorSchema = z
.object({
at: instant,
order: z.unknown().transform((v) => (v === "asc" ? "asc" : "desc")),
pages: count,
resume: resumeSchema.optional().catch(undefined),
})
.transform(({ resume, ...c }): GoogleHealthBackfillCursor =>
resume ? { ...c, resume } : c,
);
const endSchema = z
.object({
type: z.enum(MeasurementTypes),
at: instant,
reason: z.enum(["horizon", "no_progress"]),
attempts: z.number().int().min(1).optional().catch(undefined),
endedAt: instant.optional().catch(undefined),
})
.transform(({ attempts, endedAt, ...e }): GoogleHealthBackfillEnd => ({
...e,
type: e.type as MeasurementType,
...(attempts !== undefined ? { attempts } : {}),
...(endedAt !== undefined ? { endedAt } : {}),
}));
const record = z.record(z.string(), z.unknown()).catch({});
const keyList = z
.array(z.unknown())
.catch([])
.transform(
(keys) =>
new Set(
keys.filter(
(key): key is string =>
typeof key === "string" && key.length <= MAX_KEY_LENGTH,
),
),
);
/** Entries of a keyed record that parse, by their own schema, one by one. */
function keyedEntries<T>(value: unknown, schema: z.ZodType<T>): Map<string, T> {
const out = new Map<string, T>();
for (const [key, raw] of Object.entries(record.parse(value))) {
if (key.length > MAX_KEY_LENGTH) continue;
const parsed = schema.safeParse(raw);
if (parsed.success) out.set(key, parsed.data);
}
return out;
}
/** Read the `backfillProgress` column. */
export function readGoogleHealthBackfillProgress(
value: unknown,
): GoogleHealthBackfillProgress {
const column = record.parse(value);
return {
done: keyList.parse(column.done),
cursors: keyedEntries(column.cursors, cursorSchema),
edgeFilterUnsupported: keyList.parse(column.edgeFilterUnsupported),
ended: keyedEntries(column.ended, endSchema),
};
}
/** The ended collections as the column stores them. */
function endedJson(ended: Map<string, GoogleHealthBackfillEnd>) {
return Object.fromEntries(
[...ended].map(([key, e]) => [
key,
{
type: e.type,
at: e.at,
reason: e.reason,
...(e.attempts !== undefined ? { attempts: e.attempts } : {}),
...(e.endedAt !== undefined ? { endedAt: e.endedAt } : {}),
},
]),
);
}
/**
* The column's value for a backfill in progress. The two optional parts are
* written only once there is something to say, so a backfill that never met
* them keeps the shape it always had.
*/
export function googleHealthBackfillProgressJson(
progress: GoogleHealthBackfillProgress,
) {
const { done, cursors, edgeFilterUnsupported, ended } = progress;
return {
done: [...done],
cursors: Object.fromEntries(
[...cursors].map(([key, c]) => [
key,
{
at: c.at,
order: c.order,
pages: c.pages,
...(c.resume ? { resume: c.resume } : {}),
},
]),
),
...(edgeFilterUnsupported.size > 0
? { edgeFilterUnsupported: [...edgeFilterUnsupported] }
: {}),
...(ended.size > 0 ? { ended: endedJson(ended) } : {}),
};
}
/**
* The column's value to keep once the backfill is stamped complete: only
* where collections ended short for good, or nothing at all.
*/
export function googleHealthBackfillProgressAfterCompletion(
ended: Map<string, GoogleHealthBackfillEnd>,
): { ended: ReturnType<typeof endedJson> } | null {
const final = new Map(
[...ended].filter(([, e]) => googleHealthBackfillEndIsFinal(e)),
);
return final.size > 0 ? { ended: endedJson(final) } : null;
}
/**
* Where each data type's imported history ends short of its beginning, and
* why, as the status routes publish it (`backfillEnded`), by type.
*/
export interface GoogleHealthBackfillEndedEntry {
type: MeasurementType;
/** The oldest instant the imported history reaches back to. */
endsAt: string;
reason: GoogleHealthBackfillEnd["reason"];
}
/**
* The column's ended collections, as published: only the final ends. One
* still waiting for another attempt is not where its history ends yet.
*/
export function googleHealthBackfillEndedList(
value: unknown,
): GoogleHealthBackfillEndedEntry[] {
return [...readGoogleHealthBackfillProgress(value).ended.values()]
.filter(googleHealthBackfillEndIsFinal)
.map((e) => ({ type: e.type, endsAt: e.at, reason: e.reason }))
.sort((a, b) => a.type.localeCompare(b.type));
}
/**
* The collections waiting for another attempt after one ended without
* progress, and the earliest instant (epoch ms) one of them is due, or null
* when none waits.
*/
export function googleHealthBackfillRetryPending(
ended: Map<string, GoogleHealthBackfillEnd>,
): { keys: string[]; dueAt: number } | null {
const keys: string[] = [];
let dueAt = Infinity;
for (const [key, e] of ended) {
const at = googleHealthBackfillEndRetryAt(e);
if (at === null) continue;
keys.push(key);
dueAt = Math.min(dueAt, at);
}
return keys.length > 0 ? { keys, dueAt } : null;
}
+43 -280
View File
@@ -18,7 +18,6 @@
import { AsyncLocalStorage } from "node:async_hooks";
import { prisma } from "@/lib/db";
import type { MeasurementType } from "@/generated/prisma/client";
import { MeasurementType as MeasurementTypes } from "@/generated/prisma/enums";
import { encrypt, decrypt } from "@/lib/crypto";
import { annotate, getEvent } from "@/lib/logging/context";
import type { GoogleHealthCollectionKey } from "./collection-keys";
@@ -27,6 +26,10 @@ import {
googleHealthBackfillEndRetryAt,
type GoogleHealthBackfillEnd,
} from "./backfill-bounds";
import {
googleHealthBackfillProgressJson,
type GoogleHealthBackfillCursor,
} from "./backfill-progress";
import { userDayKey } from "@/lib/tz/format";
import { isP2002 } from "@/lib/prisma-errors";
import {
@@ -411,60 +414,6 @@ export async function runWithGoogleHealthSyncCycle<T>(
};
}
/**
* How far a full-history backfill has got, carried from one attempt of the
* backfill to the next.
*
* Two grains. A collection walked to the end is noted in `done`, and the next
* attempt skips it. A collection read page by page (the spot readings: heart
* rate, weight, glucose, ...) also notes, after every page it writes, the
* edge it has reached: the oldest sample written so far when the walk runs
* newest first, as Google's list does. The next attempt asks Google only for
* the samples at or beyond that edge, so a collection that does not fit into
* one attempt is worked through over several instead of being read from its
* first page every time.
*
* The edge is a sample time, not Google's page token: page tokens are not
* documented to outlive the walk that received them, a sample time is a
* filter Google documents. The one sample on the edge itself is read twice
* and overwrites its own row.
*
* Before the edge existed only `done` was kept, and a heart-rate history of
* years that one attempt could not finish (its budget, a rate limit, one
* failed page among thousands) started again from the newest page on every
* attempt and never reached the end (#1220).
*
* The upper bound (`<=`) a newest-first walk resumes with is valid filter
* grammar but has not been seen working against the live API. A collection
* whose resumed request Google refuses with a 400 is noted in
* `edgeFilterUnsupported`, and its walks from then on read without the bound
* and pass over the samples beyond the edge themselves (see
* `forEachDataPointPage`).
*/
export interface GoogleHealthBackfillCursor {
/** The edge: the sample instant (ISO 8601) the walk has written up to. */
at: string;
/** The order the walk returned samples in: newest first, or oldest first. */
order: "desc" | "asc";
/**
* Pages this collection has written across every attempt so far. Kept for
* the record only: it bounds nothing. It once capped the walk at ten
* thousand pages, which a dense heart-rate history passes in under three
* years, and since it never reset the backfill stopped there for good
* (#1220). The walk is bounded by its progress and the backfill horizon
* instead (`backfill-bounds.ts`).
*/
pages: number;
/**
* Only for a collection read without the edge bound: where its last walk
* stopped in the unbounded list (see `GoogleHealthResumePoint`), and the
* furthest position any walk of it has saved. A saved place counts as the
* slice's progress only when it is further than every one before it, so a
* chain of slices that keep passing over the same pages ends.
*/
resume?: { token: string; page: number; reached: number };
}
/** What one collection came to in one backfill attempt. */
export interface GoogleHealthBackfillCollectionReport {
key: string;
@@ -497,209 +446,13 @@ interface BackfillResumeTracker {
ended: Map<string, GoogleHealthBackfillEnd>;
/**
* Collections walked again this attempt after an earlier one ended without
* progress, with how many attempts had ended that way.
* progress, with how many attempts in a row had ended that way and the edge
* they ended at.
*/
retrying: Map<string, number>;
retrying: Map<string, { attempts: number; at: string }>;
}
const backfillResumeStorage = new AsyncLocalStorage<BackfillResumeTracker>();
function progressRecord(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
/** Read the `backfillProgress` column into the set of finished collections. */
export function readGoogleHealthBackfillDone(value: unknown): Set<string> {
const done = progressRecord(value).done;
return new Set(
Array.isArray(done)
? done.filter(
(key): key is string => typeof key === "string" && key.length <= 64,
)
: [],
);
}
/** Read the `backfillProgress` column into the per-collection edges. */
export function readGoogleHealthBackfillCursors(
value: unknown,
): Map<string, GoogleHealthBackfillCursor> {
const out = new Map<string, GoogleHealthBackfillCursor>();
const cursors = progressRecord(progressRecord(value).cursors);
for (const [key, raw] of Object.entries(cursors)) {
if (key.length > 64) continue;
const c = progressRecord(raw);
const at = typeof c.at === "string" ? c.at : null;
if (!at || Number.isNaN(Date.parse(at))) continue;
const r = progressRecord(c.resume);
const count = (v: unknown): number =>
typeof v === "number" && Number.isFinite(v) && v > 0 ? Math.trunc(v) : 0;
const resume =
typeof r.token === "string" &&
r.token.length > 0 &&
r.token.length <= 4096 &&
count(r.page) > 0
? {
token: r.token,
page: count(r.page),
reached: Math.max(count(r.reached), count(r.page)),
}
: undefined;
out.set(key, {
at: new Date(at).toISOString(),
order: c.order === "asc" ? "asc" : "desc",
pages: count(c.pages),
...(resume ? { resume } : {}),
});
}
return out;
}
/**
* Read the `backfillProgress` column into the collections whose backfill
* ended short of the start of their history (the horizon, or a walk that
* stopped getting anywhere), each with where it ended and why. Survives the
* completion stamp, so the integration card can say where history ends.
*/
export function readGoogleHealthBackfillEnded(
value: unknown,
): Map<string, GoogleHealthBackfillEnd> {
const out = new Map<string, GoogleHealthBackfillEnd>();
const ended = progressRecord(progressRecord(value).ended);
for (const [key, raw] of Object.entries(ended)) {
if (key.length > 64) continue;
const e = progressRecord(raw);
const at = typeof e.at === "string" ? e.at : null;
if (!at || Number.isNaN(Date.parse(at))) continue;
if (e.reason !== "horizon" && e.reason !== "no_progress") continue;
if (typeof e.type !== "string" || !(e.type in MeasurementTypes)) {
continue;
}
const attempts =
typeof e.attempts === "number" &&
Number.isInteger(e.attempts) &&
e.attempts >= 1
? e.attempts
: undefined;
const endedAt =
typeof e.endedAt === "string" && !Number.isNaN(Date.parse(e.endedAt))
? new Date(e.endedAt).toISOString()
: undefined;
out.set(key, {
type: e.type as MeasurementType,
at: new Date(at).toISOString(),
reason: e.reason,
...(attempts !== undefined ? { attempts } : {}),
...(endedAt !== undefined ? { endedAt } : {}),
});
}
return out;
}
/**
* Where each data type's imported history ends short of its beginning, and
* why, as the status routes publish it (`backfillEnded`), by type.
*/
export interface GoogleHealthBackfillEndedEntry {
type: MeasurementType;
/** The oldest instant the imported history reaches back to. */
endsAt: string;
reason: GoogleHealthBackfillEnd["reason"];
}
/**
* The `backfillProgress` column's ended collections, as published: only the
* final ends. One still waiting for another attempt is not where its history
* ends yet.
*/
export function googleHealthBackfillEndedList(
value: unknown,
): GoogleHealthBackfillEndedEntry[] {
return [...readGoogleHealthBackfillEnded(value).values()]
.filter(googleHealthBackfillEndIsFinal)
.map((e) => ({ type: e.type, endsAt: e.at, reason: e.reason }))
.sort((a, b) => a.type.localeCompare(b.type));
}
/**
* The `backfillProgress` value to keep once the backfill is stamped complete:
* only where collections ended short, or nothing at all.
*/
export function googleHealthBackfillProgressAfterCompletion(
ended: Map<string, GoogleHealthBackfillEnd>,
): {
ended: Record<string, GoogleHealthBackfillEndJson>;
} | null {
const final = new Map(
[...ended].filter(([, e]) => googleHealthBackfillEndIsFinal(e)),
);
return final.size > 0 ? { ended: endedJson(final) } : null;
}
/** One ended collection as the column stores it. */
type GoogleHealthBackfillEndJson = {
type: string;
at: string;
reason: string;
attempts?: number;
endedAt?: string;
};
/** The ended collections as the column stores them. */
function endedJson(
ended: Map<string, GoogleHealthBackfillEnd>,
): Record<string, GoogleHealthBackfillEndJson> {
return Object.fromEntries(
[...ended].map(([key, e]) => [
key,
{
type: e.type,
at: e.at,
reason: e.reason,
...(e.attempts !== undefined ? { attempts: e.attempts } : {}),
...(e.endedAt !== undefined ? { endedAt: e.endedAt } : {}),
},
]),
);
}
/**
* The collections waiting for another attempt after one ended without
* progress, and the earliest instant (epoch ms) one of them is due, or null
* when none waits.
*/
export function googleHealthBackfillRetryPending(
ended: Map<string, GoogleHealthBackfillEnd>,
): { keys: string[]; dueAt: number } | null {
const keys: string[] = [];
let dueAt = Infinity;
for (const [key, e] of ended) {
const at = googleHealthBackfillEndRetryAt(e);
if (at === null) continue;
keys.push(key);
dueAt = Math.min(dueAt, at);
}
return keys.length > 0 ? { keys, dueAt } : null;
}
/**
* Read the `backfillProgress` column into the collections Google refused the
* edge filter on.
*/
export function readGoogleHealthBackfillEdgeFilterUnsupported(
value: unknown,
): Set<string> {
const keys = progressRecord(value).edgeFilterUnsupported;
return new Set(
Array.isArray(keys)
? keys.filter(
(key): key is string => typeof key === "string" && key.length <= 64,
)
: [],
);
}
/**
* Write the tracker's progress with one change applied, and apply it to the
* tracker only once the write has landed: progress that is not on disk is
@@ -720,26 +473,12 @@ async function persistBackfillProgress(
await prisma.googleHealthConnection.update({
where: { userId: tracker.userId },
data: {
backfillProgress: {
done: [...next.done],
cursors: Object.fromEntries(
[...next.cursors].map(([key, c]) => [
key,
{
at: c.at,
order: c.order,
pages: c.pages,
...(c.resume ? { resume: c.resume } : {}),
},
]),
),
// Only once there is something to say, so a backfill that never met
// the refusal keeps the shape it always had.
...(unsupported.size > 0
? { edgeFilterUnsupported: [...unsupported] }
: {}),
...(ended.size > 0 ? { ended: endedJson(ended) } : {}),
},
backfillProgress: googleHealthBackfillProgressJson({
done: next.done,
cursors: next.cursors,
edgeFilterUnsupported: unsupported,
ended,
}),
},
});
if (unsupported !== tracker.edgeFilterUnsupported) {
@@ -857,20 +596,40 @@ export async function recordGoogleHealthBackfillEnd(
const resume = backfillResumeStorage.getStore();
if (!resume) return;
const ended = new Map(resume.ended);
// Counted across attempts that end at the same edge: a walk that got
// further back before it stalled again made progress, and its count starts
// over. Only attempts in a row that reach nowhere make the end final.
const parked = resume.retrying.get(key);
const sameEdge =
parked !== undefined && Date.parse(parked.at) === Date.parse(end.at);
ended.set(
key,
end.reason === "no_progress"
? {
...end,
// Counted across attempts: the one walked again this attempt
// carries the count it was parked with.
attempts: (resume.retrying.get(key) ?? 0) + 1,
attempts: sameEdge ? parked.attempts + 1 : 1,
endedAt: new Date().toISOString(),
}
: end,
);
// A fixed shape, not the whole note: the slice report carries every end in
// full (`googleHealth.backfill.slice`). This names the one that just ended
// and counts the rest. It is meta, not the event's action, because the job
// names the action once the slice is over and that would replace it.
const recorded = ended.get(key)!;
const final = [...ended.values()].filter(googleHealthBackfillEndIsFinal);
annotate({
meta: { "googleHealth.backfill.ended": Object.fromEntries(ended) },
meta: {
"googleHealth.backfill.ended": {
collection: key,
type: recorded.type,
reason: recorded.reason,
final: googleHealthBackfillEndIsFinal(recorded),
endedCount: ended.size,
finalCount: final.length,
types: [...new Set([...ended.values()].map((e) => e.type))].sort(),
},
},
});
try {
await persistBackfillProgress(resume, {
@@ -1073,8 +832,12 @@ export async function runGoogleHealthCollection(
if (parked) {
// Its earlier attempt ended without progress and its wait is over: walk
// it again from its edge. The note goes with the next write; the count
// stays with the attempt, so a second end without progress is final.
resume.retrying.set(key, parked.attempts ?? 1);
// and the edge stay with the attempt, so a second end at the same edge
// is final.
resume.retrying.set(key, {
attempts: parked.attempts ?? 1,
at: parked.at,
});
resume.ended.delete(key);
}
const startedAt = performance.now();
@@ -110,6 +110,38 @@ function stubCompletion(
);
}
/**
* Postgres names the nights the sleep read reconstructs; answer the night
* page query with the one night the stage rows make, and every other raw
* query with nothing.
*/
function nightPageOver(rows: Array<{ value: number; measuredAt: Date }>) {
const start = Math.min(
...rows.map((r) => r.measuredAt.getTime() - r.value * 60_000),
);
const end = Math.max(...rows.map((r) => r.measuredAt.getTime()));
vi.mocked(prisma.$queryRaw).mockImplementation(((
_strings: TemplateStringsArray,
...values: unknown[]
) =>
Promise.resolve(
values.some((v) =>
String((v as { sql?: string })?.sql ?? "").includes("sess AS"),
)
? [
{
night: new Date(end).toLocaleDateString("sv-SE", {
timeZone: "Europe/Berlin",
}),
span_start: new Date(start),
span_end: new Date(end),
total: 1,
},
]
: [],
)) as never);
}
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(prisma.insightStatusCache.findUnique).mockResolvedValue(null);
@@ -167,6 +199,7 @@ describe("generateMetricStatus — SLEEP_DURATION night reconstruction (iOS E2)"
vi.mocked(prisma.measurement.findMany).mockResolvedValue(
stageRows as never,
);
nightPageOver(stageRows);
const capture = { systemPrompt: null, userPrompt: null } as {
systemPrompt: string | null;
@@ -190,7 +223,7 @@ describe("generateMetricStatus — SLEEP_DURATION night reconstruction (iOS E2)"
expect(prompt).not.toContain('"value": 90');
});
it("the snapshot graded series is the deduped night total, never the ~20 h raw-stage sum (A4)", async () => {
it("the snapshot graded series is the deduped night total, never the ~20 h raw-stage sum", async () => {
// The ~20.3 h symptom: a source writes BOTH a bare ASLEEP aggregate AND the
// granular CORE/DEEP/REM partition for the same span. Pre-fix the graded
// `series` came from `buildGradedSeriesWithRollups`, which folds bare + each
@@ -249,6 +282,7 @@ describe("generateMetricStatus — SLEEP_DURATION night reconstruction (iOS E2)"
vi.mocked(prisma.measurement.findMany).mockResolvedValue(
stageRows as never,
);
nightPageOver(stageRows);
const capture = { systemPrompt: null, userPrompt: null } as {
systemPrompt: string | null;
@@ -0,0 +1,34 @@
import { describe, expect, it } from "vitest";
import { resolveRecoveryEmpty } from "../recovery-empty";
/**
* The recovery page's empty reason is decided here, on the server, from the
* summaries and the Oura connection; clients render the answer.
*/
describe("resolveRecoveryEmpty", () => {
const none = { connected: false, scopeDenied: [] };
it("has no reason when any recovery type has a reading", () => {
expect(resolveRecoveryEmpty({ RESILIENCE: { count: 1 } }, none)).toBeNull();
expect(resolveRecoveryEmpty({ DAY_STRAIN: { count: 3 } }, none)).toBeNull();
});
it("does not count types the page does not chart", () => {
expect(resolveRecoveryEmpty({ RECOVERY_SCORE: { count: 30 } }, none)).toBe(
"general",
);
});
it("asks for a reconnect only when the grant leaves resilience out", () => {
expect(
resolveRecoveryEmpty(
{},
{ connected: true, scopeDenied: ["resilience"] },
),
).toBe("oura_permission");
expect(
resolveRecoveryEmpty({}, { connected: true, scopeDenied: ["spo2"] }),
).toBe("oura_no_resilience");
});
});
@@ -113,10 +113,12 @@ function seedSymptomDrivesSleep() {
functionalImpact: burden,
symptomLinks: [],
});
// A night of five to eight hours, waking at 07:00 UTC: a main sleep, not
// a midday nap, so the nap rule leaves it the night.
measurements.push({
type: "SLEEP_DURATION",
value: 8 - burden,
measuredAt: new Date(BASE + (i + 1) * 86_400_000),
value: (8 - burden) * 60,
measuredAt: new Date(BASE + (i + 1) * 86_400_000 - 5 * 3_600_000),
});
}
measurementFindMany.mockResolvedValue(measurements);
@@ -76,6 +76,7 @@ const sleepScore: SleepScoreValue = {
],
windowNights: 20,
rhythmNights: 20,
rhythmMinNights: 3,
series: [78, 82],
};
@@ -24,7 +24,6 @@ import { mapSleep, type OuraSleep } from "@/lib/oura/client";
import {
computeSleepScore,
reconstructNights,
isDaytimeNap,
rhythmGap,
scoreConsistency,
} from "../sleep-score";
@@ -174,18 +173,6 @@ describe("Sleep Score rhythm on an Oura month", () => {
expect(sub("timing").gap).toBeUndefined();
});
it("tells a nap from a night by length and time of day together", () => {
// Short and in the afternoon: a nap.
expect(isDaytimeNap(60, 14 * 60)).toBe(true);
// Short but at night: the night.
expect(isDaytimeNap(150, 2 * 60)).toBe(false);
expect(isDaytimeNap(150, 22 * 60)).toBe(false);
// A late sleeper's short night, centred at 09:00.
expect(isDaytimeNap(150, 9 * 60)).toBe(false);
// Long in the daytime (after a night shift): a main sleep.
expect(isDaytimeNap(420, 13 * 60)).toBe(false);
});
it("scores the consistency of a steady sleeper, not 0", async () => {
const result = await computeSleepScore("u1", PROFILE, { now: NOW });
expect(result.status).toBe("ok");
+15 -65
View File
@@ -38,11 +38,11 @@ import type {
SleepStage,
} from "@/generated/prisma/client";
import { prisma } from "@/lib/db";
import { wallClockInTz } from "@/lib/tz/wall-clock";
import { resolveUserTimezone } from "@/lib/tz/resolver";
import { shiftDateKey } from "@/lib/tz/format";
import { startOfLocalDayKey } from "@/lib/tz/local-day";
import {
asleepMidpointMinutes,
reconstructSleepNights,
type SleepStageRow,
} from "@/lib/analytics/sleep-night";
@@ -115,6 +115,11 @@ export interface SleepScoreValue {
* `RHYTHM_MIN_NIGHTS` both are withheld with `gap: "few_nights"`.
*/
rhythmNights: number;
/**
* `RHYTHM_MIN_NIGHTS`, published so a client names the floor it shows
* ("2 of 3 nights") from the server instead of carrying its own copy.
*/
rhythmMinNights: number;
/**
* v1.42 — the score of every scorable night in the window (oldest → newest,
* capped to `SPARKLINE_MAX_POINTS`), so a surface can show the score's
@@ -434,34 +439,16 @@ export function reconstructNights(
: n.asleepMinutes > 0 && awakeMinutes > 0
? n.asleepMinutes + awakeMinutes
: null;
// Midpoint = centre of the MAIN session's asleep span: its wake instant
// minus half its own asleep minutes. Not the whole night's: a nap on the
// same wake day moves the night's latest instant to the end of the nap and
// adds its minutes, which put the midpoint of a 23:00 to 07:00 night with a
// 14:00 nap at about 11:00 and read every such day as a ~7-hour shift in
// consistency and timing.
//
// And only when that main session is a main sleep, not a daytime nap. On
// a day the ring or watch recorded no night (left on the charger, taken
// off), the day's largest session is an afternoon nap, and its 14:00
// midpoint sat ten hours from every other night: a single such day in a
// month pushed the midpoint spread past the 90-minute full scale and read
// the whole rhythm as Consistency 0. Such a day still counts its minutes;
// it just has no place on the clock. A short session at night is still
// the night (insomnia, a newborn, split sleep), so it keeps its place.
// Midpoint = centre of the MAIN session's asleep span. Not the whole
// night's: a nap on the same wake day would drag it into the afternoon.
// The main session is the nap convention's (`pickMainNightAndNaps`), so
// a day whose only sleep is a daytime nap has none: it still counts its
// minutes, it just has no place on the clock, exactly as the night view
// and the list show it without a main night.
const main = n.mainSession;
const mainMidpoint = main
? minutesOfDay(
new Date(main.end.getTime() - (main.asleepMinutes / 2) * 60_000),
tz,
)
const midpoint = main
? asleepMidpointMinutes(main.end, main.asleepMinutes, tz)
: null;
const midpoint =
main &&
mainMidpoint != null &&
!isDaytimeNap(main.asleepMinutes, mainMidpoint)
? mainMidpoint
: null;
return {
night: n.night,
asleepMinutes: n.asleepMinutes,
@@ -475,44 +462,6 @@ export function reconstructNights(
});
}
/**
* Below this much asleep time a session can be a nap. Three hours is where
* wearables draw the line between a nap and a main sleep (Oura types a
* period `long_sleep` from three hours on).
*/
const MAIN_SLEEP_MIN_MINUTES = 180;
/**
* The night window, in local minutes-of-day: a session whose midpoint falls
* from 20:00 to 10:00 is night sleep however short it is. Wide enough for a
* late chronotype (asleep 04:00 to 12:00 has its midpoint at 08:00) and for
* an early one, and clear of the afternoon where naps sit.
*/
const NIGHT_WINDOW_START = 20 * 60;
const NIGHT_WINDOW_END = 10 * 60;
/**
* True for a session that is a nap rather than a night: short (under
* `MAIN_SLEEP_MIN_MINUTES` asleep) AND centred in the daytime. A short night
* is still the night; a long daytime sleep (after a night shift) is still a
* main sleep. Pure.
*/
export function isDaytimeNap(
asleepMinutes: number,
midpointMinutes: number,
): boolean {
if (asleepMinutes >= MAIN_SLEEP_MIN_MINUTES) return false;
const atNight =
midpointMinutes >= NIGHT_WINDOW_START || midpointMinutes < NIGHT_WINDOW_END;
return !atNight;
}
function minutesOfDay(d: Date, tz: string): number {
if (tz === "UTC") return d.getUTCHours() * 60 + d.getUTCMinutes();
const { hour, minute } = wallClockInTz(d, tz);
return hour * 60 + minute;
}
// ── compute ─────────────────────────────────────────────────────────────
export interface SleepScoreOpts {
@@ -696,6 +645,7 @@ export async function computeSleepScore(
subScores,
windowNights: scorableNights.length,
rhythmNights: midpoints.length,
rhythmMinNights: RHYTHM_MIN_NIGHTS,
series,
},
coverage,
+11 -29
View File
@@ -71,10 +71,7 @@ import {
summarizeSeries,
} from "@/lib/insights/status-shared";
import { runStatusCompletion } from "@/lib/insights/status-provider";
import {
reconstructSleepNights,
type SleepStageRow,
} from "@/lib/analytics/sleep-night";
import { readSleepNights } from "@/lib/measurements/list/sleep-nights";
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
import { resolveUserTimezone, userDayKey } from "@/lib/tz/resolver";
import {
@@ -295,31 +292,16 @@ export async function generateMetricStatus(args: {
const points: Array<{ measuredAt: Date; value: number }> = [];
let measurements: Array<{ measuredAt: Date }> = [];
if (isSleep) {
const [priorityJson, sleepRows] = await Promise.all([
loadUserSourcePriority(args.userId),
prisma.measurement.findMany({
where: {
userId: args.userId,
type: "SLEEP_DURATION",
deletedAt: null,
},
orderBy: { measuredAt: "desc" },
take: 365 * 6, // ≈ 5 stage rows/night × source-count for a year
// Writer-level collapse: two HealthKit apps behind one source
// (watch stages vs phone in-bed) must not blend into one night.
select: {
value: true,
measuredAt: true,
sleepStage: true,
source: true,
deviceType: true,
},
}),
]);
const nights = reconstructSleepNights(
sleepRows as SleepStageRow[],
userTz,
priorityJson,
// The latest 365 nights, the sleep twin of the 365 latest readings the
// other types read. A cut on stage rows instead (2190 of them) reached
// back a few weeks on a stage-rich watch and began mid-night.
const nights = (
await readSleepNights(
args.userId,
userTz,
await loadUserSourcePriority(args.userId),
{ maxNights: 365, newest: true },
)
).filter((n) => n.asleepMinutes > 0);
for (const n of nights) {
points.push({ measuredAt: n.measuredAt, value: n.asleepMinutes });
+27
View File
@@ -0,0 +1,27 @@
import { prisma } from "@/lib/db";
import { readOuraScopeDenied } from "@/lib/oura/credentials";
import {
hasRecoveryData,
resolveRecoveryEmpty,
type RecoveryEmptyReason,
} from "./recovery-empty";
/**
* The recovery page's empty reason for one account. Reads the Oura
* connection only when no recovery type has a reading, so a page with data
* never pays for it.
*/
export async function loadRecoveryEmpty(
userId: string,
summaries: Record<string, { count?: number } | null | undefined>,
): Promise<RecoveryEmptyReason | null> {
if (hasRecoveryData(summaries)) return null;
const user = await prisma.user.findUnique({
where: { id: userId },
select: { ouraAccessTokenEncrypted: true, ouraScopeDenied: true },
});
return resolveRecoveryEmpty(summaries, {
connected: !!user?.ouraAccessTokenEncrypted,
scopeDenied: readOuraScopeDenied(user?.ouraScopeDenied),
});
}
+60
View File
@@ -0,0 +1,60 @@
/**
* Why the recovery page is empty, decided once on the server.
*
* The page charts the device-native recovery signals below. When none of
* them has a reading, the analytics slice publishes the reason beside the
* summaries (`recoveryEmpty`), and every client renders from it rather than
* working it out from the connection status itself:
*
* - `oura_permission`: Oura is connected and the stored grant leaves out
* the scope its resilience level sits behind; one reconnect fixes it.
* - `oura_no_resilience`: Oura is connected and no level has arrived. The
* server cannot tell "the grant covers it" from "not recorded yet" (the
* scope note is cleared on every reconnect and written only by a sync
* that reached Oura), so this case never claims coverage.
* - `general`: anyone else.
*
* `null` means the page has something to chart.
*
* Pure, so the page can share the type list it charts from.
*/
/** Every summary type the recovery page charts or shows as a tile. */
export const RECOVERY_SUMMARY_TYPES = [
"ANS_CHARGE",
"DAY_STRAIN",
"WORKOUT_STRAIN",
"CARDIO_LOAD",
"AVERAGE_HEART_RATE",
"MAX_HEART_RATE",
"ENERGY_EXPENDITURE_KJ",
"RESILIENCE",
] as const;
export const RECOVERY_EMPTY_REASONS = [
"general",
"oura_permission",
"oura_no_resilience",
] as const;
export type RecoveryEmptyReason = (typeof RECOVERY_EMPTY_REASONS)[number];
/** True when any recovery type has a reading. */
export function hasRecoveryData(
summaries: Record<string, { count?: number } | null | undefined>,
): boolean {
return RECOVERY_SUMMARY_TYPES.some(
(type) => (summaries[type]?.count ?? 0) > 0,
);
}
export function resolveRecoveryEmpty(
summaries: Record<string, { count?: number } | null | undefined>,
oura: { connected: boolean; scopeDenied: readonly string[] },
): RecoveryEmptyReason | null {
if (hasRecoveryData(summaries)) return null;
if (!oura.connected) return "general";
return oura.scopeDenied.includes("resilience")
? "oura_permission"
: "oura_no_resilience";
}
@@ -63,12 +63,13 @@ vi.mock("@/lib/safe-fetch", async (importOriginal) => {
return { ...actual, safeFetch: safeFetchMock };
});
vi.mock("@/lib/google-health/sync-core", () => ({
GOOGLE_HEALTH_INTEGRATION_KEY: "google-health",
readGoogleHealthBackfillDone: () => new Set<string>(resumeState.doneBefore),
readGoogleHealthBackfillCursors: () => new Map(),
readGoogleHealthBackfillEdgeFilterUnsupported: () => new Set<string>(),
readGoogleHealthBackfillEnded: () => new Map(),
vi.mock("@/lib/google-health/backfill-progress", () => ({
readGoogleHealthBackfillProgress: () => ({
done: new Set<string>(resumeState.doneBefore),
cursors: new Map(),
edgeFilterUnsupported: new Set<string>(),
ended: new Map(),
}),
googleHealthBackfillProgressAfterCompletion: (ended: Map<string, unknown>) =>
ended.size > 0 ? { ended: Object.fromEntries(ended) } : null,
googleHealthBackfillRetryPending: (
@@ -79,6 +80,10 @@ vi.mock("@/lib/google-health/sync-core", () => ({
.map(([key]) => key);
return keys.length > 0 ? { keys, dueAt: resumeState.retryDueAt } : null;
},
}));
vi.mock("@/lib/google-health/sync-core", () => ({
GOOGLE_HEALTH_INTEGRATION_KEY: "google-health",
runWithGoogleHealthBackfillResume: async (
_userId: string,
done: Set<string>,
@@ -124,6 +129,7 @@ vi.mock("@/lib/google-health/sync", () => ({
import {
GOOGLE_HEALTH_BACKFILL_MAX_SLICE,
GOOGLE_HEALTH_BACKFILL_THROTTLED_DELAY_SECONDS,
googleHealthBackfillRetryKey,
runGoogleHealthBackfillForUser,
} from "../google-health-backfill";
import { INTEGRATION_BACKFILL_CONTINUATION_DELAY_SECONDS } from "@/lib/jobs/integration-backfill-admission";
@@ -395,6 +401,46 @@ describe("runGoogleHealthBackfillForUser — slices", () => {
});
});
it("keys a waiting retry on when it is due, so a boot's slice cannot start a second chain", async () => {
resumeState.doneBefore = GOOGLE_HEALTH_COLLECTION_KEYS.filter(
(k) => k !== "fetchHeartRate",
);
resumeState.parks = ["fetchHeartRate"];
resumeState.retryDueAt = Date.now() + 20 * 3_600_000;
syncUserGoogleHealthMock.mockResolvedValue({ imported: 0, failed: false });
// The slice that parked the collection, then a boot's first slice meeting
// the same wait from the same note.
await runGoogleHealthBackfillForUser("u1", undefined, 7);
await runGoogleHealthBackfillForUser("u1", undefined, 0);
const keys = bossSend.mock.calls.map(
(call) => (call[2] as { singletonKey: string }).singletonKey,
);
expect(keys).toEqual([
googleHealthBackfillRetryKey("u1", resumeState.retryDueAt),
googleHealthBackfillRetryKey("u1", resumeState.retryDueAt),
]);
expect(keys[0]).toBe(
`google-health-backfill|u1|retry-${new Date(resumeState.retryDueAt).toISOString()}`,
);
});
it("sends a retry that is already due as an ordinary next slice", async () => {
resumeState.doneBefore = GOOGLE_HEALTH_COLLECTION_KEYS.filter(
(k) => k !== "fetchHeartRate",
);
resumeState.parks = ["fetchHeartRate"];
resumeState.retryDueAt = Date.now() - 1_000;
syncUserGoogleHealthMock.mockResolvedValue({ imported: 0, failed: false });
await runGoogleHealthBackfillForUser("u1", undefined, 7);
expect(bossSend.mock.calls[0]![2]).toMatchObject({
singletonKey: "google-health-backfill|u1|slice-8",
startAfter: INTEGRATION_BACKFILL_CONTINUATION_DELAY_SECONDS,
});
});
it("does not stamp over a parked collection when the retry cannot be queued", async () => {
resumeState.doneBefore = GOOGLE_HEALTH_COLLECTION_KEYS.filter(
(k) => k !== "fetchHeartRate",
+1 -1
View File
@@ -110,7 +110,7 @@ export function buildFeedbackBuckets(
row.metricSourceType,
row.providerType,
row.promptVersion,
].join("");
].join("\x01");
const existing = map.get(key);
if (existing) {
if (row.helpful) existing.helpful += 1;
+71 -34
View File
@@ -17,18 +17,19 @@ import { annotate } from "@/lib/logging/context";
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
import {
enqueueIntegrationBackfillAdmission,
INTEGRATION_BACKFILL_ADMISSION_QUEUE,
INTEGRATION_BACKFILL_CONTINUATION_DELAY_SECONDS,
integrationBackfillAdmissionSendOptions,
integrationBackfillIdentity,
integrationBackfillSourceOptions,
type IntegrationBackfillAdmissionPayload,
} from "@/lib/jobs/integration-backfill-admission";
import { runWithGoogleHealthBackfillResume } from "@/lib/google-health/sync-core";
import {
googleHealthBackfillProgressAfterCompletion,
readGoogleHealthBackfillCursors,
readGoogleHealthBackfillDone,
readGoogleHealthBackfillEdgeFilterUnsupported,
readGoogleHealthBackfillEnded,
googleHealthBackfillRetryPending,
runWithGoogleHealthBackfillResume,
} from "@/lib/google-health/sync-core";
readGoogleHealthBackfillProgress,
} from "@/lib/google-health/backfill-progress";
import { googleHealthBackfillEndIsFinal } from "@/lib/google-health/backfill-bounds";
import {
isGoogleHealthParked,
@@ -104,6 +105,23 @@ export const GOOGLE_HEALTH_BACKFILL_MAX_SLICE =
) +
2;
/**
* The singleton key of a slice that waits for a parked collection's retry:
* the import's identity and the instant the retry is due. The instant comes
* from the stored note, so every slice that reads the same note derives the
* same key.
*/
export function googleHealthBackfillRetryKey(
userId: string,
dueAt: number,
): string {
const identity = integrationBackfillIdentity({
kind: "google-health-backfill",
data: { userId, enqueuedAt: "" },
});
return `${identity}|retry-${new Date(dueAt).toISOString()}`;
}
/** Test seam: a slice's budget, when not the production one. */
export interface GoogleHealthBackfillSliceOptions {
sliceMs?: number;
@@ -198,14 +216,12 @@ export async function runGoogleHealthBackfillForUser(
});
return { imported: 0 };
}
const done = readGoogleHealthBackfillDone(connection?.backfillProgress);
const cursors = readGoogleHealthBackfillCursors(connection?.backfillProgress);
const edgeFilterUnsupported = readGoogleHealthBackfillEdgeFilterUnsupported(
connection?.backfillProgress,
);
const endedBefore = readGoogleHealthBackfillEnded(
connection?.backfillProgress,
);
const {
done,
cursors,
edgeFilterUnsupported,
ended: endedBefore,
} = readGoogleHealthBackfillProgress(connection?.backfillProgress);
// A collection that ended short for good has read all it is going to, so
// it counts as finished alongside the ones read to the end. One parked
// after a first end without progress does not: it waits for another try.
@@ -285,23 +301,36 @@ export async function runGoogleHealthBackfillForUser(
);
}
const sendNextSlice = async (delaySeconds: number): Promise<boolean> => {
/**
* Queue the next slice. A slice that waits for a parked collection's retry
* goes under a key named after when the retry is due, instead of its slice
* number: every slice that meets the same wait, a boot's included, names the
* same job, and the queue keeps one. Under the slice number a boot during
* the day-long wait started a second chain beside the waiting one.
*/
const sendNextSlice = async (
delaySeconds: number,
retryDueAt?: number,
): Promise<boolean> => {
if (slice + 1 > GOOGLE_HEALTH_BACKFILL_MAX_SLICE) return false;
const boss = getGlobalBoss();
const sent = boss
? await enqueueIntegrationBackfillAdmission(
boss,
{
kind: "google-health-backfill",
data: {
userId,
enqueuedAt: new Date().toISOString(),
slice: slice + 1,
},
},
delaySeconds,
)
: null;
if (!boss) return false;
const payload: IntegrationBackfillAdmissionPayload = {
kind: "google-health-backfill",
data: {
userId,
enqueuedAt: new Date().toISOString(),
slice: slice + 1,
},
};
const sent =
retryDueAt === undefined
? await enqueueIntegrationBackfillAdmission(boss, payload, delaySeconds)
: await boss.send(INTEGRATION_BACKFILL_ADMISSION_QUEUE, payload, {
...integrationBackfillAdmissionSendOptions(payload),
singletonKey: googleHealthBackfillRetryKey(userId, retryDueAt),
startAfter: delaySeconds,
});
return Boolean(sent);
};
@@ -315,11 +344,18 @@ export async function runGoogleHealthBackfillForUser(
(key) => done.has(key) || pending.keys.includes(key),
);
if (pending && (onlyRetriesLeft || !failed)) {
const waitSeconds = Math.ceil((pending.dueAt - Date.now()) / 1000);
const delaySeconds = Math.max(
INTEGRATION_BACKFILL_CONTINUATION_DELAY_SECONDS,
Math.ceil((pending.dueAt - Date.now()) / 1000),
waitSeconds,
);
if (await sendNextSlice(delaySeconds)) {
// A retry already due is an ordinary next slice; only a wait is keyed on
// when it ends.
const delayed =
waitSeconds > INTEGRATION_BACKFILL_CONTINUATION_DELAY_SECONDS;
if (
await sendNextSlice(delaySeconds, delayed ? pending.dueAt : undefined)
) {
annotate({
action: {
name: "google_health.backfill.retry_scheduled",
@@ -328,9 +364,10 @@ export async function runGoogleHealthBackfillForUser(
});
return { imported, continued: true };
}
// Not queued (most often one is already waiting under the same slice
// key, sent by an earlier boot's slice). Nothing is lost: the marker is
// not stamped, so the next boot offers the backfill again.
// Not queued: most often the same retry is already waiting, sent by the
// slice that parked the collection, and this is a boot's slice meeting
// the same wait. Nothing is lost either way: the marker is not stamped,
// so the next boot offers the backfill again.
annotate({
action: {
name: "google_health.backfill.retry_not_queued",
+233
View File
@@ -0,0 +1,233 @@
import { prisma } from "@/lib/db";
import { annotate } from "@/lib/logging/context";
import { apiSuccess } from "@/lib/api-response";
import { metricKeyForType } from "@/lib/measurements/cumulative-day-sum";
import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority";
import {
canonicalDailyTimestamp,
dayKeyForUserTz,
} from "@/lib/measurements/drain-per-sample-cumulative";
import { localDayWindow } from "@/lib/tz/local-day";
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
import { resolveStoredTimezone } from "@/lib/tz/resolver";
import {
Prisma,
type MeasurementSource,
type MeasurementType,
} from "@/generated/prisma/client";
import { pageWithWindowTotal } from "./paging";
/**
* The day-grouped list for a cumulative type, paged over local days.
*
* Everything heavy runs in Postgres. The first query picks the page: the
* distinct local days that hold a reading, sorted and cut by `limit` /
* `offset`, each row carrying the distinct-day count of the whole filtered
* set as `meta.total` from a window count. The second folds only the page's
* days, one row per day, source, device and unit with its sum and sample
* count, and the source ladder picks among those groups exactly as it would
* among the raw rows (it keeps or drops whole source and device groups per
* day, never single rows). So a page costs its own days whatever the length
* of the history, every day is complete, and no raw row reaches the heap.
*
* Until v1.43.6 the branch read the newest `limit` raw rows (the web asked
* for 5000), bucketed them and returned every bucket. A long history came
* back as thousands of rows in one response, and every day older than the
* 5000th raw row was missing with nothing to say so.
*
* A day exists for the page when any reading of the type falls on it; the
* source collapse narrows a day's groups but never empties one, so the count
* and the folded rows agree. A value range would filter single readings and
* leave partial days behind, so the validator refuses `valueMin` /
* `valueMax` beside `groupBy=day`.
*
* Every page reads the filtered history to find its days. After the nightly
* drain a cumulative type holds about one row per day and source, so the
* scan is the index-only walk over days it would be anyway.
*/
export async function dayGroupedListResponse(
user: { id: string; timezone?: string | null },
opts: {
type: MeasurementType;
limit: number;
offset: number;
sortDir: "asc" | "desc";
sourceEq?: MeasurementSource;
from?: Date;
to?: Date;
},
) {
const { type, limit, offset, sortDir } = opts;
// One resolver for the stored zone, shared with the drill-down and the
// sleep list: a usable IANA name or the instance default, never a value
// Postgres would refuse. The drill-down must cut the same days.
const tz = await resolveStoredTimezone(user.timezone);
// The request's filters, as SQL: the same ones the plain list applies.
const filters = Prisma.sql`
m."user_id" = ${user.id}
AND m."type" = ${type}::measurement_type
AND m."deleted_at" IS NULL
${opts.sourceEq ? Prisma.sql`AND m."source" = ${opts.sourceEq}::measurement_source` : Prisma.empty}
${opts.from ? Prisma.sql`AND m."measured_at" >= ${opts.from}` : Prisma.empty}
${opts.to ? Prisma.sql`AND m."measured_at" <= ${opts.to}` : Prisma.empty}
`;
const localDay = Prisma.sql`to_char((m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD')`;
// Direction comes from the Zod enum, never from raw input. The window
// count runs before LIMIT, so every row of the page carries the
// distinct-day total of the whole filtered set.
const direction = Prisma.raw(sortDir === "asc" ? "ASC" : "DESC");
const { rows: pageDays, total } = await pageWithWindowTotal(
await prisma.$queryRaw<Array<{ day_key: string; total: number }>>`
SELECT d.day_key, (COUNT(*) OVER ())::int AS total
FROM (
SELECT ${localDay} AS day_key
FROM measurements m
WHERE ${filters}
GROUP BY 1
) d
ORDER BY d.day_key ${direction}
LIMIT ${limit} OFFSET ${offset}
`,
offset,
async () =>
Number(
(
await prisma.$queryRaw<Array<{ total: number }>>`
SELECT COUNT(DISTINCT ${localDay})::int AS total
FROM measurements m
WHERE ${filters}
`
)[0]?.total ?? 0,
),
);
const dayKeys = pageDays.map((d) => d.day_key);
// The page's days are consecutive in the sorted day set, so the span from
// the first one's start to the last one's end holds exactly their rows.
let groups: Array<{
day_key: string;
source: MeasurementSource;
device_type: string | null;
unit: string;
total: number;
n: number;
}> = [];
if (dayKeys.length > 0) {
const sorted = [...dayKeys].sort();
const spanStart = localDayWindow(sorted[0], tz).dayStart;
const spanEnd = localDayWindow(sorted[sorted.length - 1], tz).dayEnd;
// Newest group first within a day, so the day's label comes from its
// latest reading, as it did when raw rows were read newest first.
groups = await prisma.$queryRaw`
SELECT
${localDay} AS day_key,
m."source"::text AS source,
m."device_type" AS device_type,
m."unit" AS unit,
SUM(m."value")::float8 AS total,
COUNT(*)::int AS n
FROM measurements m
WHERE ${filters}
AND m."measured_at" >= ${spanStart}
AND m."measured_at" < ${spanEnd}
GROUP BY 1, 2, 3, 4
ORDER BY 1, MAX(m."measured_at") DESC, 2, 3, 4
`;
}
// v1.29.6 — collapse overlapping sources to the ladder-canonical
// reading per day BEFORE summing. Without this, a day with both an
// Apple Health AND a Withings write for the same cumulative metric
// (e.g. steps) summed BOTH sources into one inflated total — a day
// the dashboard tile (which reads through the source-collapsed
// rollup, `collapseRollupRowsBySource` in
// `src/lib/rollups/measurement-read.ts`) showed correctly. Uses the
// same `metricKeyForType` ladder lookup the rollup collapse uses, so
// this list and the tile agree on which source wins. A type with no
// ladder (e.g. TIME_IN_DAYLIGHT, FALL_COUNT) has nothing to collapse
// against — every group passes through unchanged. Each group stands at
// its day's local noon, so the picker files it under the day SQL cut.
const groupRows = groups.map((g) => ({
...g,
type,
deviceType: g.device_type,
measuredAt: canonicalDailyTimestamp(g.day_key, tz),
}));
const metricKey = metricKeyForType(type);
const priorityJson = metricKey ? await loadUserSourcePriority(user.id) : null;
const canonicalGroups = metricKey
? pickCanonicalSourceRows(groupRows, metricKey, priorityJson, (d) =>
dayKeyForUserTz(d, tz),
).canonicalRows
: groupRows;
// Sum per local day; unit and source come from the day's first kept group
// so the rendered row carries a sensible label even when sources mix.
const buckets = new Map<
string,
{ total: number; sampleCount: number; unit: string; source: string }
>();
for (const g of canonicalGroups) {
const slot = buckets.get(g.day_key);
if (!slot) {
buckets.set(g.day_key, {
total: g.total,
sampleCount: g.n,
unit: g.unit,
source: g.source,
});
} else {
slot.total += g.total;
slot.sampleCount += g.n;
}
}
const rowsScanned = groups.reduce((sum, g) => sum + g.n, 0);
const rowsKept = canonicalGroups.reduce((sum, g) => sum + g.n, 0);
// Rows follow the page's day order from SQL.
const measurements = dayKeys.flatMap((dayKey) => {
const b = buckets.get(dayKey);
if (!b) return [];
return [
{
// Synthetic id so the list-row key stays stable across pages
// and the row never collides with a real `Measurement.id`.
id: `day:${type}:${dayKey}`,
type,
value: b.total,
unit: b.unit,
source: b.source,
// Anchor the row's timestamp to local noon so the row sorts
// cleanly between same-day spot samples — matches the
// `canonicalDailyTimestamp` convention used by the drain.
measuredAt: canonicalDailyTimestamp(dayKey, tz).toISOString(),
notes: null,
dayKey,
sampleCount: b.sampleCount,
},
];
});
annotate({
action: { name: "measurement.list" },
meta: {
total,
type,
groupBy: "day",
days: measurements.length,
rowsScanned,
droppedDuplicates: rowsScanned - rowsKept,
},
});
return apiSuccess({
measurements,
meta: {
total,
limit,
offset,
groupBy: "day",
droppedDuplicates: rowsScanned - rowsKept,
},
});
}
+17
View File
@@ -0,0 +1,17 @@
/**
* A page cut in SQL that carries the size of the whole set.
*
* The page query selects `(COUNT(*) OVER ())::int AS total` beside its rows:
* the window count runs before `LIMIT`, so every row of the page carries the
* count of the whole filtered set and one round trip answers both. A page
* past the end has no row to carry it; only then is the set counted on its
* own, by `countAll`.
*/
export async function pageWithWindowTotal<T extends { total: number }>(
rows: T[],
offset: number,
countAll: () => Promise<number>,
): Promise<{ rows: T[]; total: number }> {
if (rows.length > 0) return { rows, total: Number(rows[0].total) };
return { rows, total: offset > 0 ? await countAll() : 0 };
}
+169
View File
@@ -0,0 +1,169 @@
/**
* The SLEEP_DURATION list: one row per night, or one night's stage segments.
*
* Sleep is stored one row per STAGE per night (minutes); a plain list would
* leak every stage segment as its own "value". The list collapses the raw rows
* into the canonical per-night shape used by the chart series and the
* dashboard tile.
*
* Two modes:
* - `dayKey` set → drill-down: the night's stage SEGMENTS (the canonical
* source's per-stage spans), read exactly as the night view reads them.
* - no `dayKey` → one synthetic row per night, value = the main sleep's TIME
* ASLEEP in minutes (the canonical unit), with the per-night stage
* breakdown, nap count and awakenings in the row so the UI can caption the
* night without a second fetch.
*
* The nights are paged in Postgres (`sleepNightPage`) and only the stage rows
* of the page's nights are read. Every page night gets a row, so a page is as
* long as its limit and `meta.total` counts exactly the rows a full walk sees.
* A day whose only sleep is daytime naps has no main night: its row carries
* 0 minutes asleep and its naps, as `/api/sleep/night` answers the same day
* with `main: null` and the naps.
*/
import { annotate } from "@/lib/logging/context";
import { apiSuccess } from "@/lib/api-response";
import { getUnitForType } from "@/lib/validations/measurement";
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
import { resolveStoredTimezone } from "@/lib/tz/resolver";
import {
pickMainNightAndNaps,
reconstructSleepSessions,
type SleepSession,
} from "@/lib/analytics/sleep-night";
import {
readNightSessions,
readNightSpanRows,
sleepNightPage,
} from "./sleep-nights";
/**
* A sleep page holds at most this many nights. The nights' stage rows are
* read and reconstructed in the process, so the bound sits on the nights;
* the applied value comes back as `meta.limit`, and `meta.total` still counts
* every night, so a caller asking for more pages on rather than losing any.
*/
export const SLEEP_NIGHT_PAGE_MAX = 1000;
export async function sleepListResponse(
user: { id: string; timezone?: string | null },
opts: {
from?: Date;
to?: Date;
limit: number;
offset: number;
sortDir: "asc" | "desc";
dayKey?: string;
},
) {
const { from, to, offset, sortDir, dayKey } = opts;
const limit = Math.min(opts.limit, SLEEP_NIGHT_PAGE_MAX);
const [tz, priorityJson] = await Promise.all([
resolveStoredTimezone(user.timezone),
loadUserSourcePriority(user.id),
]);
const dayUnit = getUnitForType("SLEEP_DURATION");
if (dayKey) {
const { sessions } = await readNightSessions(
user.id,
dayKey,
tz,
priorityJson,
{ from, to },
);
const { main, naps } = pickMainNightAndNaps(sessions);
// One row per stage segment of the main night + each nap, so the
// drill-down reveals exactly the rows that summed into the row.
const measurements = [main, ...naps]
.filter((s): s is SleepSession => s != null)
.flatMap((s) => s.segments)
.sort((a, b) =>
sortDir === "asc"
? a.start.getTime() - b.start.getTime()
: b.start.getTime() - a.start.getTime(),
)
.map((seg, i) => ({
id: `sleep-seg:${dayKey}:${i}`,
type: "SLEEP_DURATION",
value: Math.round(seg.minutes),
unit: dayUnit,
source: main?.source ?? naps[0]?.source ?? "MANUAL",
measuredAt: seg.end.toISOString(),
sleepStage: seg.stage,
notes: null,
}));
annotate({
action: { name: "measurement.list" },
meta: {
total: measurements.length,
type: "SLEEP_DURATION",
dayKey,
mode: "sleep-drill-down",
},
});
return apiSuccess({
measurements,
meta: { total: measurements.length, limit, offset: 0, dayKey },
});
}
const page = await sleepNightPage(user.id, tz, {
from,
to,
sortDir,
limit,
offset,
});
const rows = await readNightSpanRows(user.id, page.nights, { from, to });
const byDay = new Map<string, SleepSession[]>();
for (const s of reconstructSleepSessions(rows, tz, priorityJson)) {
const list = byDay.get(s.night) ?? [];
list.push(s);
byDay.set(s.night, list);
}
// Rows follow the page's night order from SQL.
const measurements = page.nights.map(({ night, spanEnd }) => {
const { main, naps } = pickMainNightAndNaps(byDay.get(night) ?? []);
const lastNap = naps.at(-1);
return {
// Synthetic id mirrors the series route's `sleep:<night>` shape.
id: `sleep:${night}`,
type: "SLEEP_DURATION",
value: main ? Math.round(main.asleepMinutes) : 0,
unit: dayUnit,
source: main?.source ?? naps[0]?.source ?? "MANUAL",
measuredAt: (main?.end ?? lastNap?.end ?? spanEnd).toISOString(),
notes: null,
dayKey: night,
// Drives the drill-down chevron: the stage segments behind the row,
// the main night's and the naps', exactly what the drill-down returns.
sampleCount: [main, ...naps].reduce(
(sum, s) => sum + (s?.segments.length ?? 0),
0,
),
sleepStages: (main?.stages ?? {}) as Record<string, number>,
inBedMinutes: main?.inBedMinutes ?? null,
awakeMinutes: main?.awakeMinutes ?? null,
napCount: naps.length,
napAsleepMinutes: naps.reduce((sum, n) => sum + n.asleepMinutes, 0),
awakenings: main?.awakenings ?? 0,
};
});
annotate({
action: { name: "measurement.list" },
meta: {
total: page.total,
type: "SLEEP_DURATION",
mode: "sleep-per-night",
nights: measurements.length,
rowsScanned: rows.length,
},
});
return apiSuccess({
measurements,
meta: { total: page.total, limit, offset, groupBy: "night" },
});
}
+321
View File
@@ -0,0 +1,321 @@
/**
* Sleep nights read from Postgres, a page or a night at a time.
*
* Sleep is stored one row per stage, so a long history is a large row set:
* every reader here keeps the stage rows that reach the process to the
* nights it was asked about, and the reconstruction itself stays the one in
* `sleep-night.ts`. The SQL below only decides which nights exist and where
* their sessions lie; what a night holds is always the JS reconstruction's.
*/
import { prisma } from "@/lib/db";
import { Prisma } from "@/generated/prisma/client";
import type { MeasurementSource, SleepStage } from "@/generated/prisma/client";
import {
ASLEEP_STAGES,
SESSION_GAP_MS,
reconstructSleepNights,
reconstructSleepSessions,
type SleepNight,
type SleepSession,
} from "@/lib/analytics/sleep-night";
import { localDayWindow } from "@/lib/tz/local-day";
import { pageWithWindowTotal } from "./paging";
/** A stage row as every sleep reader selects it. */
export interface SleepRow {
id: string;
value: number;
measuredAt: Date;
sleepStage: SleepStage | null;
source: MeasurementSource;
deviceType: string | null;
}
/** The read bounds a list request passes through, both inclusive. */
export interface SleepReadBounds {
from?: Date;
to?: Date;
}
/**
* How far a one-night read reaches either side of its day. Sessions split at
* a three-hour gap, so a session ending on that day starts inside the reach
* unless it lasted over a day and a half, and every session the cut
* truncates ends on another day.
*/
export const NIGHT_READ_REACH_MS = 36 * 60 * 60 * 1000;
/**
* The asleep stages as a SQL list. A whitelist splice: each value is asserted
* against the closed enum shape before it is joined, so nothing from a
* request can reach it, and the list is `ASLEEP_STAGES`, the very set the
* JS reconstruction counts as asleep.
*/
const ASLEEP_STAGE_SQL = Prisma.raw(
ASLEEP_STAGES.map((stage) => {
if (!/^[A-Z_]+$/.test(stage)) throw new Error(`bad sleep stage ${stage}`);
return `'${stage}'`;
}).join(", "),
);
/**
* Stage rows between two instants (inclusive), oldest first, narrowed by the
* request's own bounds.
*/
export async function readSleepRows(
userId: string,
gte: Date,
lte: Date,
bounds: SleepReadBounds = {},
): Promise<SleepRow[]> {
return prisma.measurement.findMany({
where: {
userId,
type: "SLEEP_DURATION",
deletedAt: null,
AND: [
{ measuredAt: { gte, lte } },
...(bounds.from ? [{ measuredAt: { gte: bounds.from } }] : []),
...(bounds.to ? [{ measuredAt: { lte: bounds.to } }] : []),
],
},
orderBy: [{ measuredAt: "asc" }, { id: "asc" }],
select: {
id: true,
value: true,
measuredAt: true,
sleepStage: true,
source: true,
// Writer-level collapse: two HealthKit apps behind one source (watch
// stages vs phone in-bed) must not blend into one night.
deviceType: true,
},
});
}
/**
* The sessions of one wake day: the stage rows within `NIGHT_READ_REACH_MS`
* of the day, reconstructed, and the sessions filed under it kept.
*/
export async function readNightSessions(
userId: string,
day: string,
tz: string,
priorityJson: unknown,
bounds: SleepReadBounds = {},
): Promise<{ sessions: SleepSession[]; rows: SleepRow[] }> {
const { dayStart, dayEnd } = localDayWindow(day, tz);
const rows = await readSleepRows(
userId,
new Date(dayStart.getTime() - NIGHT_READ_REACH_MS),
new Date(dayEnd.getTime() + NIGHT_READ_REACH_MS),
bounds,
);
const sessions = reconstructSleepSessions(rows, tz, priorityJson).filter(
(s) => s.night === day,
);
return { sessions, rows };
}
/**
* The wake day of the most recent night that holds sleep, or null. Found
* from the latest asleep row: the night holding it is the newest one, and
* its sessions lie within the one-night reach of that row.
*/
export async function latestSleepNight(
userId: string,
tz: string,
priorityJson: unknown,
): Promise<string | null> {
const latest = await prisma.measurement.findFirst({
where: {
userId,
type: "SLEEP_DURATION",
deletedAt: null,
value: { gt: 0 },
OR: [{ sleepStage: null }, { sleepStage: { in: [...ASLEEP_STAGES] } }],
},
orderBy: { measuredAt: "desc" },
select: { measuredAt: true },
});
if (!latest) return null;
const at = latest.measuredAt.getTime();
const rows = await readSleepRows(
userId,
new Date(at - NIGHT_READ_REACH_MS),
new Date(at + NIGHT_READ_REACH_MS),
);
const scorable = reconstructSleepSessions(rows, tz, priorityJson).filter(
(s) => s.asleepMinutes > 0,
);
return scorable.at(-1)?.night ?? null;
}
/** One night of a page: its wake day and the span its sessions cover. */
export interface SleepNightSpan {
night: string;
spanStart: Date;
spanEnd: Date;
}
/**
* One page of sleep nights, chosen in Postgres.
*
* The stage rows are cut into sessions the way `reconstructSleepSessions`
* cuts them: ordered by start (`measured_at` minus the minutes), a new
* session wherever a row starts more than `SESSION_GAP_MS` after every
* earlier row has ended. A session counts when it holds an asleep row (an
* `ASLEEP_STAGES` stage, or a stage-less row, with minutes), and its night is
* the local day its last row ends on: the reconstruction's own rule, so a
* page night and a reconstructed night name the same day. Nights are grouped,
* sorted and cut by `limit` / `offset`; `total` counts the whole filtered set.
* `spanStart` / `spanEnd` bound the night's counting sessions, so the caller
* reads exactly their rows.
*
* Before the cut, rows are folded into hour buckets by their end: min start,
* max end, any asleep. That cannot move a session boundary. Two rows whose
* ends lie less than the session gap apart always share a session (the later
* one starts before its own end, which is within the gap of the earlier end),
* so a bucket narrower than the gap only ever holds rows of one session, and
* its bounds are that group's bounds. It cuts the rows the window functions
* sort from every stage to a handful per night.
*/
export async function sleepNightPage(
userId: string,
tz: string,
opts: SleepReadBounds & {
sortDir: "asc" | "desc";
limit: number;
offset: number;
},
): Promise<{ nights: SleepNightSpan[]; total: number }> {
const gapSeconds = SESSION_GAP_MS / 1000;
// Direction comes from the Zod enum, never from raw input.
const direction = Prisma.raw(opts.sortDir === "asc" ? "ASC" : "DESC");
const nightsCte = Prisma.sql`
WITH r AS (
SELECT
MIN(m."measured_at" - m."value" * interval '1 minute') AS s,
MAX(m."measured_at") AS e,
BOOL_OR(m."value" > 0 AND (m."sleep_stage" IS NULL
OR m."sleep_stage" IN (${ASLEEP_STAGE_SQL}))) AS asleep
FROM measurements m
WHERE m."user_id" = ${userId}
AND m."type" = 'SLEEP_DURATION'::measurement_type
AND m."deleted_at" IS NULL
${opts.from ? Prisma.sql`AND m."measured_at" >= ${opts.from}` : Prisma.empty}
${opts.to ? Prisma.sql`AND m."measured_at" <= ${opts.to}` : Prisma.empty}
GROUP BY date_bin(interval '1 hour', m."measured_at", TIMESTAMPTZ '2000-01-01 00:00:00+00')
), o AS (
SELECT r.*, MAX(e) OVER (
ORDER BY s, e ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
) AS prev_end
FROM r
), g AS (
SELECT o.*, SUM(
CASE WHEN prev_end IS NULL
OR s - prev_end > make_interval(secs => ${gapSeconds}::double precision) THEN 1 ELSE 0 END
) OVER (ORDER BY s, e ROWS UNBOUNDED PRECEDING) AS sid
FROM o
), sess AS (
SELECT MIN(s) AS s, MAX(e) AS e
FROM g
GROUP BY sid
HAVING BOOL_OR(asleep)
), nights AS (
SELECT
to_char((e AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD') AS night,
MIN(s) AS span_start,
MAX(e) AS span_end
FROM sess
GROUP BY 1
)
`;
const { rows, total } = await pageWithWindowTotal(
await prisma.$queryRaw<
Array<{ night: string; span_start: Date; span_end: Date; total: number }>
>`
${nightsCte}
SELECT night, span_start, span_end, (COUNT(*) OVER ())::int AS total
FROM nights
ORDER BY night ${direction}
LIMIT ${opts.limit} OFFSET ${opts.offset}
`,
opts.offset,
async () =>
Number(
(
await prisma.$queryRaw<Array<{ total: number }>>`
${nightsCte}
SELECT COUNT(*)::int AS total FROM nights
`
)[0]?.total ?? 0,
),
);
return {
nights: rows.map((r) => ({
night: r.night,
spanStart: r.span_start,
spanEnd: r.span_end,
})),
total,
};
}
/**
* The stage rows of a set of page nights, read in one span. The nights are
* consecutive in the sorted night set, so the span from the first one's start
* to the last one's end holds their sessions whole; any other session in it
* holds no sleep.
*/
export async function readNightSpanRows(
userId: string,
nights: readonly SleepNightSpan[],
bounds: SleepReadBounds = {},
): Promise<SleepRow[]> {
if (nights.length === 0) return [];
let spanStart = nights[0].spanStart;
let spanEnd = nights[0].spanEnd;
for (const n of nights) {
if (n.spanStart < spanStart) spanStart = n.spanStart;
if (n.spanEnd > spanEnd) spanEnd = n.spanEnd;
}
return readSleepRows(userId, spanStart, spanEnd, bounds);
}
/** How many nights one chunked read reconstructs at a time. */
const NIGHT_CHUNK = 90;
/**
* Every night in a window as `reconstructSleepNights` builds it, oldest
* first, without holding the window's stage rows at once: Postgres names the
* nights and their spans, and the rows are read and reconstructed ninety
* nights at a time. Any window length costs a bounded heap, so a reader never
* has to cut a long range short to stay safe. `maxNights` keeps the oldest
* nights of the window, or with `newest` the most recent ones.
*/
export async function readSleepNights(
userId: string,
tz: string,
priorityJson: unknown,
bounds: SleepReadBounds & { maxNights: number; newest?: boolean },
): Promise<SleepNight[]> {
const page = await sleepNightPage(userId, tz, {
from: bounds.from,
to: bounds.to,
sortDir: bounds.newest ? "desc" : "asc",
limit: bounds.maxNights,
offset: 0,
});
const nights = bounds.newest ? page.nights.reverse() : page.nights;
const out: SleepNight[] = [];
for (let i = 0; i < nights.length; i += NIGHT_CHUNK) {
const chunk = nights.slice(i, i + NIGHT_CHUNK);
const keys = new Set(chunk.map((n) => n.night));
const rows = await readNightSpanRows(userId, chunk, bounds);
for (const night of reconstructSleepNights(rows, tz, priorityJson)) {
if (keys.has(night.night)) out.push(night);
}
}
return out;
}
@@ -23,7 +23,9 @@ import { walkSourceFiles } from "@/__tests__/helpers/source-files";
import {
HEAP_CHECK_INTERVAL_MS,
HEAP_CRITICAL_RATIO,
HEAP_PRESSURE_COOLDOWN_MS,
HEAP_PRESSURE_RATIO,
HEAP_PRESSURE_LIST_MAX,
HEAP_SAMPLE_INTERVAL_ELEVATED_MS,
HEAP_SAMPLE_INTERVAL_MS,
@@ -175,6 +177,41 @@ describe("heap pressure report", () => {
expect(h.named(HEAP_TASK_NAMES.pressure)).toHaveLength(2);
});
it("reports again at 95 % inside the cooldown, once per climb", () => {
const h = harness();
h.state.used = 900;
h.run(60_000);
// Ten minutes later, well inside the cooldown, the heap nears its limit.
h.state.used = 800;
h.run(10 * 60_000);
h.state.used = 960;
h.run(60_000);
const pressure = h.named(HEAP_TASK_NAMES.pressure);
expect(pressure.map((p) => p.meta.pressure_ratio)).toEqual([
HEAP_PRESSURE_RATIO,
HEAP_CRITICAL_RATIO,
]);
expect(pressure[1].meta).toHaveProperty("in_flight_oldest");
// Hovering at 95 % writes nothing more; back under 85 % re-arms it.
h.run(5 * 60_000);
expect(h.named(HEAP_TASK_NAMES.pressure)).toHaveLength(2);
h.state.used = 840;
h.run(HEAP_CHECK_INTERVAL_MS);
h.state.used = 970;
h.run(HEAP_CHECK_INTERVAL_MS);
expect(h.named(HEAP_TASK_NAMES.pressure)).toHaveLength(3);
});
it("writes one report at the tier a sudden jump reaches", () => {
const h = harness();
h.state.used = 990;
h.run(60_000);
const pressure = h.named(HEAP_TASK_NAMES.pressure);
expect(pressure).toHaveLength(1);
expect(pressure[0].meta.pressure_ratio).toBe(HEAP_CRITICAL_RATIO);
});
it("writes no snapshot unless opted in, and at most one when it is", () => {
const off = harness();
off.state.used = 900;
+29 -8
View File
@@ -9,6 +9,7 @@ import {
import { envFlag, envOr } from "@/lib/env";
import { WideEventBuilder } from "@/lib/logging/event-builder";
import { emitIfSampled } from "@/lib/logging/transports";
import { megabytes } from "@/lib/support/heap-limit";
import { inFlightRegistry, type InFlightRegistry } from "./in-flight";
@@ -32,7 +33,11 @@ import { inFlightRegistry, type InFlightRegistry } from "./in-flight";
* flight and the routes most of them share (see `in-flight.ts`). It
* re-arms only after the heap has fallen back under 70 % AND thirty minutes
* have passed, so a heap hovering at the line writes one report, not one
* every check.
* every check. A second tier at 95 % writes the same report with no
* cooldown at all, and re-arms as soon as the heap is back under 85 %:
* a heap that climbs to its limit within the half hour after an 85 %
* report would otherwise die with only that older report on record, the
* requests that pushed it over missing from it.
* - `runtime.heap.snapshot`, at `warn`, only with the opt-in below: where
* the snapshot was written, or that writing it failed.
*
@@ -63,6 +68,7 @@ export const HEAP_SAMPLE_INTERVAL_MS = 5 * 60_000;
export const HEAP_SAMPLE_INTERVAL_ELEVATED_MS = 60_000;
export const HEAP_ELEVATED_RATIO = 0.7;
export const HEAP_PRESSURE_RATIO = 0.85;
export const HEAP_CRITICAL_RATIO = 0.95;
export const HEAP_PRESSURE_COOLDOWN_MS = 30 * 60_000;
export const HEAP_PRESSURE_LIST_MAX = 50;
@@ -101,10 +107,6 @@ export interface HeapMonitorDeps {
writeSnapshot: () => SnapshotOutcome;
}
function megabytes(bytes: number): number {
return Math.round(bytes / 1024 / 1024);
}
function defaultReadHeap(): { used: number; limit: number } {
const stats = getHeapStatistics();
return { used: stats.used_heap_size, limit: stats.heap_size_limit };
@@ -205,6 +207,7 @@ export function writeDiagnosticHeapSnapshot(): SnapshotOutcome {
export class HeapMonitor {
private lastSampleAt: number | null = null;
private pressureArmed = true;
private criticalArmed = true;
private lastPressureAt: number | null = null;
private snapshotTaken = false;
@@ -225,11 +228,29 @@ export class HeapMonitor {
this.pressureArmed = true;
}
if (!this.criticalArmed && ratio < HEAP_PRESSURE_RATIO) {
this.criticalArmed = true;
}
if (this.pressureArmed && ratio >= HEAP_PRESSURE_RATIO) {
this.pressureArmed = false;
this.lastPressureAt = at;
this.lastSampleAt = at;
this.reportPressure(at);
// A heap that jumps straight past 95 % gets one report, at the tier
// it reached.
const critical = ratio >= HEAP_CRITICAL_RATIO;
if (critical) this.criticalArmed = false;
this.reportPressure(
at,
critical ? HEAP_CRITICAL_RATIO : HEAP_PRESSURE_RATIO,
);
return;
}
if (this.criticalArmed && ratio >= HEAP_CRITICAL_RATIO) {
this.criticalArmed = false;
this.lastSampleAt = at;
this.reportPressure(at, HEAP_CRITICAL_RATIO);
return;
}
@@ -245,7 +266,7 @@ export class HeapMonitor {
}
}
private reportPressure(at: number): void {
private reportPressure(at: number, tier: number): void {
const { registry, emit, readFigures } = this.deps;
const inFlight = registry.report(at, HEAP_PRESSURE_LIST_MAX);
const willSnapshot = this.deps.snapshotEnabled && !this.snapshotTaken;
@@ -253,7 +274,7 @@ export class HeapMonitor {
// process over, the report is already on stdout.
emit(HEAP_TASK_NAMES.pressure, "warn", {
...readFigures(),
pressure_ratio: HEAP_PRESSURE_RATIO,
pressure_ratio: tier,
in_flight_oldest: inFlight.oldest,
in_flight_by_route: inFlight.by_route,
snapshot: willSnapshot ? "pending" : "disabled",
+3 -1
View File
@@ -18,6 +18,7 @@
* to the derived contract cannot leave this document behind.
*/
import { z } from "zod/v4";
import { recoveryEmptySchema } from "./recovery-empty";
import type { ZodOpenApiObject } from "zod-openapi";
import { SCORE_PILLAR_IDS } from "@/lib/analytics/score/types";
@@ -140,11 +141,12 @@ const analyticsSummariesSlice = z
),
lastSeenByType,
sleepSourceDiscrepancy,
recoveryEmpty: recoveryEmptySchema,
})
.meta({
id: "AnalyticsSummariesSlice",
description:
"What `?slice=summaries` answers: the per-type summaries and their freshness, and nothing thick. Four keys, and the caller gets them from two SQL passes rather than the default slice's chain.",
"What `?slice=summaries` answers: the per-type summaries and their freshness, the recovery page's empty reason, and nothing thick. Five keys, and the caller gets them from two SQL passes rather than the default slice's chain.",
});
// ── Correlations ─────────────────────────────────────────────────────
+9 -1
View File
@@ -5,6 +5,7 @@
* Schemas come from `src/lib/validations/*` where shared with the
* runtime request parsing, so the wire contract stays single-source.
*/
import { recoveryEmptySchema } from "../recovery-empty";
import { trendSentiment, trendVerdict } from "../trend-verdict";
import { z } from "zod/v4";
import { measurementSourceEnum, measurementTypeEnum } from "../shared";
@@ -324,7 +325,7 @@ export const sleepScoreValue = z
.enum(["few_nights", "no_main_sleep"])
.optional()
.describe(
"Only on a null `consistency` / `timing`: why it is withheld. `few_nights` — fewer than three nights in the window have a main sleep on the clock (count in `rhythmNights`); `no_main_sleep` — the latest day had only a daytime nap. Absent otherwise.",
"Only on a null `consistency` / `timing`: why it is withheld. `few_nights` — fewer than `rhythmMinNights` nights in the window have a main sleep on the clock (count in `rhythmNights`); `no_main_sleep` — the latest day had only a daytime nap. Absent otherwise.",
),
}),
)
@@ -336,6 +337,12 @@ export const sleepScoreValue = z
.describe(
"Nights in the window with a main sleep placed on the clock — what consistency and timing are computed from. A daytime nap is not one; a short night is.",
),
rhythmMinNights: z
.number()
.int()
.describe(
'How many `rhythmNights` consistency and timing need before they are scored; below it they carry `gap: "few_nights"`.',
),
series: z
.array(z.number())
.describe(
@@ -2032,6 +2039,7 @@ export const analyticsSummariesSliceResponse = z
),
lastSeenByType: z.record(z.string(), lastSeenSlot),
sleepSourceDiscrepancy,
recoveryEmpty: recoveryEmptySchema,
})
.meta({
id: "AnalyticsSummariesSlice",
+3 -3
View File
@@ -465,7 +465,7 @@ const sleepNightResponse = z
.meta({
id: "SleepNightResource",
description:
"One reconstructed sleep night: the main session's hypnogram segments + breakdown, plus same-wake-day naps surfaced separately.",
"One reconstructed sleep night: the main session's hypnogram segments + breakdown, plus same-wake-day naps surfaced separately. A session is a nap when it holds under three hours asleep centred between 10:00 and 20:00 local time; the main session is the longest session that is not a nap. A day whose only sleep is naps has `main: null` and its naps.",
});
// ── Sleep rhythm (sleep-debt + chronotype, v1.17.0) ──────────────────
@@ -804,7 +804,7 @@ export const measurementPaths: NonNullable<ZodOpenApiObject["paths"]> = {
summary: "List measurements",
description:
"Filter by type + date range. Response is paged via offset/limit; default limit 100.\n\n" +
"With `groupBy=day` on a cumulative type the rows are one sum per local day, and `limit` / `offset` page over those days; every returned day is complete. `type=SLEEP_DURATION` returns one row per night, paged the same way over nights. On both, `meta.total` counts the days or nights in the filtered window. A day-sum page takes `limit` as asked (up to the list's 5000); a sleep page holds at most 1000 nights, and `meta.limit` echoes the value applied.",
"With `groupBy=day` on a cumulative type the rows are one sum per local day, and `limit` / `offset` page over those days; every returned day is complete, which is why `valueMin` / `valueMax` (a filter on single readings) are refused with 422 beside `groupBy=day`. `type=SLEEP_DURATION` returns one row per night, paged the same way over nights; every night of the page has a row, and a day whose only sleep is daytime naps carries `value` 0 with its naps in `napCount` / `napAsleepMinutes`. Its `dayKey` is the same day `/api/sleep/night?date=` answers for. On both, `meta.total` counts the days or nights in the filtered window. A day-sum page takes `limit` as asked (up to the list's 5000); a sleep page holds at most 1000 nights, and `meta.limit` echoes the value applied.",
requestParams: {
query: listMeasurementsSchema,
},
@@ -1118,7 +1118,7 @@ export const measurementPaths: NonNullable<ZodOpenApiObject["paths"]> = {
tags: ["Measurements"],
summary: "Reconstructed sleep night (hypnogram source, v1.11.5)",
description:
"Returns one night's reconstructed sleep session for the phase-progression (hypnogram) view: the canonical source's stage segments (each with an absolute start/end), the per-stage breakdown, asleep/in-bed/awake totals, the mid-sleep awakenings count, and same-wake-day naps surfaced separately. Session-clustered, keyed by the local wake day, collapsed to one source via the sleep priority ladder so two sources never overlay. `date` omitted returns the most recent night. A read-only view over existing per-stage SLEEP_DURATION rows — no schema, no new measurement type.",
"Returns one night's reconstructed sleep session for the phase-progression (hypnogram) view: the canonical source's stage segments (each with an absolute start/end), the per-stage breakdown, asleep/in-bed/awake totals, the mid-sleep awakenings count, and same-wake-day naps surfaced separately. Session-clustered, keyed by the local wake day, collapsed to one source via the sleep priority ladder so two sources never overlay. `date` omitted returns the most recent night holding sleep, however old. Any past date is answered, with no window cut. A read-only view over existing per-stage SLEEP_DURATION rows — no schema, no new measurement type.",
requestParams: {
query: z.object({
date: z
+10
View File
@@ -0,0 +1,10 @@
import { z } from "zod/v4";
import { RECOVERY_EMPTY_REASONS } from "@/lib/insights/recovery-empty";
/** Why the recovery page is empty; shared by both slim-slice schemas. */
export const recoveryEmptySchema = z
.enum(RECOVERY_EMPTY_REASONS)
.nullable()
.describe(
"Why the recovery page has nothing to chart, resolved on the server; null when any recovery type (ANS_CHARGE, DAY_STRAIN, WORKOUT_STRAIN, CARDIO_LOAD, AVERAGE_HEART_RATE, MAX_HEART_RATE, ENERGY_EXPENDITURE_KJ, RESILIENCE) has a reading. `oura_permission`: Oura is connected and its grant leaves out the scope the resilience level needs, one reconnect fixes it. `oura_no_resilience`: Oura is connected and no level has arrived yet. `general`: anyone else. Clients render the empty note from this value and never work it out from the connection status.",
);
+6 -1
View File
@@ -14,9 +14,14 @@ import { getHeapStatistics } from "node:v8";
/** Below this, the heap is flagged. Between the 1g (524) and 1536m (792) heaps. */
export const LOW_HEAP_LIMIT_MB = 700;
/** Bytes as whole megabytes (MiB), the unit every heap figure is shown in. */
export function megabytes(bytes: number): number {
return Math.round(bytes / 1024 / 1024);
}
/** V8's heap limit for this process, in megabytes. */
export function heapLimitMb(): number {
return Math.round(getHeapStatistics().heap_size_limit / 1024 / 1024);
return megabytes(getHeapStatistics().heap_size_limit);
}
export function isHeapLimitLow(limitMb: number): boolean {
+4 -3
View File
@@ -25,6 +25,7 @@ import {
heapLimitMb,
isHeapLimitLow,
lowHeapLimitAdvice,
megabytes,
} from "@/lib/support/heap-limit";
import { shouldEmitSecureCookie } from "@/lib/auth/secure-cookie";
import { isOidcConfigured, isOidcOnly } from "@/lib/auth/oidc";
@@ -692,8 +693,8 @@ function or(value: string | number | null | undefined, fallback = "unknown") {
: String(value);
}
function megabytes(bytes: number | null): string {
return bytes === null ? "unknown" : `${Math.round(bytes / 1024 / 1024)} MB`;
function sizeLine(bytes: number | null): string {
return bytes === null ? "unknown" : `${megabytes(bytes)} MB`;
}
function runLine(run: QueueRunFact | null): string {
@@ -741,7 +742,7 @@ export function renderSupportReport(facts: SupportReportFacts): string {
section("Database");
row("PostgreSQL", facts.database.version);
row("Server time zone", facts.database.timezone);
row("Size", megabytes(facts.database.sizeBytes));
row("Size", sizeLine(facts.database.sizeBytes));
row(
"Last migration",
facts.database.lastMigration
+18 -5
View File
@@ -589,7 +589,7 @@ export const listMeasurementsSchema = z
// Omitting `aggregate` keeps the raw wire shape (iOS contract); the
// chart-data client must opt in explicitly.
aggregate: z.enum(["raw", "daily", "weekly", "monthly"]).optional(),
// v1.4.36 W1 — opt-in source switch for daily-aggregate reads. When
// v1.4.36 — opt-in source switch for daily-aggregate reads. When
// `source=rollup` + `aggregate=daily`, the route reads from the
// persistent `measurement_rollups` DAY buckets instead of running a
// live `date_trunc` GROUP BY scan over the raw `measurements` table.
@@ -612,7 +612,7 @@ export const listMeasurementsSchema = z
// `value: { gte, lte }` range. Server-authoritative; validated min<=max.
valueMin: z.coerce.number().finite().optional(),
valueMax: z.coerce.number().finite().optional(),
// v1.4.37 W7c — list-view "one row per day" mode for cumulative
// v1.4.37 — list-view "one row per day" mode for cumulative
// types (steps, active energy, distance, flights, daylight). When
// `groupBy=day` is set and `type` is a cumulative HK type, the route
// returns one synthesised row per user-TZ day with `value` = SUM
@@ -623,18 +623,18 @@ export const listMeasurementsSchema = z
// picks the page's local days in Postgres and reports the distinct-day
// count as `meta.total` (#1245).
groupBy: z.enum(["day"]).optional(),
// v1.4.37 W7c — drill-down to per-sample rows for a single day in
// v1.4.37 — drill-down to per-sample rows for a single day in
// the user's IANA timezone. Format `YYYY-MM-DD`; the route resolves
// the day boundary against the user's `User.timezone`. Used by the
// expandable list row to reveal the chunks that contributed to the
// collapsed daily total.
//
// v1.4.37 W10 — a drill-down returns one bounded page rather than a
// v1.4.37 — a drill-down returns one bounded page rather than a
// cursor, so it refuses a non-zero `offset`.
dayKey: z
.string()
.regex(/^\d{4}-\d{2}-\d{2}$/, "dayKey must be YYYY-MM-DD")
// v1.4.37 W10 — a `YYYY-MM-DD` string can satisfy the regex while
// v1.4.37 — a `YYYY-MM-DD` string can satisfy the regex while
// still being an impossible calendar date (`2026-02-30`,
// `2026-13-01`). `new Date("2026-02-30T00:00:00Z")` silently
// overflows to March 2, so the drill-down would return rows from
@@ -665,6 +665,19 @@ export const listMeasurementsSchema = z
"limit must be <= 1000 when dayKey is set; drill-down responses are capped at 1000 rows",
path: ["limit"],
})
// A value range filters single readings, and a day-grouped row is the sum
// of a whole day: filtering its readings would return partial days summed
// as if they were whole. The combination has no honest answer, so it is
// refused rather than served.
.refine(
({ groupBy, valueMin, valueMax }) =>
groupBy !== "day" || (valueMin == null && valueMax == null),
{
message:
"valueMin / valueMax filter single readings and cannot be combined with groupBy=day",
path: ["groupBy"],
},
)
// v1.18.5 — an inverted value range (`valueMin > valueMax`) is a client
// bug, not an empty result set; reject it at the validator with a 422 so
// the filter UI can surface the mismatch instead of silently returning