mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-11 22:02:16 +00:00
Merge branch 'fix/v1437-review' into release/v1.43.7
# Conflicts: # src/app/api/analytics/route.ts
This commit is contained in:
commit
45c027aee2
217 files changed
+3636
-2127
No files matched your search
@@ -250,8 +250,9 @@ describe("readCoachCorrelations", () => {
|
||||
// Next-day sleep falls as today's burden rises (8h baseline − burden).
|
||||
measurements.push({
|
||||
type: "SLEEP_DURATION",
|
||||
value: 8 - burden,
|
||||
measuredAt: new Date(base + (i + 1) * 86_400_000),
|
||||
// A night of hours waking at 07:00, not a midday minute-long nap.
|
||||
value: (8 - burden) * 60,
|
||||
measuredAt: new Date(base + (i + 1) * 86_400_000 - 5 * 3_600_000),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -346,8 +347,9 @@ describe("readCoachCorrelations", () => {
|
||||
// Higher pressure today → shorter sleep tomorrow (a clean negative r).
|
||||
measurements.push({
|
||||
type: "SLEEP_DURATION",
|
||||
value: 8 - swing * 0.5,
|
||||
measuredAt: new Date(base + (i + 1) * 86_400_000),
|
||||
// A night of hours waking at 07:00, not a midday minute-long nap.
|
||||
value: (8 - swing * 0.5) * 60,
|
||||
measuredAt: new Date(base + (i + 1) * 86_400_000 - 5 * 3_600_000),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ import {
|
||||
reconstructSleepNights,
|
||||
reconstructSleepSessions,
|
||||
pickMainNightAndNaps,
|
||||
isDaytimeNap,
|
||||
separateNightsAndNaps,
|
||||
summarizeSleepNights,
|
||||
type SleepStageRow,
|
||||
} from "@/lib/analytics/sleep-night";
|
||||
@@ -594,6 +596,80 @@ describe("pickMainNightAndNaps", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("the one nap rule", () => {
|
||||
it("tells a nap from a night by length and time of day together", () => {
|
||||
// Short and in the afternoon: a nap.
|
||||
expect(isDaytimeNap(60, 14 * 60)).toBe(true);
|
||||
// Short but at night: the night.
|
||||
expect(isDaytimeNap(150, 2 * 60)).toBe(false);
|
||||
expect(isDaytimeNap(150, 22 * 60)).toBe(false);
|
||||
// A late sleeper's short night, centred at 09:00.
|
||||
expect(isDaytimeNap(150, 9 * 60)).toBe(false);
|
||||
// Long in the daytime (after a night shift): a main sleep.
|
||||
expect(isDaytimeNap(420, 13 * 60)).toBe(false);
|
||||
});
|
||||
|
||||
it("gives a day of naps alone no main night", () => {
|
||||
// 14:00 to 15:00 Berlin, nothing overnight (the watch was charging).
|
||||
const rows: SleepStageRow[] = [row("2026-06-04T13:00:00.000Z", "CORE", 60)];
|
||||
const sessions = reconstructSleepSessions(rows, "Europe/Berlin");
|
||||
const { main, naps } = pickMainNightAndNaps(sessions);
|
||||
expect(main).toBeNull();
|
||||
expect(naps).toHaveLength(1);
|
||||
expect(naps[0].daytimeNap).toBe(true);
|
||||
const nights = reconstructSleepNights(rows, "Europe/Berlin");
|
||||
expect(nights[0].asleepMinutes).toBe(60);
|
||||
expect(nights[0].mainSession).toBeNull();
|
||||
expect(separateNightsAndNaps(sessions)).toEqual([
|
||||
{ night: "2026-06-04", main: null, naps },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps a short night as the main sleep over a longer afternoon nap", () => {
|
||||
const rows: SleepStageRow[] = [
|
||||
// 02:00 to 03:30 Berlin: a broken night, 90 minutes.
|
||||
row("2026-06-04T01:30:00.000Z", "CORE", 90),
|
||||
// 13:00 to 15:00 Berlin: a two-hour nap.
|
||||
row("2026-06-04T13:00:00.000Z", "CORE", 120),
|
||||
];
|
||||
const sessions = reconstructSleepSessions(rows, "Europe/Berlin");
|
||||
const { main, naps } = pickMainNightAndNaps(sessions);
|
||||
expect(main?.asleepMinutes).toBe(90);
|
||||
expect(naps.map((n) => n.asleepMinutes)).toEqual([120]);
|
||||
// The merged night places the same session on the clock.
|
||||
const [night] = reconstructSleepNights(rows, "Europe/Berlin");
|
||||
expect(night.mainSession?.asleepMinutes).toBe(90);
|
||||
expect(night.mainSession?.end.getTime()).toBe(main?.end.getTime());
|
||||
});
|
||||
|
||||
it("files a session under the day its last row ends on, over every writer", () => {
|
||||
// The watch stops at 23:50 UTC; the phone's in-bed span runs to 00:10.
|
||||
const rows: SleepStageRow[] = [
|
||||
srcRow("2026-06-03T23:50:00.000Z", "CORE", 170, "WHOOP"),
|
||||
srcRow("2026-06-04T00:10:00.000Z", "IN_BED", 200, "APPLE_HEALTH"),
|
||||
];
|
||||
const sessions = reconstructSleepSessions(rows, "UTC");
|
||||
const nights = reconstructSleepNights(rows, "UTC");
|
||||
expect(sessions.map((s) => s.night)).toEqual(["2026-06-04"]);
|
||||
expect(nights.map((n) => n.night)).toEqual(["2026-06-04"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a zero-minute granular row", () => {
|
||||
it("does not strip the bare aggregate that carries the night", () => {
|
||||
// A writer's bare 6-hour night beside a REM row of no length: the REM
|
||||
// row partitions nothing, so the night is the aggregate's six hours.
|
||||
const rows: SleepStageRow[] = [
|
||||
row("2026-06-04T06:00:00.000Z", "ASLEEP", 360),
|
||||
row("2026-06-04T06:00:00.000Z", "REM", 0),
|
||||
];
|
||||
const [session] = reconstructSleepSessions(rows, "UTC");
|
||||
expect(session.asleepMinutes).toBe(360);
|
||||
expect(session.stages.ASLEEP).toBe(360);
|
||||
expect(reconstructSleepNights(rows, "UTC")[0].asleepMinutes).toBe(360);
|
||||
});
|
||||
});
|
||||
|
||||
describe("summarizeSleepNights", () => {
|
||||
it("summarises per-night totals, not per-stage rows", () => {
|
||||
const rows: SleepStageRow[] = [
|
||||
|
||||
@@ -373,6 +373,38 @@ describe("reference-score pillars", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("holds regularity back until 21 nights carry a midpoint, not 21 nights", () => {
|
||||
const nights = days(24, 480).map((point, index) => ({
|
||||
night: point.day,
|
||||
asleepMinutes: point.value,
|
||||
// Four nap-only days count toward the level but have no place on the
|
||||
// clock, which leaves 20 midpoints.
|
||||
midpoint: index % 6 === 0 ? null : 180,
|
||||
}));
|
||||
const result = computeSleepPillar({
|
||||
...live,
|
||||
asOf: NOW,
|
||||
ageYears: 40,
|
||||
nights,
|
||||
sources: ["APPLE_HEALTH"],
|
||||
});
|
||||
expect(result.status).toBe("ok");
|
||||
if (result.status === "ok") {
|
||||
expect(result.value.observed.label).not.toContain("regularity");
|
||||
}
|
||||
const enough = computeSleepPillar({
|
||||
...live,
|
||||
asOf: NOW,
|
||||
ageYears: 40,
|
||||
nights: nights.map((n) => ({ ...n, midpoint: 180 })),
|
||||
sources: ["APPLE_HEALTH"],
|
||||
});
|
||||
if (enough.status === "ok") {
|
||||
expect(enough.value.observed.label).toContain("regularity");
|
||||
}
|
||||
expect(enough.status).toBe("ok");
|
||||
});
|
||||
|
||||
it("scores exactly 28 sleep calendar keys", () => {
|
||||
const nights = days(29, 600).map((point) => ({
|
||||
night: point.day,
|
||||
|
||||
@@ -62,8 +62,11 @@ export function computeSleepPillar(
|
||||
const midpoints = nights
|
||||
.map((night) => night.midpoint)
|
||||
.filter((midpoint): midpoint is number => midpoint != null);
|
||||
// The floor is on the nights that have a place on the clock: a day whose
|
||||
// only sleep was a nap counts toward the level but carries no midpoint, so
|
||||
// 21 nights could hold far fewer midpoints than the floor asks for.
|
||||
const regularity =
|
||||
nights.length >= SLEEP_REGULARITY_MIN_NIGHTS
|
||||
midpoints.length >= SLEEP_REGULARITY_MIN_NIGHTS
|
||||
? scoreConsistency(midpoints)
|
||||
: null;
|
||||
// These are the existing Sleep Score's own declared sub-score weights,
|
||||
|
||||
@@ -48,7 +48,7 @@
|
||||
* measured on the absolute UTC instant, so it is immune to DST shifts.
|
||||
*
|
||||
* Each session is keyed by the LOCAL WAKE DAY — `userDayKey` of the session's
|
||||
* LAST segment (its end instant) in the user's timezone. This matches the
|
||||
* LAST row end over every writer, in the user's timezone. This matches the
|
||||
* Apple Health convention (a sleep session is attributed to the date you wake
|
||||
* up) and keeps the keying tz/DST-correct because it runs on the real instant.
|
||||
*
|
||||
@@ -58,9 +58,12 @@
|
||||
* day under one key so the headline "time asleep" for a day is the day's
|
||||
* total. `reconstructSleepSessions` keeps each session SEPARATE, and
|
||||
* `pickMainNightAndNaps` applies the convention the UI needs: the MAIN night
|
||||
* is the session with the most asleep minutes (normally the overnight block),
|
||||
* and every other session on the same wake day is a NAP, surfaced separately
|
||||
* and never folded into the main night's headline.
|
||||
* is the main sleep with the most asleep minutes (normally the overnight
|
||||
* block), and every other session on the same wake day is a NAP, surfaced
|
||||
* separately and never folded into the main night's headline. A session is
|
||||
* a nap rather than a main sleep when it is short and centred in the daytime
|
||||
* (`isDaytimeNap`), so a day with an afternoon nap and no recorded night has
|
||||
* no main night at all, rather than the nap standing in for one.
|
||||
*
|
||||
* Writer de-dup
|
||||
* -------------
|
||||
@@ -127,6 +130,7 @@
|
||||
*/
|
||||
import type { MeasurementSource, SleepStage } from "@/generated/prisma/client";
|
||||
import { userDayKey } from "@/lib/tz/format";
|
||||
import { wallClockInTz } from "@/lib/tz/wall-clock";
|
||||
import { summarize, type DataSummary } from "@/lib/analytics/trends";
|
||||
import {
|
||||
getDeviceTypeLadder,
|
||||
@@ -151,6 +155,17 @@ const GRANULAR_ASLEEP_STAGES: ReadonlySet<SleepStage> = new Set<SleepStage>([
|
||||
"DEEP",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Every stage that counts as asleep: the granular partition plus the bare
|
||||
* `ASLEEP` aggregate. A stage-less row counts as asleep too, but it has no
|
||||
* stage to list. The sleep list's SQL session cut splices this list, so the
|
||||
* two decide "does this session hold sleep?" from one definition.
|
||||
*/
|
||||
export const ASLEEP_STAGES: readonly SleepStage[] = [
|
||||
"ASLEEP",
|
||||
...GRANULAR_ASLEEP_STAGES,
|
||||
];
|
||||
|
||||
/**
|
||||
* True when any row in the set carries a GRANULAR asleep stage
|
||||
* (CORE / DEEP / REM). When this holds, the bare `ASLEEP` aggregate (and any
|
||||
@@ -162,7 +177,10 @@ const GRANULAR_ASLEEP_STAGES: ReadonlySet<SleepStage> = new Set<SleepStage>([
|
||||
function sawGranularStage(rows: readonly SleepStageRow[]): boolean {
|
||||
for (const r of rows) {
|
||||
const stage = r.sleepStage;
|
||||
if (stage != null && GRANULAR_ASLEEP_STAGES.has(stage)) return true;
|
||||
// A zero-minute granular row partitions nothing: it must not strip the
|
||||
// bare aggregate that does carry the night.
|
||||
if (stage != null && GRANULAR_ASLEEP_STAGES.has(stage) && r.value > 0)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -199,7 +217,7 @@ function asleepMinutesOf(rows: readonly SleepStageRow[]): number {
|
||||
const stage = seg.stage;
|
||||
if (stage != null && GRANULAR_ASLEEP_STAGES.has(stage)) {
|
||||
granular += seg.minutes;
|
||||
sawGranular = true;
|
||||
if (seg.minutes > 0) sawGranular = true;
|
||||
} else if (stage === "ASLEEP" || stage == null) {
|
||||
// Bare unspecified aggregate (`ASLEEP`) or a stage-less legacy/manual
|
||||
// row — only used when no granular stage carries the night.
|
||||
@@ -258,6 +276,70 @@ function inBedEnvelopeMinutes(rows: readonly SleepStageRow[]): number | null {
|
||||
*/
|
||||
export const SESSION_GAP_MS = 3 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Below this much asleep time a session can be a nap. Three hours is where
|
||||
* wearables draw the line between a nap and a main sleep (Oura types a
|
||||
* period `long_sleep` from three hours on).
|
||||
*/
|
||||
export const MAIN_SLEEP_MIN_MINUTES = 180;
|
||||
|
||||
/**
|
||||
* The night window, in local minutes-of-day: a session whose midpoint falls
|
||||
* from 20:00 to 10:00 is night sleep however short it is. Wide enough for a
|
||||
* late chronotype (asleep 04:00 to 12:00 has its midpoint at 08:00) and for
|
||||
* an early one, and clear of the afternoon where naps sit.
|
||||
*/
|
||||
export const NIGHT_WINDOW_START_MINUTES = 20 * 60;
|
||||
export const NIGHT_WINDOW_END_MINUTES = 10 * 60;
|
||||
|
||||
/**
|
||||
* True for a session that is a nap rather than a main sleep: short (under
|
||||
* `MAIN_SLEEP_MIN_MINUTES` asleep) AND centred in the daytime. A short night
|
||||
* is still the night (insomnia, a newborn, split sleep); a long daytime sleep
|
||||
* (after a night shift) is still a main sleep. The one nap rule: the nap
|
||||
* convention, the sleep score's midpoint and every list read it. Pure.
|
||||
*/
|
||||
export function isDaytimeNap(
|
||||
asleepMinutes: number,
|
||||
midpointMinutes: number,
|
||||
): boolean {
|
||||
if (asleepMinutes >= MAIN_SLEEP_MIN_MINUTES) return false;
|
||||
const atNight =
|
||||
midpointMinutes >= NIGHT_WINDOW_START_MINUTES ||
|
||||
midpointMinutes < NIGHT_WINDOW_END_MINUTES;
|
||||
return !atNight;
|
||||
}
|
||||
|
||||
/**
|
||||
* The centre of a session's asleep span as local minutes-of-day: its wake
|
||||
* instant minus half its asleep minutes, read on the sleeper's wall clock.
|
||||
*/
|
||||
export function asleepMidpointMinutes(
|
||||
end: Date,
|
||||
asleepMinutes: number,
|
||||
tz: string,
|
||||
): number {
|
||||
const mid = new Date(end.getTime() - (asleepMinutes / 2) * 60_000);
|
||||
if (tz === "UTC") return mid.getUTCHours() * 60 + mid.getUTCMinutes();
|
||||
const { hour, minute } = wallClockInTz(mid, tz);
|
||||
return hour * 60 + minute;
|
||||
}
|
||||
|
||||
/**
|
||||
* The wake day a session is filed under: the local day its last row ends on,
|
||||
* over every writer in the session. One rule for every reader, the SQL page
|
||||
* cut of the sleep list included, which sees every writer and cannot know the
|
||||
* winning one. Filing by the winner alone named a different day whenever a
|
||||
* losing writer's in-bed span ran past midnight after the winner stopped, and
|
||||
* the list and the night view then disagreed about which day the night was.
|
||||
*/
|
||||
function wakeDayOf(rows: readonly SleepStageRow[], tz: string): string {
|
||||
let end = rows[0].measuredAt;
|
||||
for (const r of rows)
|
||||
if (r.measuredAt.getTime() > end.getTime()) end = r.measuredAt;
|
||||
return userDayKey(end, tz);
|
||||
}
|
||||
|
||||
export interface SleepStageRow {
|
||||
value: number;
|
||||
measuredAt: Date;
|
||||
@@ -271,12 +353,12 @@ export interface SleepStageRow {
|
||||
*/
|
||||
source?: MeasurementSource | null;
|
||||
/**
|
||||
* Device-type tag of the stage row (`Measurement.deviceType`, v1.4.25
|
||||
* W8c). Distinguishes WRITERS behind one source: several HealthKit apps
|
||||
* Device-type tag of the stage row (`Measurement.deviceType`, v1.4.25).
|
||||
* Distinguishes WRITERS behind one source: several HealthKit apps
|
||||
* write sleep under `source = APPLE_HEALTH` (watch granular stages, phone
|
||||
* coarse in-bed, vendor-app re-exports), and collapsing on source alone
|
||||
* blends them. Optional — callers that don't select the column (and every
|
||||
* row stored before W8c) collapse per source exactly as before.
|
||||
* row stored before v1.4.25) collapse per source exactly as before.
|
||||
*/
|
||||
deviceType?: string | null;
|
||||
}
|
||||
@@ -309,9 +391,9 @@ export interface SleepNight {
|
||||
stages: Partial<Record<SleepStage, number>>;
|
||||
/**
|
||||
* Non-null when two writer buckets reported clearly different asleep
|
||||
* totals for the night's MAIN session — the same-wake-day session with
|
||||
* totals for the night's MAIN session — the same-wake-day main sleep with
|
||||
* the most asleep minutes, normally the overnight block (see
|
||||
* `sessionSourceDiscrepancy`). Sessions never cross-compare: a nap's
|
||||
* `pickMainNightAndNaps` and `sessionSourceDiscrepancy`). Sessions never cross-compare: a nap's
|
||||
* writers are not measured against the overnight's. Observational only —
|
||||
* the served `asleepMinutes` stays the winning writer's total; the
|
||||
* annotation lets the UI mark the number with a discreet
|
||||
@@ -319,12 +401,13 @@ export interface SleepNight {
|
||||
*/
|
||||
sourceDiscrepancy: SleepSourceDiscrepancy | null;
|
||||
/**
|
||||
* The night's MAIN session alone: the same-wake-day session with the most
|
||||
* asleep minutes (the one `sourceDiscrepancy` reads), its wake instant and
|
||||
* its own asleep minutes. A nap on the same wake day is in the night's
|
||||
* totals but not here, so a reader that places the night on the clock (the
|
||||
* sleep midpoint) is not dragged into the afternoon by it. Null when no
|
||||
* session carries asleep minutes.
|
||||
* The night's MAIN session alone, chosen by the nap convention
|
||||
* (`pickMainNightAndNaps`: the main sleep with the most asleep minutes, the
|
||||
* one `sourceDiscrepancy` reads), its wake instant and its own asleep
|
||||
* minutes. A nap on the same wake day is in the night's totals but not
|
||||
* here, so a reader that places the night on the clock (the sleep midpoint)
|
||||
* is not dragged into the afternoon by it. Null when no session carries
|
||||
* asleep minutes, and on a day whose only sleep is daytime naps.
|
||||
*/
|
||||
mainSession: { end: Date; asleepMinutes: number } | null;
|
||||
}
|
||||
@@ -643,13 +726,9 @@ export function reconstructSleepNights(
|
||||
const canonical = pickSessionWriter(session, sleepLadder, deviceLadder);
|
||||
const kept = session.filter((r) => writerKeyOf(r) === canonical);
|
||||
const pool = kept.length > 0 ? kept : session;
|
||||
// The wake day is the LATEST segment END in the session (Apple Health
|
||||
// attributes a session to the morning you wake up).
|
||||
const wakeInstant = pool.reduce(
|
||||
(max, r) => (r.measuredAt.getTime() > max.getTime() ? r.measuredAt : max),
|
||||
pool[0].measuredAt,
|
||||
);
|
||||
const key = userDayKey(wakeInstant, tz);
|
||||
// The wake day is the session's latest row end (Apple Health attributes
|
||||
// a session to the morning you wake up), over every writer.
|
||||
const key = wakeDayOf(session, tz);
|
||||
const list = byNight.get(key) ?? [];
|
||||
list.push({ pool, all: session });
|
||||
byNight.set(key, list);
|
||||
@@ -671,9 +750,9 @@ export function reconstructSleepNights(
|
||||
// edits.
|
||||
let latest = nightSessions[0].pool[0].measuredAt;
|
||||
const stages: Partial<Record<SleepStage, number>> = {};
|
||||
// MAIN session of the night = most asleep minutes (same per-session
|
||||
// asleep total the headline sums; tie → later wake instant, matching
|
||||
// `pickMainNightAndNaps`). Its FULL raw row set (every writer,
|
||||
// MAIN session of the night = the main sleep (not a daytime nap) with
|
||||
// the most asleep minutes (same per-session asleep total the headline
|
||||
// sums; tie → later wake instant), exactly as `pickMainNightAndNaps`. Its FULL raw row set (every writer,
|
||||
// pre-collapse) feeds the night's source-discrepancy annotation —
|
||||
// per session, so a nap's writers never cross-compare against the
|
||||
// overnight block's.
|
||||
@@ -731,6 +810,10 @@ export function reconstructSleepNights(
|
||||
}
|
||||
if (
|
||||
sessionAsleep > 0 &&
|
||||
!isDaytimeNap(
|
||||
sessionAsleep,
|
||||
asleepMidpointMinutes(new Date(sessionEnd), sessionAsleep, tz),
|
||||
) &&
|
||||
(sessionAsleep > mainSessionAsleep ||
|
||||
(sessionAsleep === mainSessionAsleep && sessionEnd > mainSessionEnd))
|
||||
) {
|
||||
@@ -844,6 +927,12 @@ export interface SleepSession {
|
||||
* "sources disagree" hint.
|
||||
*/
|
||||
sourceDiscrepancy: SleepSourceDiscrepancy | null;
|
||||
/**
|
||||
* `true` when the session is a daytime nap rather than a main sleep
|
||||
* (`isDaytimeNap`). Decided here, once, so the nap convention, the sleep
|
||||
* score and every list read the same answer.
|
||||
*/
|
||||
daytimeNap: boolean;
|
||||
}
|
||||
|
||||
/** Resolve a stage row to its absolute span (start = end − duration). */
|
||||
@@ -1079,7 +1168,7 @@ export function reconstructSleepSessions(
|
||||
const asleep = asleepMinutesOf(pool);
|
||||
const canonicalSource = sourceOfWriterKey(canonical);
|
||||
sessions.push({
|
||||
night: userDayKey(latest, tz),
|
||||
night: wakeDayOf(session, tz),
|
||||
source:
|
||||
canonicalSource === NO_SOURCE
|
||||
? null
|
||||
@@ -1101,21 +1190,28 @@ export function reconstructSleepSessions(
|
||||
// losing bucket's disagreement is visible; the served totals above
|
||||
// stay winner-only.
|
||||
sourceDiscrepancy: sessionSourceDiscrepancy(session),
|
||||
daytimeNap: isDaytimeNap(
|
||||
asleep,
|
||||
asleepMidpointMinutes(latest, asleep, tz),
|
||||
),
|
||||
});
|
||||
}
|
||||
return sessions.sort((a, b) => a.start.getTime() - b.start.getTime());
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the NAP convention to a wake-day's sessions: the MAIN night is the
|
||||
* session with the most asleep minutes (normally the overnight block); every
|
||||
* other session on the same wake day is a nap, surfaced separately and never
|
||||
* folded into the main night's headline.
|
||||
* Apply the NAP convention to a wake-day's sessions. The MAIN night is the
|
||||
* main-sleep session (see `isDaytimeNap`) with the most asleep minutes,
|
||||
* normally the overnight block; every other scorable session on the same wake
|
||||
* day is a nap, surfaced separately and never folded into the main night's
|
||||
* headline. A day whose only sleep is daytime naps has no main night: `main`
|
||||
* is null and the naps are all there is.
|
||||
*
|
||||
* Returns the main session plus the naps (sorted by start). When the input
|
||||
* holds sessions across multiple wake days, only the sessions matching the
|
||||
* main session's wake day are considered naps — call once per wake day, or
|
||||
* pass a single wake day's sessions.
|
||||
* main session's wake day (or, without one, the latest session's) are
|
||||
* considered naps — call once per wake day, or pass a single wake day's
|
||||
* sessions.
|
||||
*/
|
||||
export function pickMainNightAndNaps(sessions: readonly SleepSession[]): {
|
||||
main: SleepSession | null;
|
||||
@@ -1123,14 +1219,22 @@ export function pickMainNightAndNaps(sessions: readonly SleepSession[]): {
|
||||
} {
|
||||
const scorable = sessions.filter((s) => s.asleepMinutes > 0);
|
||||
if (scorable.length === 0) return { main: null, naps: [] };
|
||||
// Main = most asleep minutes; tie-break on the later end (overnight wins).
|
||||
const main = [...scorable].sort((a, b) => {
|
||||
if (b.asleepMinutes !== a.asleepMinutes)
|
||||
return b.asleepMinutes - a.asleepMinutes;
|
||||
return b.end.getTime() - a.end.getTime();
|
||||
})[0];
|
||||
// Main = most asleep minutes among the main sleeps; tie-break on the later
|
||||
// end (overnight wins).
|
||||
const main =
|
||||
scorable
|
||||
.filter((s) => !s.daytimeNap)
|
||||
.sort((a, b) => {
|
||||
if (b.asleepMinutes !== a.asleepMinutes)
|
||||
return b.asleepMinutes - a.asleepMinutes;
|
||||
return b.end.getTime() - a.end.getTime();
|
||||
})[0] ?? null;
|
||||
const day =
|
||||
main?.night ??
|
||||
scorable.reduce((a, b) => (b.end.getTime() > a.end.getTime() ? b : a))
|
||||
.night;
|
||||
const naps = scorable
|
||||
.filter((s) => s !== main && s.night === main.night)
|
||||
.filter((s) => s !== main && s.night === day)
|
||||
.sort((a, b) => a.start.getTime() - b.start.getTime());
|
||||
return { main, naps };
|
||||
}
|
||||
@@ -1139,8 +1243,11 @@ export function pickMainNightAndNaps(sessions: readonly SleepSession[]): {
|
||||
export interface SleepDaySplit {
|
||||
/** Wake-day key (YYYY-MM-DD) in the user's timezone. */
|
||||
night: string;
|
||||
/** The overnight block — the session with the most asleep minutes. */
|
||||
main: SleepSession;
|
||||
/**
|
||||
* The main sleep (see `pickMainNightAndNaps`), or null on a day whose only
|
||||
* sleep is daytime naps.
|
||||
*/
|
||||
main: SleepSession | null;
|
||||
/** Every other scorable session on the same wake day, sorted by start. */
|
||||
naps: SleepSession[];
|
||||
}
|
||||
@@ -1154,8 +1261,8 @@ export interface SleepDaySplit {
|
||||
* than growing its own idea of a night. There is exactly one nap classifier in
|
||||
* this codebase and it is `pickMainNightAndNaps`; this is its multi-day form.
|
||||
*
|
||||
* Days whose sessions carry no asleep minutes are dropped (no main night to
|
||||
* report). The result is sorted ascending by wake day.
|
||||
* Days whose sessions carry no asleep minutes are dropped; a day of naps
|
||||
* alone is kept with `main` null. The result is sorted ascending by wake day.
|
||||
*/
|
||||
export function separateNightsAndNaps(
|
||||
sessions: readonly SleepSession[],
|
||||
@@ -1169,7 +1276,7 @@ export function separateNightsAndNaps(
|
||||
const days: SleepDaySplit[] = [];
|
||||
for (const [night, daySessions] of byDay) {
|
||||
const { main, naps } = pickMainNightAndNaps(daySessions);
|
||||
if (!main) continue;
|
||||
if (!main && naps.length === 0) continue;
|
||||
days.push({ night, main, naps });
|
||||
}
|
||||
return days.sort((a, b) => a.night.localeCompare(b.night));
|
||||
|
||||
@@ -73,6 +73,8 @@ export function buildSleepStageComposition(
|
||||
const perNight: SleepStageCompositionNight[] = [];
|
||||
|
||||
for (const { night, main, naps } of days) {
|
||||
// A day of naps alone has no night to compose.
|
||||
if (!main) continue;
|
||||
const nightStages: Record<string, number> = {};
|
||||
for (const [stage, mins] of Object.entries(main.stages)) {
|
||||
if (mins == null) continue;
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { ApiError } from "./api-fetch";
|
||||
|
||||
/**
|
||||
* What a failed read means for the surface that made it.
|
||||
*
|
||||
* - `unauthenticated`: 401. The session has ended. The surface says so and
|
||||
* offers the way back to sign-in; drawing nothing would leave a page that
|
||||
* looks empty for a reason the reader cannot see.
|
||||
* - `refused`: 403. The viewer may not read this, typically a shared or
|
||||
* delegated record whose grant leaves the section out. Nothing failed that
|
||||
* a retry could fix, so the surface draws its refused state, or nothing.
|
||||
* - `throttled`: 429. The read budget is spent for now; an immediate retry
|
||||
* would only be refused again.
|
||||
* - `failed`: everything else, a real read failure worth a retry.
|
||||
*/
|
||||
export type ReadFailureKind =
|
||||
"unauthenticated" | "refused" | "throttled" | "failed";
|
||||
|
||||
export function readFailureKind(error: unknown): ReadFailureKind {
|
||||
if (error instanceof ApiError) {
|
||||
if (error.status === 401) return "unauthenticated";
|
||||
if (error.status === 403) return "refused";
|
||||
if (error.status === 429) return "throttled";
|
||||
}
|
||||
return "failed";
|
||||
}
|
||||
|
||||
/** The sign-in page, returning to where the reader is now. */
|
||||
export function signInHref(): string {
|
||||
if (typeof window === "undefined") return "/auth/login";
|
||||
const here = `${window.location.pathname}${window.location.search}`;
|
||||
return `/auth/login?next=${encodeURIComponent(here)}`;
|
||||
}
|
||||
@@ -53,7 +53,10 @@ describe("stripImageMetadata — passthrough", () => {
|
||||
describe("stripImageMetadata — JPEG", () => {
|
||||
// SOI + APP0(JFIF) + APP1(Exif w/ GPS) + APP2(ICC) + COM + DQT + SOS + scan.
|
||||
const soi = Buffer.from([0xff, 0xd8]);
|
||||
const app0 = jpegSegment(0xe0, Buffer.from("JFIF\0\0\0\0\0\0"));
|
||||
const app0 = jpegSegment(
|
||||
0xe0,
|
||||
Buffer.from("JFIF\0\x01\x01\0\0\x01\0\x01\0\0"),
|
||||
);
|
||||
const exifPayload = Buffer.concat([
|
||||
Buffer.from("Exif\0\0"),
|
||||
// A recognisable GPS marker string we can assert is gone.
|
||||
|
||||
@@ -31,8 +31,8 @@ import {
|
||||
MAXMIND_DOWNLOAD_BASE,
|
||||
} from "@/lib/geo/geolite2-fetch";
|
||||
|
||||
const CITY_BYTES = Buffer.from("CITY-MMDB-CONTENT-");
|
||||
const ASN_BYTES = Buffer.from("ASN-MMDB-CONTENT-");
|
||||
const CITY_BYTES = Buffer.from("CITY-MMDB-CONTENT-\x01\x02\x03");
|
||||
const ASN_BYTES = Buffer.from("ASN-MMDB-CONTENT-\x04\x05\x06");
|
||||
|
||||
/** Build a MaxMind-shaped tar.gz: `<Edition>_<date>/<Edition>.mmdb`. */
|
||||
function buildEditionTarball(
|
||||
|
||||
@@ -9,9 +9,10 @@
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { prismaMock, safeFetchMock } = vi.hoisted(() => ({
|
||||
const { prismaMock, safeFetchMock, annotateMock } = vi.hoisted(() => ({
|
||||
prismaMock: { googleHealthConnection: { update: vi.fn() } },
|
||||
safeFetchMock: vi.fn(),
|
||||
annotateMock: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/db", () => ({ prisma: prismaMock }));
|
||||
vi.mock("@/lib/safe-fetch", async (importOriginal) => ({
|
||||
@@ -19,7 +20,7 @@ vi.mock("@/lib/safe-fetch", async (importOriginal) => ({
|
||||
safeFetch: safeFetchMock,
|
||||
}));
|
||||
vi.mock("@/lib/logging/context", () => ({
|
||||
annotate: () => {},
|
||||
annotate: annotateMock,
|
||||
getEvent: () => null,
|
||||
}));
|
||||
|
||||
@@ -32,14 +33,16 @@ import {
|
||||
type GoogleHealthBackfillEnd,
|
||||
} from "../backfill-bounds";
|
||||
import {
|
||||
googleHealthBackfillEndedList,
|
||||
googleHealthBackfillProgressAfterCompletion,
|
||||
googleHealthBackfillRetryPending,
|
||||
readGoogleHealthBackfillEnded,
|
||||
recordGoogleHealthBackfillEnd,
|
||||
runGoogleHealthCollection,
|
||||
runWithGoogleHealthBackfillResume,
|
||||
} from "../sync-core";
|
||||
import {
|
||||
googleHealthBackfillEndedList,
|
||||
googleHealthBackfillProgressAfterCompletion,
|
||||
googleHealthBackfillRetryPending,
|
||||
readGoogleHealthBackfillProgress,
|
||||
} from "../backfill-progress";
|
||||
import {
|
||||
forEachDataPointPage,
|
||||
GOOGLE_HEALTH_DATA_TYPES,
|
||||
@@ -312,13 +315,31 @@ describe("a collection ended short", () => {
|
||||
-1,
|
||||
)![0].data.backfillProgress as unknown;
|
||||
expect(
|
||||
readGoogleHealthBackfillEnded(written).get("fetchHeartRate"),
|
||||
readGoogleHealthBackfillProgress(written).ended.get("fetchHeartRate"),
|
||||
).toEqual(note);
|
||||
// Waiting for another try is not where the history ends.
|
||||
expect(googleHealthBackfillEndedList(written)).toEqual([]);
|
||||
expect(googleHealthBackfillProgressAfterCompletion(ended)).toBeNull();
|
||||
});
|
||||
|
||||
it("annotates an end in one fixed shape", async () => {
|
||||
await slice(stalls, new Map([["fetchWeight", HORIZON_END]]));
|
||||
const notes = annotateMock.mock.calls
|
||||
.map(([fields]) => fields.meta?.["googleHealth.backfill.ended"])
|
||||
.filter(Boolean);
|
||||
expect(notes).toEqual([
|
||||
{
|
||||
collection: "fetchHeartRate",
|
||||
type: "PULSE",
|
||||
reason: "no_progress",
|
||||
final: false,
|
||||
endedCount: 2,
|
||||
finalCount: 1,
|
||||
types: ["PULSE", "WEIGHT"],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("is not walked again before its day is up", async () => {
|
||||
const walk = vi.fn(async () => 1);
|
||||
const { skipped } = await slice(
|
||||
@@ -351,6 +372,31 @@ describe("a collection ended short", () => {
|
||||
).toMatchObject({ ...END, attempts: 2 });
|
||||
});
|
||||
|
||||
it("starts the count over when the retry got further back before it stalled again", async () => {
|
||||
const further = { ...END, at: "2023-01-02T00:00:00.000Z" };
|
||||
const done = new Set<string>();
|
||||
const { ended, collections } = await slice(
|
||||
async () => {
|
||||
noteGoogleHealthDeferred();
|
||||
await recordGoogleHealthBackfillEnd("fetchHeartRate", further);
|
||||
return 4;
|
||||
},
|
||||
new Map<string, GoogleHealthBackfillEnd>([
|
||||
["fetchHeartRate", parked(1, 25)],
|
||||
]),
|
||||
done,
|
||||
);
|
||||
// Progress, then a new stall: parked again, not final.
|
||||
expect(ended.get("fetchHeartRate")).toMatchObject({
|
||||
...further,
|
||||
attempts: 1,
|
||||
});
|
||||
expect(done.has("fetchHeartRate")).toBe(false);
|
||||
expect(collections).toEqual([
|
||||
expect.objectContaining({ key: "fetchHeartRate", status: "stopped" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("goes back to an ordinary walk when the second attempt gets somewhere", async () => {
|
||||
const { ended, collections } = await slice(
|
||||
async () => {
|
||||
|
||||
@@ -20,12 +20,14 @@ vi.mock("@/lib/logging/context", () => ({
|
||||
}));
|
||||
|
||||
import {
|
||||
readGoogleHealthBackfillCursors,
|
||||
recordGoogleHealthBackfillPage,
|
||||
runGoogleHealthCollection,
|
||||
runWithGoogleHealthBackfillResume,
|
||||
type GoogleHealthBackfillCursor,
|
||||
} from "../sync-core";
|
||||
import {
|
||||
readGoogleHealthBackfillProgress,
|
||||
type GoogleHealthBackfillCursor,
|
||||
} from "../backfill-progress";
|
||||
import {
|
||||
googleHealthSliceBudget,
|
||||
runWithGoogleHealthClientOutcome,
|
||||
@@ -124,7 +126,7 @@ describe("the resume edge of a page-by-page collection", () => {
|
||||
});
|
||||
// And it reads back the way it was written.
|
||||
expect(
|
||||
readGoogleHealthBackfillCursors(last.data.backfillProgress).get(
|
||||
readGoogleHealthBackfillProgress(last.data.backfillProgress).cursors.get(
|
||||
"fetchHeartRate",
|
||||
),
|
||||
).toEqual(cursors.get("fetchHeartRate"));
|
||||
|
||||
@@ -46,13 +46,14 @@ import { GOOGLE_HEALTH_NO_PROGRESS_PAGES } from "../backfill-bounds";
|
||||
import {
|
||||
googleHealthBackfillCursor,
|
||||
noteGoogleHealthEdgeFilterUnsupported,
|
||||
readGoogleHealthBackfillCursors,
|
||||
readGoogleHealthBackfillEdgeFilterUnsupported,
|
||||
recordGoogleHealthBackfillPage,
|
||||
recordGoogleHealthBackfillResumePoint,
|
||||
runWithGoogleHealthBackfillResume,
|
||||
type GoogleHealthBackfillCursor,
|
||||
} from "../sync-core";
|
||||
import {
|
||||
readGoogleHealthBackfillProgress,
|
||||
type GoogleHealthBackfillCursor,
|
||||
} from "../backfill-progress";
|
||||
|
||||
const MINUTE = 60_000;
|
||||
const NOW = Date.parse("2026-01-01T00:00:00Z");
|
||||
@@ -566,7 +567,8 @@ describe("the note that a collection refuses the edge filter", () => {
|
||||
|
||||
// Slice two: starts from what slice one left on disk.
|
||||
server.requests = [];
|
||||
const unsupported = readGoogleHealthBackfillEdgeFilterUnsupported(progress);
|
||||
const unsupported =
|
||||
readGoogleHealthBackfillProgress(progress).edgeFilterUnsupported;
|
||||
await runWithGoogleHealthBackfillResume(
|
||||
"u1",
|
||||
new Set(),
|
||||
@@ -576,7 +578,7 @@ describe("the note that a collection refuses the edge filter", () => {
|
||||
const { rows } = await walk({ resumable: true, edge: cursor.edge });
|
||||
expect(rows).toHaveLength(TOTAL - 19);
|
||||
},
|
||||
readGoogleHealthBackfillCursors(progress),
|
||||
readGoogleHealthBackfillProgress(progress).cursors,
|
||||
unsupported,
|
||||
);
|
||||
expect(server.requests.some((f) => f.includes("<="))).toBe(false);
|
||||
@@ -616,7 +618,7 @@ describe("the note that a collection refuses the edge filter", () => {
|
||||
page: 4,
|
||||
});
|
||||
expect(first.advanced.has("fetchHeartRate")).toBe(true);
|
||||
const saved = readGoogleHealthBackfillCursors(lastProgress());
|
||||
const saved = readGoogleHealthBackfillProgress(lastProgress()).cursors;
|
||||
expect(saved.get("fetchHeartRate")).toEqual({
|
||||
...cursor,
|
||||
resume: { token: "4", page: 4, reached: 4 },
|
||||
@@ -628,8 +630,9 @@ describe("the note that a collection refuses the edge filter", () => {
|
||||
const back = await run(new Map(saved), { token: "2", page: 2 });
|
||||
expect(back.advanced.size).toBe(0);
|
||||
expect(
|
||||
readGoogleHealthBackfillCursors(lastProgress()).get("fetchHeartRate")
|
||||
?.resume,
|
||||
readGoogleHealthBackfillProgress(lastProgress()).cursors.get(
|
||||
"fetchHeartRate",
|
||||
)?.resume,
|
||||
).toEqual({ token: "2", page: 2, reached: 4 });
|
||||
|
||||
// A page written keeps the place; the next slice is handed it.
|
||||
@@ -644,7 +647,7 @@ describe("the note that a collection refuses the edge filter", () => {
|
||||
new Map(saved),
|
||||
new Set(["fetchHeartRate"]),
|
||||
);
|
||||
const next = readGoogleHealthBackfillCursors(lastProgress());
|
||||
const next = readGoogleHealthBackfillProgress(lastProgress()).cursors;
|
||||
expect(next.get("fetchHeartRate")?.resume).toEqual(
|
||||
saved.get("fetchHeartRate")?.resume,
|
||||
);
|
||||
@@ -663,10 +666,12 @@ describe("the note that a collection refuses the edge filter", () => {
|
||||
|
||||
it("reads only well-formed keys", () => {
|
||||
expect([
|
||||
...readGoogleHealthBackfillEdgeFilterUnsupported({
|
||||
...readGoogleHealthBackfillProgress({
|
||||
edgeFilterUnsupported: ["fetchHeartRate", 3, "x".repeat(65)],
|
||||
}),
|
||||
}).edgeFilterUnsupported,
|
||||
]).toEqual(["fetchHeartRate"]);
|
||||
expect(readGoogleHealthBackfillEdgeFilterUnsupported(null).size).toBe(0);
|
||||
expect(
|
||||
readGoogleHealthBackfillProgress(null).edgeFilterUnsupported.size,
|
||||
).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* The `backfillProgress` column of a Google Health connection: how far a
|
||||
* full-history backfill has got, carried from one attempt to the next, and
|
||||
* where a data type's history ended short of its beginning.
|
||||
*
|
||||
* One schema reads the column and one function writes it. The column is
|
||||
* JSON the code itself wrote, but it outlives releases, so the read is
|
||||
* lenient per entry: a malformed cursor or end is dropped on its own and the
|
||||
* rest of the note still counts. A collection key longer than 64 characters
|
||||
* is never one of ours and is dropped as well.
|
||||
*/
|
||||
import { z } from "zod";
|
||||
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
import { MeasurementType as MeasurementTypes } from "@/generated/prisma/enums";
|
||||
import {
|
||||
googleHealthBackfillEndIsFinal,
|
||||
googleHealthBackfillEndRetryAt,
|
||||
type GoogleHealthBackfillEnd,
|
||||
} from "./backfill-bounds";
|
||||
|
||||
/**
|
||||
* How far a full-history backfill has got with one collection.
|
||||
*
|
||||
* Two grains. A collection walked to the end is noted in `done`, and the next
|
||||
* attempt skips it. A collection read page by page (the spot readings: heart
|
||||
* rate, weight, glucose, ...) also notes, after every page it writes, the
|
||||
* edge it has reached: the oldest sample written so far when the walk runs
|
||||
* newest first, as Google's list does. The next attempt asks Google only for
|
||||
* the samples at or beyond that edge, so a collection that does not fit into
|
||||
* one attempt is worked through over several instead of being read from its
|
||||
* first page every time.
|
||||
*
|
||||
* The edge is a sample time, not Google's page token: page tokens are not
|
||||
* documented to outlive the walk that received them, a sample time is a
|
||||
* filter Google documents. The one sample on the edge itself is read twice
|
||||
* and overwrites its own row.
|
||||
*
|
||||
* Before the edge existed only `done` was kept, and a heart-rate history of
|
||||
* years that one attempt could not finish (its budget, a rate limit, one
|
||||
* failed page among thousands) started again from the newest page on every
|
||||
* attempt and never reached the end (#1220).
|
||||
*
|
||||
* The upper bound (`<=`) a newest-first walk resumes with is valid filter
|
||||
* grammar but has not been seen working against the live API. A collection
|
||||
* whose resumed request Google refuses with a 400 is noted in
|
||||
* `edgeFilterUnsupported`, and its walks from then on read without the bound
|
||||
* and pass over the samples beyond the edge themselves (see
|
||||
* `forEachDataPointPage`).
|
||||
*/
|
||||
export interface GoogleHealthBackfillCursor {
|
||||
/** The edge: the sample instant (ISO 8601) the walk has written up to. */
|
||||
at: string;
|
||||
/** The order the walk returned samples in: newest first, or oldest first. */
|
||||
order: "desc" | "asc";
|
||||
/**
|
||||
* Pages this collection has written across every attempt so far. Kept for
|
||||
* the record only: it bounds nothing. It once capped the walk at ten
|
||||
* thousand pages, which a dense heart-rate history passes in under three
|
||||
* years, and since it never reset the backfill stopped there for good
|
||||
* (#1220). The walk is bounded by its progress and the backfill horizon
|
||||
* instead (`backfill-bounds.ts`).
|
||||
*/
|
||||
pages: number;
|
||||
/**
|
||||
* Only for a collection read without the edge bound: where its last walk
|
||||
* stopped in the unbounded list (see `GoogleHealthResumePoint`), and the
|
||||
* furthest position any walk of it has saved. A saved place counts as the
|
||||
* slice's progress only when it is further than every one before it, so a
|
||||
* chain of slices that keep passing over the same pages ends.
|
||||
*/
|
||||
resume?: { token: string; page: number; reached: number };
|
||||
}
|
||||
|
||||
/** Everything the column holds, read. */
|
||||
export interface GoogleHealthBackfillProgress {
|
||||
/** Collections walked to the end. */
|
||||
done: Set<string>;
|
||||
/** Per-collection edges. */
|
||||
cursors: Map<string, GoogleHealthBackfillCursor>;
|
||||
/** Collections Google refused the edge filter on. */
|
||||
edgeFilterUnsupported: Set<string>;
|
||||
/**
|
||||
* Collections whose backfill ended short of the start of their history
|
||||
* (the horizon, or a walk that stopped getting anywhere), each with where
|
||||
* it ended and why. Survives the completion stamp, so the integration card
|
||||
* can say where history ends.
|
||||
*/
|
||||
ended: Map<string, GoogleHealthBackfillEnd>;
|
||||
}
|
||||
|
||||
const MAX_KEY_LENGTH = 64;
|
||||
|
||||
/** An instant Date can read, normalised to ISO 8601. */
|
||||
const instant = z
|
||||
.string()
|
||||
.refine((s) => !Number.isNaN(Date.parse(s)))
|
||||
.transform((s) => new Date(s).toISOString());
|
||||
|
||||
/** A whole, positive count; anything else reads as zero. */
|
||||
const count = z
|
||||
.unknown()
|
||||
.transform((v) =>
|
||||
typeof v === "number" && Number.isFinite(v) && v > 0 ? Math.trunc(v) : 0,
|
||||
);
|
||||
|
||||
const resumeSchema = z
|
||||
.object({ token: z.string().min(1).max(4096), page: count, reached: count })
|
||||
.transform((r) =>
|
||||
r.page > 0
|
||||
? { token: r.token, page: r.page, reached: Math.max(r.reached, r.page) }
|
||||
: undefined,
|
||||
);
|
||||
|
||||
const cursorSchema = z
|
||||
.object({
|
||||
at: instant,
|
||||
order: z.unknown().transform((v) => (v === "asc" ? "asc" : "desc")),
|
||||
pages: count,
|
||||
resume: resumeSchema.optional().catch(undefined),
|
||||
})
|
||||
.transform(({ resume, ...c }): GoogleHealthBackfillCursor =>
|
||||
resume ? { ...c, resume } : c,
|
||||
);
|
||||
|
||||
const endSchema = z
|
||||
.object({
|
||||
type: z.enum(MeasurementTypes),
|
||||
at: instant,
|
||||
reason: z.enum(["horizon", "no_progress"]),
|
||||
attempts: z.number().int().min(1).optional().catch(undefined),
|
||||
endedAt: instant.optional().catch(undefined),
|
||||
})
|
||||
.transform(({ attempts, endedAt, ...e }): GoogleHealthBackfillEnd => ({
|
||||
...e,
|
||||
type: e.type as MeasurementType,
|
||||
...(attempts !== undefined ? { attempts } : {}),
|
||||
...(endedAt !== undefined ? { endedAt } : {}),
|
||||
}));
|
||||
|
||||
const record = z.record(z.string(), z.unknown()).catch({});
|
||||
|
||||
const keyList = z
|
||||
.array(z.unknown())
|
||||
.catch([])
|
||||
.transform(
|
||||
(keys) =>
|
||||
new Set(
|
||||
keys.filter(
|
||||
(key): key is string =>
|
||||
typeof key === "string" && key.length <= MAX_KEY_LENGTH,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
/** Entries of a keyed record that parse, by their own schema, one by one. */
|
||||
function keyedEntries<T>(value: unknown, schema: z.ZodType<T>): Map<string, T> {
|
||||
const out = new Map<string, T>();
|
||||
for (const [key, raw] of Object.entries(record.parse(value))) {
|
||||
if (key.length > MAX_KEY_LENGTH) continue;
|
||||
const parsed = schema.safeParse(raw);
|
||||
if (parsed.success) out.set(key, parsed.data);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Read the `backfillProgress` column. */
|
||||
export function readGoogleHealthBackfillProgress(
|
||||
value: unknown,
|
||||
): GoogleHealthBackfillProgress {
|
||||
const column = record.parse(value);
|
||||
return {
|
||||
done: keyList.parse(column.done),
|
||||
cursors: keyedEntries(column.cursors, cursorSchema),
|
||||
edgeFilterUnsupported: keyList.parse(column.edgeFilterUnsupported),
|
||||
ended: keyedEntries(column.ended, endSchema),
|
||||
};
|
||||
}
|
||||
|
||||
/** The ended collections as the column stores them. */
|
||||
function endedJson(ended: Map<string, GoogleHealthBackfillEnd>) {
|
||||
return Object.fromEntries(
|
||||
[...ended].map(([key, e]) => [
|
||||
key,
|
||||
{
|
||||
type: e.type,
|
||||
at: e.at,
|
||||
reason: e.reason,
|
||||
...(e.attempts !== undefined ? { attempts: e.attempts } : {}),
|
||||
...(e.endedAt !== undefined ? { endedAt: e.endedAt } : {}),
|
||||
},
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The column's value for a backfill in progress. The two optional parts are
|
||||
* written only once there is something to say, so a backfill that never met
|
||||
* them keeps the shape it always had.
|
||||
*/
|
||||
export function googleHealthBackfillProgressJson(
|
||||
progress: GoogleHealthBackfillProgress,
|
||||
) {
|
||||
const { done, cursors, edgeFilterUnsupported, ended } = progress;
|
||||
return {
|
||||
done: [...done],
|
||||
cursors: Object.fromEntries(
|
||||
[...cursors].map(([key, c]) => [
|
||||
key,
|
||||
{
|
||||
at: c.at,
|
||||
order: c.order,
|
||||
pages: c.pages,
|
||||
...(c.resume ? { resume: c.resume } : {}),
|
||||
},
|
||||
]),
|
||||
),
|
||||
...(edgeFilterUnsupported.size > 0
|
||||
? { edgeFilterUnsupported: [...edgeFilterUnsupported] }
|
||||
: {}),
|
||||
...(ended.size > 0 ? { ended: endedJson(ended) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The column's value to keep once the backfill is stamped complete: only
|
||||
* where collections ended short for good, or nothing at all.
|
||||
*/
|
||||
export function googleHealthBackfillProgressAfterCompletion(
|
||||
ended: Map<string, GoogleHealthBackfillEnd>,
|
||||
): { ended: ReturnType<typeof endedJson> } | null {
|
||||
const final = new Map(
|
||||
[...ended].filter(([, e]) => googleHealthBackfillEndIsFinal(e)),
|
||||
);
|
||||
return final.size > 0 ? { ended: endedJson(final) } : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Where each data type's imported history ends short of its beginning, and
|
||||
* why, as the status routes publish it (`backfillEnded`), by type.
|
||||
*/
|
||||
export interface GoogleHealthBackfillEndedEntry {
|
||||
type: MeasurementType;
|
||||
/** The oldest instant the imported history reaches back to. */
|
||||
endsAt: string;
|
||||
reason: GoogleHealthBackfillEnd["reason"];
|
||||
}
|
||||
|
||||
/**
|
||||
* The column's ended collections, as published: only the final ends. One
|
||||
* still waiting for another attempt is not where its history ends yet.
|
||||
*/
|
||||
export function googleHealthBackfillEndedList(
|
||||
value: unknown,
|
||||
): GoogleHealthBackfillEndedEntry[] {
|
||||
return [...readGoogleHealthBackfillProgress(value).ended.values()]
|
||||
.filter(googleHealthBackfillEndIsFinal)
|
||||
.map((e) => ({ type: e.type, endsAt: e.at, reason: e.reason }))
|
||||
.sort((a, b) => a.type.localeCompare(b.type));
|
||||
}
|
||||
|
||||
/**
|
||||
* The collections waiting for another attempt after one ended without
|
||||
* progress, and the earliest instant (epoch ms) one of them is due, or null
|
||||
* when none waits.
|
||||
*/
|
||||
export function googleHealthBackfillRetryPending(
|
||||
ended: Map<string, GoogleHealthBackfillEnd>,
|
||||
): { keys: string[]; dueAt: number } | null {
|
||||
const keys: string[] = [];
|
||||
let dueAt = Infinity;
|
||||
for (const [key, e] of ended) {
|
||||
const at = googleHealthBackfillEndRetryAt(e);
|
||||
if (at === null) continue;
|
||||
keys.push(key);
|
||||
dueAt = Math.min(dueAt, at);
|
||||
}
|
||||
return keys.length > 0 ? { keys, dueAt } : null;
|
||||
}
|
||||
@@ -18,7 +18,6 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { prisma } from "@/lib/db";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
import { MeasurementType as MeasurementTypes } from "@/generated/prisma/enums";
|
||||
import { encrypt, decrypt } from "@/lib/crypto";
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
import type { GoogleHealthCollectionKey } from "./collection-keys";
|
||||
@@ -27,6 +26,10 @@ import {
|
||||
googleHealthBackfillEndRetryAt,
|
||||
type GoogleHealthBackfillEnd,
|
||||
} from "./backfill-bounds";
|
||||
import {
|
||||
googleHealthBackfillProgressJson,
|
||||
type GoogleHealthBackfillCursor,
|
||||
} from "./backfill-progress";
|
||||
import { userDayKey } from "@/lib/tz/format";
|
||||
import { isP2002 } from "@/lib/prisma-errors";
|
||||
import {
|
||||
@@ -411,60 +414,6 @@ export async function runWithGoogleHealthSyncCycle<T>(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* How far a full-history backfill has got, carried from one attempt of the
|
||||
* backfill to the next.
|
||||
*
|
||||
* Two grains. A collection walked to the end is noted in `done`, and the next
|
||||
* attempt skips it. A collection read page by page (the spot readings: heart
|
||||
* rate, weight, glucose, ...) also notes, after every page it writes, the
|
||||
* edge it has reached: the oldest sample written so far when the walk runs
|
||||
* newest first, as Google's list does. The next attempt asks Google only for
|
||||
* the samples at or beyond that edge, so a collection that does not fit into
|
||||
* one attempt is worked through over several instead of being read from its
|
||||
* first page every time.
|
||||
*
|
||||
* The edge is a sample time, not Google's page token: page tokens are not
|
||||
* documented to outlive the walk that received them, a sample time is a
|
||||
* filter Google documents. The one sample on the edge itself is read twice
|
||||
* and overwrites its own row.
|
||||
*
|
||||
* Before the edge existed only `done` was kept, and a heart-rate history of
|
||||
* years that one attempt could not finish (its budget, a rate limit, one
|
||||
* failed page among thousands) started again from the newest page on every
|
||||
* attempt and never reached the end (#1220).
|
||||
*
|
||||
* The upper bound (`<=`) a newest-first walk resumes with is valid filter
|
||||
* grammar but has not been seen working against the live API. A collection
|
||||
* whose resumed request Google refuses with a 400 is noted in
|
||||
* `edgeFilterUnsupported`, and its walks from then on read without the bound
|
||||
* and pass over the samples beyond the edge themselves (see
|
||||
* `forEachDataPointPage`).
|
||||
*/
|
||||
export interface GoogleHealthBackfillCursor {
|
||||
/** The edge: the sample instant (ISO 8601) the walk has written up to. */
|
||||
at: string;
|
||||
/** The order the walk returned samples in: newest first, or oldest first. */
|
||||
order: "desc" | "asc";
|
||||
/**
|
||||
* Pages this collection has written across every attempt so far. Kept for
|
||||
* the record only: it bounds nothing. It once capped the walk at ten
|
||||
* thousand pages, which a dense heart-rate history passes in under three
|
||||
* years, and since it never reset the backfill stopped there for good
|
||||
* (#1220). The walk is bounded by its progress and the backfill horizon
|
||||
* instead (`backfill-bounds.ts`).
|
||||
*/
|
||||
pages: number;
|
||||
/**
|
||||
* Only for a collection read without the edge bound: where its last walk
|
||||
* stopped in the unbounded list (see `GoogleHealthResumePoint`), and the
|
||||
* furthest position any walk of it has saved. A saved place counts as the
|
||||
* slice's progress only when it is further than every one before it, so a
|
||||
* chain of slices that keep passing over the same pages ends.
|
||||
*/
|
||||
resume?: { token: string; page: number; reached: number };
|
||||
}
|
||||
|
||||
/** What one collection came to in one backfill attempt. */
|
||||
export interface GoogleHealthBackfillCollectionReport {
|
||||
key: string;
|
||||
@@ -497,209 +446,13 @@ interface BackfillResumeTracker {
|
||||
ended: Map<string, GoogleHealthBackfillEnd>;
|
||||
/**
|
||||
* Collections walked again this attempt after an earlier one ended without
|
||||
* progress, with how many attempts had ended that way.
|
||||
* progress, with how many attempts in a row had ended that way and the edge
|
||||
* they ended at.
|
||||
*/
|
||||
retrying: Map<string, number>;
|
||||
retrying: Map<string, { attempts: number; at: string }>;
|
||||
}
|
||||
const backfillResumeStorage = new AsyncLocalStorage<BackfillResumeTracker>();
|
||||
|
||||
function progressRecord(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
/** Read the `backfillProgress` column into the set of finished collections. */
|
||||
export function readGoogleHealthBackfillDone(value: unknown): Set<string> {
|
||||
const done = progressRecord(value).done;
|
||||
return new Set(
|
||||
Array.isArray(done)
|
||||
? done.filter(
|
||||
(key): key is string => typeof key === "string" && key.length <= 64,
|
||||
)
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
/** Read the `backfillProgress` column into the per-collection edges. */
|
||||
export function readGoogleHealthBackfillCursors(
|
||||
value: unknown,
|
||||
): Map<string, GoogleHealthBackfillCursor> {
|
||||
const out = new Map<string, GoogleHealthBackfillCursor>();
|
||||
const cursors = progressRecord(progressRecord(value).cursors);
|
||||
for (const [key, raw] of Object.entries(cursors)) {
|
||||
if (key.length > 64) continue;
|
||||
const c = progressRecord(raw);
|
||||
const at = typeof c.at === "string" ? c.at : null;
|
||||
if (!at || Number.isNaN(Date.parse(at))) continue;
|
||||
const r = progressRecord(c.resume);
|
||||
const count = (v: unknown): number =>
|
||||
typeof v === "number" && Number.isFinite(v) && v > 0 ? Math.trunc(v) : 0;
|
||||
const resume =
|
||||
typeof r.token === "string" &&
|
||||
r.token.length > 0 &&
|
||||
r.token.length <= 4096 &&
|
||||
count(r.page) > 0
|
||||
? {
|
||||
token: r.token,
|
||||
page: count(r.page),
|
||||
reached: Math.max(count(r.reached), count(r.page)),
|
||||
}
|
||||
: undefined;
|
||||
out.set(key, {
|
||||
at: new Date(at).toISOString(),
|
||||
order: c.order === "asc" ? "asc" : "desc",
|
||||
pages: count(c.pages),
|
||||
...(resume ? { resume } : {}),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the `backfillProgress` column into the collections whose backfill
|
||||
* ended short of the start of their history (the horizon, or a walk that
|
||||
* stopped getting anywhere), each with where it ended and why. Survives the
|
||||
* completion stamp, so the integration card can say where history ends.
|
||||
*/
|
||||
export function readGoogleHealthBackfillEnded(
|
||||
value: unknown,
|
||||
): Map<string, GoogleHealthBackfillEnd> {
|
||||
const out = new Map<string, GoogleHealthBackfillEnd>();
|
||||
const ended = progressRecord(progressRecord(value).ended);
|
||||
for (const [key, raw] of Object.entries(ended)) {
|
||||
if (key.length > 64) continue;
|
||||
const e = progressRecord(raw);
|
||||
const at = typeof e.at === "string" ? e.at : null;
|
||||
if (!at || Number.isNaN(Date.parse(at))) continue;
|
||||
if (e.reason !== "horizon" && e.reason !== "no_progress") continue;
|
||||
if (typeof e.type !== "string" || !(e.type in MeasurementTypes)) {
|
||||
continue;
|
||||
}
|
||||
const attempts =
|
||||
typeof e.attempts === "number" &&
|
||||
Number.isInteger(e.attempts) &&
|
||||
e.attempts >= 1
|
||||
? e.attempts
|
||||
: undefined;
|
||||
const endedAt =
|
||||
typeof e.endedAt === "string" && !Number.isNaN(Date.parse(e.endedAt))
|
||||
? new Date(e.endedAt).toISOString()
|
||||
: undefined;
|
||||
out.set(key, {
|
||||
type: e.type as MeasurementType,
|
||||
at: new Date(at).toISOString(),
|
||||
reason: e.reason,
|
||||
...(attempts !== undefined ? { attempts } : {}),
|
||||
...(endedAt !== undefined ? { endedAt } : {}),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Where each data type's imported history ends short of its beginning, and
|
||||
* why, as the status routes publish it (`backfillEnded`), by type.
|
||||
*/
|
||||
export interface GoogleHealthBackfillEndedEntry {
|
||||
type: MeasurementType;
|
||||
/** The oldest instant the imported history reaches back to. */
|
||||
endsAt: string;
|
||||
reason: GoogleHealthBackfillEnd["reason"];
|
||||
}
|
||||
|
||||
/**
|
||||
* The `backfillProgress` column's ended collections, as published: only the
|
||||
* final ends. One still waiting for another attempt is not where its history
|
||||
* ends yet.
|
||||
*/
|
||||
export function googleHealthBackfillEndedList(
|
||||
value: unknown,
|
||||
): GoogleHealthBackfillEndedEntry[] {
|
||||
return [...readGoogleHealthBackfillEnded(value).values()]
|
||||
.filter(googleHealthBackfillEndIsFinal)
|
||||
.map((e) => ({ type: e.type, endsAt: e.at, reason: e.reason }))
|
||||
.sort((a, b) => a.type.localeCompare(b.type));
|
||||
}
|
||||
|
||||
/**
|
||||
* The `backfillProgress` value to keep once the backfill is stamped complete:
|
||||
* only where collections ended short, or nothing at all.
|
||||
*/
|
||||
export function googleHealthBackfillProgressAfterCompletion(
|
||||
ended: Map<string, GoogleHealthBackfillEnd>,
|
||||
): {
|
||||
ended: Record<string, GoogleHealthBackfillEndJson>;
|
||||
} | null {
|
||||
const final = new Map(
|
||||
[...ended].filter(([, e]) => googleHealthBackfillEndIsFinal(e)),
|
||||
);
|
||||
return final.size > 0 ? { ended: endedJson(final) } : null;
|
||||
}
|
||||
|
||||
/** One ended collection as the column stores it. */
|
||||
type GoogleHealthBackfillEndJson = {
|
||||
type: string;
|
||||
at: string;
|
||||
reason: string;
|
||||
attempts?: number;
|
||||
endedAt?: string;
|
||||
};
|
||||
|
||||
/** The ended collections as the column stores them. */
|
||||
function endedJson(
|
||||
ended: Map<string, GoogleHealthBackfillEnd>,
|
||||
): Record<string, GoogleHealthBackfillEndJson> {
|
||||
return Object.fromEntries(
|
||||
[...ended].map(([key, e]) => [
|
||||
key,
|
||||
{
|
||||
type: e.type,
|
||||
at: e.at,
|
||||
reason: e.reason,
|
||||
...(e.attempts !== undefined ? { attempts: e.attempts } : {}),
|
||||
...(e.endedAt !== undefined ? { endedAt: e.endedAt } : {}),
|
||||
},
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The collections waiting for another attempt after one ended without
|
||||
* progress, and the earliest instant (epoch ms) one of them is due, or null
|
||||
* when none waits.
|
||||
*/
|
||||
export function googleHealthBackfillRetryPending(
|
||||
ended: Map<string, GoogleHealthBackfillEnd>,
|
||||
): { keys: string[]; dueAt: number } | null {
|
||||
const keys: string[] = [];
|
||||
let dueAt = Infinity;
|
||||
for (const [key, e] of ended) {
|
||||
const at = googleHealthBackfillEndRetryAt(e);
|
||||
if (at === null) continue;
|
||||
keys.push(key);
|
||||
dueAt = Math.min(dueAt, at);
|
||||
}
|
||||
return keys.length > 0 ? { keys, dueAt } : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the `backfillProgress` column into the collections Google refused the
|
||||
* edge filter on.
|
||||
*/
|
||||
export function readGoogleHealthBackfillEdgeFilterUnsupported(
|
||||
value: unknown,
|
||||
): Set<string> {
|
||||
const keys = progressRecord(value).edgeFilterUnsupported;
|
||||
return new Set(
|
||||
Array.isArray(keys)
|
||||
? keys.filter(
|
||||
(key): key is string => typeof key === "string" && key.length <= 64,
|
||||
)
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the tracker's progress with one change applied, and apply it to the
|
||||
* tracker only once the write has landed: progress that is not on disk is
|
||||
@@ -720,26 +473,12 @@ async function persistBackfillProgress(
|
||||
await prisma.googleHealthConnection.update({
|
||||
where: { userId: tracker.userId },
|
||||
data: {
|
||||
backfillProgress: {
|
||||
done: [...next.done],
|
||||
cursors: Object.fromEntries(
|
||||
[...next.cursors].map(([key, c]) => [
|
||||
key,
|
||||
{
|
||||
at: c.at,
|
||||
order: c.order,
|
||||
pages: c.pages,
|
||||
...(c.resume ? { resume: c.resume } : {}),
|
||||
},
|
||||
]),
|
||||
),
|
||||
// Only once there is something to say, so a backfill that never met
|
||||
// the refusal keeps the shape it always had.
|
||||
...(unsupported.size > 0
|
||||
? { edgeFilterUnsupported: [...unsupported] }
|
||||
: {}),
|
||||
...(ended.size > 0 ? { ended: endedJson(ended) } : {}),
|
||||
},
|
||||
backfillProgress: googleHealthBackfillProgressJson({
|
||||
done: next.done,
|
||||
cursors: next.cursors,
|
||||
edgeFilterUnsupported: unsupported,
|
||||
ended,
|
||||
}),
|
||||
},
|
||||
});
|
||||
if (unsupported !== tracker.edgeFilterUnsupported) {
|
||||
@@ -857,20 +596,40 @@ export async function recordGoogleHealthBackfillEnd(
|
||||
const resume = backfillResumeStorage.getStore();
|
||||
if (!resume) return;
|
||||
const ended = new Map(resume.ended);
|
||||
// Counted across attempts that end at the same edge: a walk that got
|
||||
// further back before it stalled again made progress, and its count starts
|
||||
// over. Only attempts in a row that reach nowhere make the end final.
|
||||
const parked = resume.retrying.get(key);
|
||||
const sameEdge =
|
||||
parked !== undefined && Date.parse(parked.at) === Date.parse(end.at);
|
||||
ended.set(
|
||||
key,
|
||||
end.reason === "no_progress"
|
||||
? {
|
||||
...end,
|
||||
// Counted across attempts: the one walked again this attempt
|
||||
// carries the count it was parked with.
|
||||
attempts: (resume.retrying.get(key) ?? 0) + 1,
|
||||
attempts: sameEdge ? parked.attempts + 1 : 1,
|
||||
endedAt: new Date().toISOString(),
|
||||
}
|
||||
: end,
|
||||
);
|
||||
// A fixed shape, not the whole note: the slice report carries every end in
|
||||
// full (`googleHealth.backfill.slice`). This names the one that just ended
|
||||
// and counts the rest. It is meta, not the event's action, because the job
|
||||
// names the action once the slice is over and that would replace it.
|
||||
const recorded = ended.get(key)!;
|
||||
const final = [...ended.values()].filter(googleHealthBackfillEndIsFinal);
|
||||
annotate({
|
||||
meta: { "googleHealth.backfill.ended": Object.fromEntries(ended) },
|
||||
meta: {
|
||||
"googleHealth.backfill.ended": {
|
||||
collection: key,
|
||||
type: recorded.type,
|
||||
reason: recorded.reason,
|
||||
final: googleHealthBackfillEndIsFinal(recorded),
|
||||
endedCount: ended.size,
|
||||
finalCount: final.length,
|
||||
types: [...new Set([...ended.values()].map((e) => e.type))].sort(),
|
||||
},
|
||||
},
|
||||
});
|
||||
try {
|
||||
await persistBackfillProgress(resume, {
|
||||
@@ -1073,8 +832,12 @@ export async function runGoogleHealthCollection(
|
||||
if (parked) {
|
||||
// Its earlier attempt ended without progress and its wait is over: walk
|
||||
// it again from its edge. The note goes with the next write; the count
|
||||
// stays with the attempt, so a second end without progress is final.
|
||||
resume.retrying.set(key, parked.attempts ?? 1);
|
||||
// and the edge stay with the attempt, so a second end at the same edge
|
||||
// is final.
|
||||
resume.retrying.set(key, {
|
||||
attempts: parked.attempts ?? 1,
|
||||
at: parked.at,
|
||||
});
|
||||
resume.ended.delete(key);
|
||||
}
|
||||
const startedAt = performance.now();
|
||||
|
||||
@@ -110,6 +110,38 @@ function stubCompletion(
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Postgres names the nights the sleep read reconstructs; answer the night
|
||||
* page query with the one night the stage rows make, and every other raw
|
||||
* query with nothing.
|
||||
*/
|
||||
function nightPageOver(rows: Array<{ value: number; measuredAt: Date }>) {
|
||||
const start = Math.min(
|
||||
...rows.map((r) => r.measuredAt.getTime() - r.value * 60_000),
|
||||
);
|
||||
const end = Math.max(...rows.map((r) => r.measuredAt.getTime()));
|
||||
vi.mocked(prisma.$queryRaw).mockImplementation(((
|
||||
_strings: TemplateStringsArray,
|
||||
...values: unknown[]
|
||||
) =>
|
||||
Promise.resolve(
|
||||
values.some((v) =>
|
||||
String((v as { sql?: string })?.sql ?? "").includes("sess AS"),
|
||||
)
|
||||
? [
|
||||
{
|
||||
night: new Date(end).toLocaleDateString("sv-SE", {
|
||||
timeZone: "Europe/Berlin",
|
||||
}),
|
||||
span_start: new Date(start),
|
||||
span_end: new Date(end),
|
||||
total: 1,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
)) as never);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(prisma.insightStatusCache.findUnique).mockResolvedValue(null);
|
||||
@@ -167,6 +199,7 @@ describe("generateMetricStatus — SLEEP_DURATION night reconstruction (iOS E2)"
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue(
|
||||
stageRows as never,
|
||||
);
|
||||
nightPageOver(stageRows);
|
||||
|
||||
const capture = { systemPrompt: null, userPrompt: null } as {
|
||||
systemPrompt: string | null;
|
||||
@@ -190,7 +223,7 @@ describe("generateMetricStatus — SLEEP_DURATION night reconstruction (iOS E2)"
|
||||
expect(prompt).not.toContain('"value": 90');
|
||||
});
|
||||
|
||||
it("the snapshot graded series is the deduped night total, never the ~20 h raw-stage sum (A4)", async () => {
|
||||
it("the snapshot graded series is the deduped night total, never the ~20 h raw-stage sum", async () => {
|
||||
// The ~20.3 h symptom: a source writes BOTH a bare ASLEEP aggregate AND the
|
||||
// granular CORE/DEEP/REM partition for the same span. Pre-fix the graded
|
||||
// `series` came from `buildGradedSeriesWithRollups`, which folds bare + each
|
||||
@@ -249,6 +282,7 @@ describe("generateMetricStatus — SLEEP_DURATION night reconstruction (iOS E2)"
|
||||
vi.mocked(prisma.measurement.findMany).mockResolvedValue(
|
||||
stageRows as never,
|
||||
);
|
||||
nightPageOver(stageRows);
|
||||
|
||||
const capture = { systemPrompt: null, userPrompt: null } as {
|
||||
systemPrompt: string | null;
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { resolveRecoveryEmpty } from "../recovery-empty";
|
||||
|
||||
/**
|
||||
* The recovery page's empty reason is decided here, on the server, from the
|
||||
* summaries and the Oura connection; clients render the answer.
|
||||
*/
|
||||
describe("resolveRecoveryEmpty", () => {
|
||||
const none = { connected: false, scopeDenied: [] };
|
||||
|
||||
it("has no reason when any recovery type has a reading", () => {
|
||||
expect(resolveRecoveryEmpty({ RESILIENCE: { count: 1 } }, none)).toBeNull();
|
||||
expect(resolveRecoveryEmpty({ DAY_STRAIN: { count: 3 } }, none)).toBeNull();
|
||||
});
|
||||
|
||||
it("does not count types the page does not chart", () => {
|
||||
expect(resolveRecoveryEmpty({ RECOVERY_SCORE: { count: 30 } }, none)).toBe(
|
||||
"general",
|
||||
);
|
||||
});
|
||||
|
||||
it("asks for a reconnect only when the grant leaves resilience out", () => {
|
||||
expect(
|
||||
resolveRecoveryEmpty(
|
||||
{},
|
||||
{ connected: true, scopeDenied: ["resilience"] },
|
||||
),
|
||||
).toBe("oura_permission");
|
||||
expect(
|
||||
resolveRecoveryEmpty({}, { connected: true, scopeDenied: ["spo2"] }),
|
||||
).toBe("oura_no_resilience");
|
||||
});
|
||||
});
|
||||
@@ -113,10 +113,12 @@ function seedSymptomDrivesSleep() {
|
||||
functionalImpact: burden,
|
||||
symptomLinks: [],
|
||||
});
|
||||
// A night of five to eight hours, waking at 07:00 UTC: a main sleep, not
|
||||
// a midday nap, so the nap rule leaves it the night.
|
||||
measurements.push({
|
||||
type: "SLEEP_DURATION",
|
||||
value: 8 - burden,
|
||||
measuredAt: new Date(BASE + (i + 1) * 86_400_000),
|
||||
value: (8 - burden) * 60,
|
||||
measuredAt: new Date(BASE + (i + 1) * 86_400_000 - 5 * 3_600_000),
|
||||
});
|
||||
}
|
||||
measurementFindMany.mockResolvedValue(measurements);
|
||||
|
||||
@@ -76,6 +76,7 @@ const sleepScore: SleepScoreValue = {
|
||||
],
|
||||
windowNights: 20,
|
||||
rhythmNights: 20,
|
||||
rhythmMinNights: 3,
|
||||
series: [78, 82],
|
||||
};
|
||||
|
||||
|
||||
@@ -24,7 +24,6 @@ import { mapSleep, type OuraSleep } from "@/lib/oura/client";
|
||||
import {
|
||||
computeSleepScore,
|
||||
reconstructNights,
|
||||
isDaytimeNap,
|
||||
rhythmGap,
|
||||
scoreConsistency,
|
||||
} from "../sleep-score";
|
||||
@@ -174,18 +173,6 @@ describe("Sleep Score rhythm on an Oura month", () => {
|
||||
expect(sub("timing").gap).toBeUndefined();
|
||||
});
|
||||
|
||||
it("tells a nap from a night by length and time of day together", () => {
|
||||
// Short and in the afternoon: a nap.
|
||||
expect(isDaytimeNap(60, 14 * 60)).toBe(true);
|
||||
// Short but at night: the night.
|
||||
expect(isDaytimeNap(150, 2 * 60)).toBe(false);
|
||||
expect(isDaytimeNap(150, 22 * 60)).toBe(false);
|
||||
// A late sleeper's short night, centred at 09:00.
|
||||
expect(isDaytimeNap(150, 9 * 60)).toBe(false);
|
||||
// Long in the daytime (after a night shift): a main sleep.
|
||||
expect(isDaytimeNap(420, 13 * 60)).toBe(false);
|
||||
});
|
||||
|
||||
it("scores the consistency of a steady sleeper, not 0", async () => {
|
||||
const result = await computeSleepScore("u1", PROFILE, { now: NOW });
|
||||
expect(result.status).toBe("ok");
|
||||
|
||||
@@ -38,11 +38,11 @@ import type {
|
||||
SleepStage,
|
||||
} from "@/generated/prisma/client";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { wallClockInTz } from "@/lib/tz/wall-clock";
|
||||
import { resolveUserTimezone } from "@/lib/tz/resolver";
|
||||
import { shiftDateKey } from "@/lib/tz/format";
|
||||
import { startOfLocalDayKey } from "@/lib/tz/local-day";
|
||||
import {
|
||||
asleepMidpointMinutes,
|
||||
reconstructSleepNights,
|
||||
type SleepStageRow,
|
||||
} from "@/lib/analytics/sleep-night";
|
||||
@@ -115,6 +115,11 @@ export interface SleepScoreValue {
|
||||
* `RHYTHM_MIN_NIGHTS` both are withheld with `gap: "few_nights"`.
|
||||
*/
|
||||
rhythmNights: number;
|
||||
/**
|
||||
* `RHYTHM_MIN_NIGHTS`, published so a client names the floor it shows
|
||||
* ("2 of 3 nights") from the server instead of carrying its own copy.
|
||||
*/
|
||||
rhythmMinNights: number;
|
||||
/**
|
||||
* v1.42 — the score of every scorable night in the window (oldest → newest,
|
||||
* capped to `SPARKLINE_MAX_POINTS`), so a surface can show the score's
|
||||
@@ -434,34 +439,16 @@ export function reconstructNights(
|
||||
: n.asleepMinutes > 0 && awakeMinutes > 0
|
||||
? n.asleepMinutes + awakeMinutes
|
||||
: null;
|
||||
// Midpoint = centre of the MAIN session's asleep span: its wake instant
|
||||
// minus half its own asleep minutes. Not the whole night's: a nap on the
|
||||
// same wake day moves the night's latest instant to the end of the nap and
|
||||
// adds its minutes, which put the midpoint of a 23:00 to 07:00 night with a
|
||||
// 14:00 nap at about 11:00 and read every such day as a ~7-hour shift in
|
||||
// consistency and timing.
|
||||
//
|
||||
// And only when that main session is a main sleep, not a daytime nap. On
|
||||
// a day the ring or watch recorded no night (left on the charger, taken
|
||||
// off), the day's largest session is an afternoon nap, and its 14:00
|
||||
// midpoint sat ten hours from every other night: a single such day in a
|
||||
// month pushed the midpoint spread past the 90-minute full scale and read
|
||||
// the whole rhythm as Consistency 0. Such a day still counts its minutes;
|
||||
// it just has no place on the clock. A short session at night is still
|
||||
// the night (insomnia, a newborn, split sleep), so it keeps its place.
|
||||
// Midpoint = centre of the MAIN session's asleep span. Not the whole
|
||||
// night's: a nap on the same wake day would drag it into the afternoon.
|
||||
// The main session is the nap convention's (`pickMainNightAndNaps`), so
|
||||
// a day whose only sleep is a daytime nap has none: it still counts its
|
||||
// minutes, it just has no place on the clock, exactly as the night view
|
||||
// and the list show it without a main night.
|
||||
const main = n.mainSession;
|
||||
const mainMidpoint = main
|
||||
? minutesOfDay(
|
||||
new Date(main.end.getTime() - (main.asleepMinutes / 2) * 60_000),
|
||||
tz,
|
||||
)
|
||||
const midpoint = main
|
||||
? asleepMidpointMinutes(main.end, main.asleepMinutes, tz)
|
||||
: null;
|
||||
const midpoint =
|
||||
main &&
|
||||
mainMidpoint != null &&
|
||||
!isDaytimeNap(main.asleepMinutes, mainMidpoint)
|
||||
? mainMidpoint
|
||||
: null;
|
||||
return {
|
||||
night: n.night,
|
||||
asleepMinutes: n.asleepMinutes,
|
||||
@@ -475,44 +462,6 @@ export function reconstructNights(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Below this much asleep time a session can be a nap. Three hours is where
|
||||
* wearables draw the line between a nap and a main sleep (Oura types a
|
||||
* period `long_sleep` from three hours on).
|
||||
*/
|
||||
const MAIN_SLEEP_MIN_MINUTES = 180;
|
||||
|
||||
/**
|
||||
* The night window, in local minutes-of-day: a session whose midpoint falls
|
||||
* from 20:00 to 10:00 is night sleep however short it is. Wide enough for a
|
||||
* late chronotype (asleep 04:00 to 12:00 has its midpoint at 08:00) and for
|
||||
* an early one, and clear of the afternoon where naps sit.
|
||||
*/
|
||||
const NIGHT_WINDOW_START = 20 * 60;
|
||||
const NIGHT_WINDOW_END = 10 * 60;
|
||||
|
||||
/**
|
||||
* True for a session that is a nap rather than a night: short (under
|
||||
* `MAIN_SLEEP_MIN_MINUTES` asleep) AND centred in the daytime. A short night
|
||||
* is still the night; a long daytime sleep (after a night shift) is still a
|
||||
* main sleep. Pure.
|
||||
*/
|
||||
export function isDaytimeNap(
|
||||
asleepMinutes: number,
|
||||
midpointMinutes: number,
|
||||
): boolean {
|
||||
if (asleepMinutes >= MAIN_SLEEP_MIN_MINUTES) return false;
|
||||
const atNight =
|
||||
midpointMinutes >= NIGHT_WINDOW_START || midpointMinutes < NIGHT_WINDOW_END;
|
||||
return !atNight;
|
||||
}
|
||||
|
||||
function minutesOfDay(d: Date, tz: string): number {
|
||||
if (tz === "UTC") return d.getUTCHours() * 60 + d.getUTCMinutes();
|
||||
const { hour, minute } = wallClockInTz(d, tz);
|
||||
return hour * 60 + minute;
|
||||
}
|
||||
|
||||
// ── compute ─────────────────────────────────────────────────────────────
|
||||
|
||||
export interface SleepScoreOpts {
|
||||
@@ -696,6 +645,7 @@ export async function computeSleepScore(
|
||||
subScores,
|
||||
windowNights: scorableNights.length,
|
||||
rhythmNights: midpoints.length,
|
||||
rhythmMinNights: RHYTHM_MIN_NIGHTS,
|
||||
series,
|
||||
},
|
||||
coverage,
|
||||
|
||||
@@ -71,10 +71,7 @@ import {
|
||||
summarizeSeries,
|
||||
} from "@/lib/insights/status-shared";
|
||||
import { runStatusCompletion } from "@/lib/insights/status-provider";
|
||||
import {
|
||||
reconstructSleepNights,
|
||||
type SleepStageRow,
|
||||
} from "@/lib/analytics/sleep-night";
|
||||
import { readSleepNights } from "@/lib/measurements/list/sleep-nights";
|
||||
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
|
||||
import { resolveUserTimezone, userDayKey } from "@/lib/tz/resolver";
|
||||
import {
|
||||
@@ -295,31 +292,16 @@ export async function generateMetricStatus(args: {
|
||||
const points: Array<{ measuredAt: Date; value: number }> = [];
|
||||
let measurements: Array<{ measuredAt: Date }> = [];
|
||||
if (isSleep) {
|
||||
const [priorityJson, sleepRows] = await Promise.all([
|
||||
loadUserSourcePriority(args.userId),
|
||||
prisma.measurement.findMany({
|
||||
where: {
|
||||
userId: args.userId,
|
||||
type: "SLEEP_DURATION",
|
||||
deletedAt: null,
|
||||
},
|
||||
orderBy: { measuredAt: "desc" },
|
||||
take: 365 * 6, // ≈ 5 stage rows/night × source-count for a year
|
||||
// Writer-level collapse: two HealthKit apps behind one source
|
||||
// (watch stages vs phone in-bed) must not blend into one night.
|
||||
select: {
|
||||
value: true,
|
||||
measuredAt: true,
|
||||
sleepStage: true,
|
||||
source: true,
|
||||
deviceType: true,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
const nights = reconstructSleepNights(
|
||||
sleepRows as SleepStageRow[],
|
||||
userTz,
|
||||
priorityJson,
|
||||
// The latest 365 nights, the sleep twin of the 365 latest readings the
|
||||
// other types read. A cut on stage rows instead (2190 of them) reached
|
||||
// back a few weeks on a stage-rich watch and began mid-night.
|
||||
const nights = (
|
||||
await readSleepNights(
|
||||
args.userId,
|
||||
userTz,
|
||||
await loadUserSourcePriority(args.userId),
|
||||
{ maxNights: 365, newest: true },
|
||||
)
|
||||
).filter((n) => n.asleepMinutes > 0);
|
||||
for (const n of nights) {
|
||||
points.push({ measuredAt: n.measuredAt, value: n.asleepMinutes });
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { prisma } from "@/lib/db";
|
||||
import { readOuraScopeDenied } from "@/lib/oura/credentials";
|
||||
import {
|
||||
hasRecoveryData,
|
||||
resolveRecoveryEmpty,
|
||||
type RecoveryEmptyReason,
|
||||
} from "./recovery-empty";
|
||||
|
||||
/**
|
||||
* The recovery page's empty reason for one account. Reads the Oura
|
||||
* connection only when no recovery type has a reading, so a page with data
|
||||
* never pays for it.
|
||||
*/
|
||||
export async function loadRecoveryEmpty(
|
||||
userId: string,
|
||||
summaries: Record<string, { count?: number } | null | undefined>,
|
||||
): Promise<RecoveryEmptyReason | null> {
|
||||
if (hasRecoveryData(summaries)) return null;
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: userId },
|
||||
select: { ouraAccessTokenEncrypted: true, ouraScopeDenied: true },
|
||||
});
|
||||
return resolveRecoveryEmpty(summaries, {
|
||||
connected: !!user?.ouraAccessTokenEncrypted,
|
||||
scopeDenied: readOuraScopeDenied(user?.ouraScopeDenied),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Why the recovery page is empty, decided once on the server.
|
||||
*
|
||||
* The page charts the device-native recovery signals below. When none of
|
||||
* them has a reading, the analytics slice publishes the reason beside the
|
||||
* summaries (`recoveryEmpty`), and every client renders from it rather than
|
||||
* working it out from the connection status itself:
|
||||
*
|
||||
* - `oura_permission`: Oura is connected and the stored grant leaves out
|
||||
* the scope its resilience level sits behind; one reconnect fixes it.
|
||||
* - `oura_no_resilience`: Oura is connected and no level has arrived. The
|
||||
* server cannot tell "the grant covers it" from "not recorded yet" (the
|
||||
* scope note is cleared on every reconnect and written only by a sync
|
||||
* that reached Oura), so this case never claims coverage.
|
||||
* - `general`: anyone else.
|
||||
*
|
||||
* `null` means the page has something to chart.
|
||||
*
|
||||
* Pure, so the page can share the type list it charts from.
|
||||
*/
|
||||
|
||||
/** Every summary type the recovery page charts or shows as a tile. */
|
||||
export const RECOVERY_SUMMARY_TYPES = [
|
||||
"ANS_CHARGE",
|
||||
"DAY_STRAIN",
|
||||
"WORKOUT_STRAIN",
|
||||
"CARDIO_LOAD",
|
||||
"AVERAGE_HEART_RATE",
|
||||
"MAX_HEART_RATE",
|
||||
"ENERGY_EXPENDITURE_KJ",
|
||||
"RESILIENCE",
|
||||
] as const;
|
||||
|
||||
export const RECOVERY_EMPTY_REASONS = [
|
||||
"general",
|
||||
"oura_permission",
|
||||
"oura_no_resilience",
|
||||
] as const;
|
||||
|
||||
export type RecoveryEmptyReason = (typeof RECOVERY_EMPTY_REASONS)[number];
|
||||
|
||||
/** True when any recovery type has a reading. */
|
||||
export function hasRecoveryData(
|
||||
summaries: Record<string, { count?: number } | null | undefined>,
|
||||
): boolean {
|
||||
return RECOVERY_SUMMARY_TYPES.some(
|
||||
(type) => (summaries[type]?.count ?? 0) > 0,
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveRecoveryEmpty(
|
||||
summaries: Record<string, { count?: number } | null | undefined>,
|
||||
oura: { connected: boolean; scopeDenied: readonly string[] },
|
||||
): RecoveryEmptyReason | null {
|
||||
if (hasRecoveryData(summaries)) return null;
|
||||
if (!oura.connected) return "general";
|
||||
return oura.scopeDenied.includes("resilience")
|
||||
? "oura_permission"
|
||||
: "oura_no_resilience";
|
||||
}
|
||||
@@ -63,12 +63,13 @@ vi.mock("@/lib/safe-fetch", async (importOriginal) => {
|
||||
return { ...actual, safeFetch: safeFetchMock };
|
||||
});
|
||||
|
||||
vi.mock("@/lib/google-health/sync-core", () => ({
|
||||
GOOGLE_HEALTH_INTEGRATION_KEY: "google-health",
|
||||
readGoogleHealthBackfillDone: () => new Set<string>(resumeState.doneBefore),
|
||||
readGoogleHealthBackfillCursors: () => new Map(),
|
||||
readGoogleHealthBackfillEdgeFilterUnsupported: () => new Set<string>(),
|
||||
readGoogleHealthBackfillEnded: () => new Map(),
|
||||
vi.mock("@/lib/google-health/backfill-progress", () => ({
|
||||
readGoogleHealthBackfillProgress: () => ({
|
||||
done: new Set<string>(resumeState.doneBefore),
|
||||
cursors: new Map(),
|
||||
edgeFilterUnsupported: new Set<string>(),
|
||||
ended: new Map(),
|
||||
}),
|
||||
googleHealthBackfillProgressAfterCompletion: (ended: Map<string, unknown>) =>
|
||||
ended.size > 0 ? { ended: Object.fromEntries(ended) } : null,
|
||||
googleHealthBackfillRetryPending: (
|
||||
@@ -79,6 +80,10 @@ vi.mock("@/lib/google-health/sync-core", () => ({
|
||||
.map(([key]) => key);
|
||||
return keys.length > 0 ? { keys, dueAt: resumeState.retryDueAt } : null;
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/google-health/sync-core", () => ({
|
||||
GOOGLE_HEALTH_INTEGRATION_KEY: "google-health",
|
||||
runWithGoogleHealthBackfillResume: async (
|
||||
_userId: string,
|
||||
done: Set<string>,
|
||||
@@ -124,6 +129,7 @@ vi.mock("@/lib/google-health/sync", () => ({
|
||||
import {
|
||||
GOOGLE_HEALTH_BACKFILL_MAX_SLICE,
|
||||
GOOGLE_HEALTH_BACKFILL_THROTTLED_DELAY_SECONDS,
|
||||
googleHealthBackfillRetryKey,
|
||||
runGoogleHealthBackfillForUser,
|
||||
} from "../google-health-backfill";
|
||||
import { INTEGRATION_BACKFILL_CONTINUATION_DELAY_SECONDS } from "@/lib/jobs/integration-backfill-admission";
|
||||
@@ -395,6 +401,46 @@ describe("runGoogleHealthBackfillForUser — slices", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keys a waiting retry on when it is due, so a boot's slice cannot start a second chain", async () => {
|
||||
resumeState.doneBefore = GOOGLE_HEALTH_COLLECTION_KEYS.filter(
|
||||
(k) => k !== "fetchHeartRate",
|
||||
);
|
||||
resumeState.parks = ["fetchHeartRate"];
|
||||
resumeState.retryDueAt = Date.now() + 20 * 3_600_000;
|
||||
syncUserGoogleHealthMock.mockResolvedValue({ imported: 0, failed: false });
|
||||
|
||||
// The slice that parked the collection, then a boot's first slice meeting
|
||||
// the same wait from the same note.
|
||||
await runGoogleHealthBackfillForUser("u1", undefined, 7);
|
||||
await runGoogleHealthBackfillForUser("u1", undefined, 0);
|
||||
|
||||
const keys = bossSend.mock.calls.map(
|
||||
(call) => (call[2] as { singletonKey: string }).singletonKey,
|
||||
);
|
||||
expect(keys).toEqual([
|
||||
googleHealthBackfillRetryKey("u1", resumeState.retryDueAt),
|
||||
googleHealthBackfillRetryKey("u1", resumeState.retryDueAt),
|
||||
]);
|
||||
expect(keys[0]).toBe(
|
||||
`google-health-backfill|u1|retry-${new Date(resumeState.retryDueAt).toISOString()}`,
|
||||
);
|
||||
});
|
||||
|
||||
it("sends a retry that is already due as an ordinary next slice", async () => {
|
||||
resumeState.doneBefore = GOOGLE_HEALTH_COLLECTION_KEYS.filter(
|
||||
(k) => k !== "fetchHeartRate",
|
||||
);
|
||||
resumeState.parks = ["fetchHeartRate"];
|
||||
resumeState.retryDueAt = Date.now() - 1_000;
|
||||
syncUserGoogleHealthMock.mockResolvedValue({ imported: 0, failed: false });
|
||||
|
||||
await runGoogleHealthBackfillForUser("u1", undefined, 7);
|
||||
expect(bossSend.mock.calls[0]![2]).toMatchObject({
|
||||
singletonKey: "google-health-backfill|u1|slice-8",
|
||||
startAfter: INTEGRATION_BACKFILL_CONTINUATION_DELAY_SECONDS,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not stamp over a parked collection when the retry cannot be queued", async () => {
|
||||
resumeState.doneBefore = GOOGLE_HEALTH_COLLECTION_KEYS.filter(
|
||||
(k) => k !== "fetchHeartRate",
|
||||
|
||||
@@ -110,7 +110,7 @@ export function buildFeedbackBuckets(
|
||||
row.metricSourceType,
|
||||
row.providerType,
|
||||
row.promptVersion,
|
||||
].join("");
|
||||
].join("\x01");
|
||||
const existing = map.get(key);
|
||||
if (existing) {
|
||||
if (row.helpful) existing.helpful += 1;
|
||||
|
||||
@@ -17,18 +17,19 @@ import { annotate } from "@/lib/logging/context";
|
||||
import { getGlobalBoss } from "@/lib/jobs/boss-instance";
|
||||
import {
|
||||
enqueueIntegrationBackfillAdmission,
|
||||
INTEGRATION_BACKFILL_ADMISSION_QUEUE,
|
||||
INTEGRATION_BACKFILL_CONTINUATION_DELAY_SECONDS,
|
||||
integrationBackfillAdmissionSendOptions,
|
||||
integrationBackfillIdentity,
|
||||
integrationBackfillSourceOptions,
|
||||
type IntegrationBackfillAdmissionPayload,
|
||||
} from "@/lib/jobs/integration-backfill-admission";
|
||||
import { runWithGoogleHealthBackfillResume } from "@/lib/google-health/sync-core";
|
||||
import {
|
||||
googleHealthBackfillProgressAfterCompletion,
|
||||
readGoogleHealthBackfillCursors,
|
||||
readGoogleHealthBackfillDone,
|
||||
readGoogleHealthBackfillEdgeFilterUnsupported,
|
||||
readGoogleHealthBackfillEnded,
|
||||
googleHealthBackfillRetryPending,
|
||||
runWithGoogleHealthBackfillResume,
|
||||
} from "@/lib/google-health/sync-core";
|
||||
readGoogleHealthBackfillProgress,
|
||||
} from "@/lib/google-health/backfill-progress";
|
||||
import { googleHealthBackfillEndIsFinal } from "@/lib/google-health/backfill-bounds";
|
||||
import {
|
||||
isGoogleHealthParked,
|
||||
@@ -104,6 +105,23 @@ export const GOOGLE_HEALTH_BACKFILL_MAX_SLICE =
|
||||
) +
|
||||
2;
|
||||
|
||||
/**
|
||||
* The singleton key of a slice that waits for a parked collection's retry:
|
||||
* the import's identity and the instant the retry is due. The instant comes
|
||||
* from the stored note, so every slice that reads the same note derives the
|
||||
* same key.
|
||||
*/
|
||||
export function googleHealthBackfillRetryKey(
|
||||
userId: string,
|
||||
dueAt: number,
|
||||
): string {
|
||||
const identity = integrationBackfillIdentity({
|
||||
kind: "google-health-backfill",
|
||||
data: { userId, enqueuedAt: "" },
|
||||
});
|
||||
return `${identity}|retry-${new Date(dueAt).toISOString()}`;
|
||||
}
|
||||
|
||||
/** Test seam: a slice's budget, when not the production one. */
|
||||
export interface GoogleHealthBackfillSliceOptions {
|
||||
sliceMs?: number;
|
||||
@@ -198,14 +216,12 @@ export async function runGoogleHealthBackfillForUser(
|
||||
});
|
||||
return { imported: 0 };
|
||||
}
|
||||
const done = readGoogleHealthBackfillDone(connection?.backfillProgress);
|
||||
const cursors = readGoogleHealthBackfillCursors(connection?.backfillProgress);
|
||||
const edgeFilterUnsupported = readGoogleHealthBackfillEdgeFilterUnsupported(
|
||||
connection?.backfillProgress,
|
||||
);
|
||||
const endedBefore = readGoogleHealthBackfillEnded(
|
||||
connection?.backfillProgress,
|
||||
);
|
||||
const {
|
||||
done,
|
||||
cursors,
|
||||
edgeFilterUnsupported,
|
||||
ended: endedBefore,
|
||||
} = readGoogleHealthBackfillProgress(connection?.backfillProgress);
|
||||
// A collection that ended short for good has read all it is going to, so
|
||||
// it counts as finished alongside the ones read to the end. One parked
|
||||
// after a first end without progress does not: it waits for another try.
|
||||
@@ -285,23 +301,36 @@ export async function runGoogleHealthBackfillForUser(
|
||||
);
|
||||
}
|
||||
|
||||
const sendNextSlice = async (delaySeconds: number): Promise<boolean> => {
|
||||
/**
|
||||
* Queue the next slice. A slice that waits for a parked collection's retry
|
||||
* goes under a key named after when the retry is due, instead of its slice
|
||||
* number: every slice that meets the same wait, a boot's included, names the
|
||||
* same job, and the queue keeps one. Under the slice number a boot during
|
||||
* the day-long wait started a second chain beside the waiting one.
|
||||
*/
|
||||
const sendNextSlice = async (
|
||||
delaySeconds: number,
|
||||
retryDueAt?: number,
|
||||
): Promise<boolean> => {
|
||||
if (slice + 1 > GOOGLE_HEALTH_BACKFILL_MAX_SLICE) return false;
|
||||
const boss = getGlobalBoss();
|
||||
const sent = boss
|
||||
? await enqueueIntegrationBackfillAdmission(
|
||||
boss,
|
||||
{
|
||||
kind: "google-health-backfill",
|
||||
data: {
|
||||
userId,
|
||||
enqueuedAt: new Date().toISOString(),
|
||||
slice: slice + 1,
|
||||
},
|
||||
},
|
||||
delaySeconds,
|
||||
)
|
||||
: null;
|
||||
if (!boss) return false;
|
||||
const payload: IntegrationBackfillAdmissionPayload = {
|
||||
kind: "google-health-backfill",
|
||||
data: {
|
||||
userId,
|
||||
enqueuedAt: new Date().toISOString(),
|
||||
slice: slice + 1,
|
||||
},
|
||||
};
|
||||
const sent =
|
||||
retryDueAt === undefined
|
||||
? await enqueueIntegrationBackfillAdmission(boss, payload, delaySeconds)
|
||||
: await boss.send(INTEGRATION_BACKFILL_ADMISSION_QUEUE, payload, {
|
||||
...integrationBackfillAdmissionSendOptions(payload),
|
||||
singletonKey: googleHealthBackfillRetryKey(userId, retryDueAt),
|
||||
startAfter: delaySeconds,
|
||||
});
|
||||
return Boolean(sent);
|
||||
};
|
||||
|
||||
@@ -315,11 +344,18 @@ export async function runGoogleHealthBackfillForUser(
|
||||
(key) => done.has(key) || pending.keys.includes(key),
|
||||
);
|
||||
if (pending && (onlyRetriesLeft || !failed)) {
|
||||
const waitSeconds = Math.ceil((pending.dueAt - Date.now()) / 1000);
|
||||
const delaySeconds = Math.max(
|
||||
INTEGRATION_BACKFILL_CONTINUATION_DELAY_SECONDS,
|
||||
Math.ceil((pending.dueAt - Date.now()) / 1000),
|
||||
waitSeconds,
|
||||
);
|
||||
if (await sendNextSlice(delaySeconds)) {
|
||||
// A retry already due is an ordinary next slice; only a wait is keyed on
|
||||
// when it ends.
|
||||
const delayed =
|
||||
waitSeconds > INTEGRATION_BACKFILL_CONTINUATION_DELAY_SECONDS;
|
||||
if (
|
||||
await sendNextSlice(delaySeconds, delayed ? pending.dueAt : undefined)
|
||||
) {
|
||||
annotate({
|
||||
action: {
|
||||
name: "google_health.backfill.retry_scheduled",
|
||||
@@ -328,9 +364,10 @@ export async function runGoogleHealthBackfillForUser(
|
||||
});
|
||||
return { imported, continued: true };
|
||||
}
|
||||
// Not queued (most often one is already waiting under the same slice
|
||||
// key, sent by an earlier boot's slice). Nothing is lost: the marker is
|
||||
// not stamped, so the next boot offers the backfill again.
|
||||
// Not queued: most often the same retry is already waiting, sent by the
|
||||
// slice that parked the collection, and this is a boot's slice meeting
|
||||
// the same wait. Nothing is lost either way: the marker is not stamped,
|
||||
// so the next boot offers the backfill again.
|
||||
annotate({
|
||||
action: {
|
||||
name: "google_health.backfill.retry_not_queued",
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
import { prisma } from "@/lib/db";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import { metricKeyForType } from "@/lib/measurements/cumulative-day-sum";
|
||||
import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority";
|
||||
import {
|
||||
canonicalDailyTimestamp,
|
||||
dayKeyForUserTz,
|
||||
} from "@/lib/measurements/drain-per-sample-cumulative";
|
||||
import { localDayWindow } from "@/lib/tz/local-day";
|
||||
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
|
||||
import { resolveStoredTimezone } from "@/lib/tz/resolver";
|
||||
import {
|
||||
Prisma,
|
||||
type MeasurementSource,
|
||||
type MeasurementType,
|
||||
} from "@/generated/prisma/client";
|
||||
import { pageWithWindowTotal } from "./paging";
|
||||
|
||||
/**
|
||||
* The day-grouped list for a cumulative type, paged over local days.
|
||||
*
|
||||
* Everything heavy runs in Postgres. The first query picks the page: the
|
||||
* distinct local days that hold a reading, sorted and cut by `limit` /
|
||||
* `offset`, each row carrying the distinct-day count of the whole filtered
|
||||
* set as `meta.total` from a window count. The second folds only the page's
|
||||
* days, one row per day, source, device and unit with its sum and sample
|
||||
* count, and the source ladder picks among those groups exactly as it would
|
||||
* among the raw rows (it keeps or drops whole source and device groups per
|
||||
* day, never single rows). So a page costs its own days whatever the length
|
||||
* of the history, every day is complete, and no raw row reaches the heap.
|
||||
*
|
||||
* Until v1.43.6 the branch read the newest `limit` raw rows (the web asked
|
||||
* for 5000), bucketed them and returned every bucket. A long history came
|
||||
* back as thousands of rows in one response, and every day older than the
|
||||
* 5000th raw row was missing with nothing to say so.
|
||||
*
|
||||
* A day exists for the page when any reading of the type falls on it; the
|
||||
* source collapse narrows a day's groups but never empties one, so the count
|
||||
* and the folded rows agree. A value range would filter single readings and
|
||||
* leave partial days behind, so the validator refuses `valueMin` /
|
||||
* `valueMax` beside `groupBy=day`.
|
||||
*
|
||||
* Every page reads the filtered history to find its days. After the nightly
|
||||
* drain a cumulative type holds about one row per day and source, so the
|
||||
* scan is the index-only walk over days it would be anyway.
|
||||
*/
|
||||
export async function dayGroupedListResponse(
|
||||
user: { id: string; timezone?: string | null },
|
||||
opts: {
|
||||
type: MeasurementType;
|
||||
limit: number;
|
||||
offset: number;
|
||||
sortDir: "asc" | "desc";
|
||||
sourceEq?: MeasurementSource;
|
||||
from?: Date;
|
||||
to?: Date;
|
||||
},
|
||||
) {
|
||||
const { type, limit, offset, sortDir } = opts;
|
||||
// One resolver for the stored zone, shared with the drill-down and the
|
||||
// sleep list: a usable IANA name or the instance default, never a value
|
||||
// Postgres would refuse. The drill-down must cut the same days.
|
||||
const tz = await resolveStoredTimezone(user.timezone);
|
||||
|
||||
// The request's filters, as SQL: the same ones the plain list applies.
|
||||
const filters = Prisma.sql`
|
||||
m."user_id" = ${user.id}
|
||||
AND m."type" = ${type}::measurement_type
|
||||
AND m."deleted_at" IS NULL
|
||||
${opts.sourceEq ? Prisma.sql`AND m."source" = ${opts.sourceEq}::measurement_source` : Prisma.empty}
|
||||
${opts.from ? Prisma.sql`AND m."measured_at" >= ${opts.from}` : Prisma.empty}
|
||||
${opts.to ? Prisma.sql`AND m."measured_at" <= ${opts.to}` : Prisma.empty}
|
||||
`;
|
||||
const localDay = Prisma.sql`to_char((m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD')`;
|
||||
// Direction comes from the Zod enum, never from raw input. The window
|
||||
// count runs before LIMIT, so every row of the page carries the
|
||||
// distinct-day total of the whole filtered set.
|
||||
const direction = Prisma.raw(sortDir === "asc" ? "ASC" : "DESC");
|
||||
const { rows: pageDays, total } = await pageWithWindowTotal(
|
||||
await prisma.$queryRaw<Array<{ day_key: string; total: number }>>`
|
||||
SELECT d.day_key, (COUNT(*) OVER ())::int AS total
|
||||
FROM (
|
||||
SELECT ${localDay} AS day_key
|
||||
FROM measurements m
|
||||
WHERE ${filters}
|
||||
GROUP BY 1
|
||||
) d
|
||||
ORDER BY d.day_key ${direction}
|
||||
LIMIT ${limit} OFFSET ${offset}
|
||||
`,
|
||||
offset,
|
||||
async () =>
|
||||
Number(
|
||||
(
|
||||
await prisma.$queryRaw<Array<{ total: number }>>`
|
||||
SELECT COUNT(DISTINCT ${localDay})::int AS total
|
||||
FROM measurements m
|
||||
WHERE ${filters}
|
||||
`
|
||||
)[0]?.total ?? 0,
|
||||
),
|
||||
);
|
||||
const dayKeys = pageDays.map((d) => d.day_key);
|
||||
|
||||
// The page's days are consecutive in the sorted day set, so the span from
|
||||
// the first one's start to the last one's end holds exactly their rows.
|
||||
let groups: Array<{
|
||||
day_key: string;
|
||||
source: MeasurementSource;
|
||||
device_type: string | null;
|
||||
unit: string;
|
||||
total: number;
|
||||
n: number;
|
||||
}> = [];
|
||||
if (dayKeys.length > 0) {
|
||||
const sorted = [...dayKeys].sort();
|
||||
const spanStart = localDayWindow(sorted[0], tz).dayStart;
|
||||
const spanEnd = localDayWindow(sorted[sorted.length - 1], tz).dayEnd;
|
||||
// Newest group first within a day, so the day's label comes from its
|
||||
// latest reading, as it did when raw rows were read newest first.
|
||||
groups = await prisma.$queryRaw`
|
||||
SELECT
|
||||
${localDay} AS day_key,
|
||||
m."source"::text AS source,
|
||||
m."device_type" AS device_type,
|
||||
m."unit" AS unit,
|
||||
SUM(m."value")::float8 AS total,
|
||||
COUNT(*)::int AS n
|
||||
FROM measurements m
|
||||
WHERE ${filters}
|
||||
AND m."measured_at" >= ${spanStart}
|
||||
AND m."measured_at" < ${spanEnd}
|
||||
GROUP BY 1, 2, 3, 4
|
||||
ORDER BY 1, MAX(m."measured_at") DESC, 2, 3, 4
|
||||
`;
|
||||
}
|
||||
|
||||
// v1.29.6 — collapse overlapping sources to the ladder-canonical
|
||||
// reading per day BEFORE summing. Without this, a day with both an
|
||||
// Apple Health AND a Withings write for the same cumulative metric
|
||||
// (e.g. steps) summed BOTH sources into one inflated total — a day
|
||||
// the dashboard tile (which reads through the source-collapsed
|
||||
// rollup, `collapseRollupRowsBySource` in
|
||||
// `src/lib/rollups/measurement-read.ts`) showed correctly. Uses the
|
||||
// same `metricKeyForType` ladder lookup the rollup collapse uses, so
|
||||
// this list and the tile agree on which source wins. A type with no
|
||||
// ladder (e.g. TIME_IN_DAYLIGHT, FALL_COUNT) has nothing to collapse
|
||||
// against — every group passes through unchanged. Each group stands at
|
||||
// its day's local noon, so the picker files it under the day SQL cut.
|
||||
const groupRows = groups.map((g) => ({
|
||||
...g,
|
||||
type,
|
||||
deviceType: g.device_type,
|
||||
measuredAt: canonicalDailyTimestamp(g.day_key, tz),
|
||||
}));
|
||||
const metricKey = metricKeyForType(type);
|
||||
const priorityJson = metricKey ? await loadUserSourcePriority(user.id) : null;
|
||||
const canonicalGroups = metricKey
|
||||
? pickCanonicalSourceRows(groupRows, metricKey, priorityJson, (d) =>
|
||||
dayKeyForUserTz(d, tz),
|
||||
).canonicalRows
|
||||
: groupRows;
|
||||
|
||||
// Sum per local day; unit and source come from the day's first kept group
|
||||
// so the rendered row carries a sensible label even when sources mix.
|
||||
const buckets = new Map<
|
||||
string,
|
||||
{ total: number; sampleCount: number; unit: string; source: string }
|
||||
>();
|
||||
for (const g of canonicalGroups) {
|
||||
const slot = buckets.get(g.day_key);
|
||||
if (!slot) {
|
||||
buckets.set(g.day_key, {
|
||||
total: g.total,
|
||||
sampleCount: g.n,
|
||||
unit: g.unit,
|
||||
source: g.source,
|
||||
});
|
||||
} else {
|
||||
slot.total += g.total;
|
||||
slot.sampleCount += g.n;
|
||||
}
|
||||
}
|
||||
const rowsScanned = groups.reduce((sum, g) => sum + g.n, 0);
|
||||
const rowsKept = canonicalGroups.reduce((sum, g) => sum + g.n, 0);
|
||||
|
||||
// Rows follow the page's day order from SQL.
|
||||
const measurements = dayKeys.flatMap((dayKey) => {
|
||||
const b = buckets.get(dayKey);
|
||||
if (!b) return [];
|
||||
return [
|
||||
{
|
||||
// Synthetic id so the list-row key stays stable across pages
|
||||
// and the row never collides with a real `Measurement.id`.
|
||||
id: `day:${type}:${dayKey}`,
|
||||
type,
|
||||
value: b.total,
|
||||
unit: b.unit,
|
||||
source: b.source,
|
||||
// Anchor the row's timestamp to local noon so the row sorts
|
||||
// cleanly between same-day spot samples — matches the
|
||||
// `canonicalDailyTimestamp` convention used by the drain.
|
||||
measuredAt: canonicalDailyTimestamp(dayKey, tz).toISOString(),
|
||||
notes: null,
|
||||
dayKey,
|
||||
sampleCount: b.sampleCount,
|
||||
},
|
||||
];
|
||||
});
|
||||
|
||||
annotate({
|
||||
action: { name: "measurement.list" },
|
||||
meta: {
|
||||
total,
|
||||
type,
|
||||
groupBy: "day",
|
||||
days: measurements.length,
|
||||
rowsScanned,
|
||||
droppedDuplicates: rowsScanned - rowsKept,
|
||||
},
|
||||
});
|
||||
return apiSuccess({
|
||||
measurements,
|
||||
meta: {
|
||||
total,
|
||||
limit,
|
||||
offset,
|
||||
groupBy: "day",
|
||||
droppedDuplicates: rowsScanned - rowsKept,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* A page cut in SQL that carries the size of the whole set.
|
||||
*
|
||||
* The page query selects `(COUNT(*) OVER ())::int AS total` beside its rows:
|
||||
* the window count runs before `LIMIT`, so every row of the page carries the
|
||||
* count of the whole filtered set and one round trip answers both. A page
|
||||
* past the end has no row to carry it; only then is the set counted on its
|
||||
* own, by `countAll`.
|
||||
*/
|
||||
export async function pageWithWindowTotal<T extends { total: number }>(
|
||||
rows: T[],
|
||||
offset: number,
|
||||
countAll: () => Promise<number>,
|
||||
): Promise<{ rows: T[]; total: number }> {
|
||||
if (rows.length > 0) return { rows, total: Number(rows[0].total) };
|
||||
return { rows, total: offset > 0 ? await countAll() : 0 };
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* The SLEEP_DURATION list: one row per night, or one night's stage segments.
|
||||
*
|
||||
* Sleep is stored one row per STAGE per night (minutes); a plain list would
|
||||
* leak every stage segment as its own "value". The list collapses the raw rows
|
||||
* into the canonical per-night shape used by the chart series and the
|
||||
* dashboard tile.
|
||||
*
|
||||
* Two modes:
|
||||
* - `dayKey` set → drill-down: the night's stage SEGMENTS (the canonical
|
||||
* source's per-stage spans), read exactly as the night view reads them.
|
||||
* - no `dayKey` → one synthetic row per night, value = the main sleep's TIME
|
||||
* ASLEEP in minutes (the canonical unit), with the per-night stage
|
||||
* breakdown, nap count and awakenings in the row so the UI can caption the
|
||||
* night without a second fetch.
|
||||
*
|
||||
* The nights are paged in Postgres (`sleepNightPage`) and only the stage rows
|
||||
* of the page's nights are read. Every page night gets a row, so a page is as
|
||||
* long as its limit and `meta.total` counts exactly the rows a full walk sees.
|
||||
* A day whose only sleep is daytime naps has no main night: its row carries
|
||||
* 0 minutes asleep and its naps, as `/api/sleep/night` answers the same day
|
||||
* with `main: null` and the naps.
|
||||
*/
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { apiSuccess } from "@/lib/api-response";
|
||||
import { getUnitForType } from "@/lib/validations/measurement";
|
||||
import { loadUserSourcePriority } from "@/lib/rollups/measurement-read";
|
||||
import { resolveStoredTimezone } from "@/lib/tz/resolver";
|
||||
import {
|
||||
pickMainNightAndNaps,
|
||||
reconstructSleepSessions,
|
||||
type SleepSession,
|
||||
} from "@/lib/analytics/sleep-night";
|
||||
import {
|
||||
readNightSessions,
|
||||
readNightSpanRows,
|
||||
sleepNightPage,
|
||||
} from "./sleep-nights";
|
||||
|
||||
/**
|
||||
* A sleep page holds at most this many nights. The nights' stage rows are
|
||||
* read and reconstructed in the process, so the bound sits on the nights;
|
||||
* the applied value comes back as `meta.limit`, and `meta.total` still counts
|
||||
* every night, so a caller asking for more pages on rather than losing any.
|
||||
*/
|
||||
export const SLEEP_NIGHT_PAGE_MAX = 1000;
|
||||
|
||||
export async function sleepListResponse(
|
||||
user: { id: string; timezone?: string | null },
|
||||
opts: {
|
||||
from?: Date;
|
||||
to?: Date;
|
||||
limit: number;
|
||||
offset: number;
|
||||
sortDir: "asc" | "desc";
|
||||
dayKey?: string;
|
||||
},
|
||||
) {
|
||||
const { from, to, offset, sortDir, dayKey } = opts;
|
||||
const limit = Math.min(opts.limit, SLEEP_NIGHT_PAGE_MAX);
|
||||
const [tz, priorityJson] = await Promise.all([
|
||||
resolveStoredTimezone(user.timezone),
|
||||
loadUserSourcePriority(user.id),
|
||||
]);
|
||||
const dayUnit = getUnitForType("SLEEP_DURATION");
|
||||
|
||||
if (dayKey) {
|
||||
const { sessions } = await readNightSessions(
|
||||
user.id,
|
||||
dayKey,
|
||||
tz,
|
||||
priorityJson,
|
||||
{ from, to },
|
||||
);
|
||||
const { main, naps } = pickMainNightAndNaps(sessions);
|
||||
// One row per stage segment of the main night + each nap, so the
|
||||
// drill-down reveals exactly the rows that summed into the row.
|
||||
const measurements = [main, ...naps]
|
||||
.filter((s): s is SleepSession => s != null)
|
||||
.flatMap((s) => s.segments)
|
||||
.sort((a, b) =>
|
||||
sortDir === "asc"
|
||||
? a.start.getTime() - b.start.getTime()
|
||||
: b.start.getTime() - a.start.getTime(),
|
||||
)
|
||||
.map((seg, i) => ({
|
||||
id: `sleep-seg:${dayKey}:${i}`,
|
||||
type: "SLEEP_DURATION",
|
||||
value: Math.round(seg.minutes),
|
||||
unit: dayUnit,
|
||||
source: main?.source ?? naps[0]?.source ?? "MANUAL",
|
||||
measuredAt: seg.end.toISOString(),
|
||||
sleepStage: seg.stage,
|
||||
notes: null,
|
||||
}));
|
||||
annotate({
|
||||
action: { name: "measurement.list" },
|
||||
meta: {
|
||||
total: measurements.length,
|
||||
type: "SLEEP_DURATION",
|
||||
dayKey,
|
||||
mode: "sleep-drill-down",
|
||||
},
|
||||
});
|
||||
return apiSuccess({
|
||||
measurements,
|
||||
meta: { total: measurements.length, limit, offset: 0, dayKey },
|
||||
});
|
||||
}
|
||||
|
||||
const page = await sleepNightPage(user.id, tz, {
|
||||
from,
|
||||
to,
|
||||
sortDir,
|
||||
limit,
|
||||
offset,
|
||||
});
|
||||
const rows = await readNightSpanRows(user.id, page.nights, { from, to });
|
||||
const byDay = new Map<string, SleepSession[]>();
|
||||
for (const s of reconstructSleepSessions(rows, tz, priorityJson)) {
|
||||
const list = byDay.get(s.night) ?? [];
|
||||
list.push(s);
|
||||
byDay.set(s.night, list);
|
||||
}
|
||||
|
||||
// Rows follow the page's night order from SQL.
|
||||
const measurements = page.nights.map(({ night, spanEnd }) => {
|
||||
const { main, naps } = pickMainNightAndNaps(byDay.get(night) ?? []);
|
||||
const lastNap = naps.at(-1);
|
||||
return {
|
||||
// Synthetic id mirrors the series route's `sleep:<night>` shape.
|
||||
id: `sleep:${night}`,
|
||||
type: "SLEEP_DURATION",
|
||||
value: main ? Math.round(main.asleepMinutes) : 0,
|
||||
unit: dayUnit,
|
||||
source: main?.source ?? naps[0]?.source ?? "MANUAL",
|
||||
measuredAt: (main?.end ?? lastNap?.end ?? spanEnd).toISOString(),
|
||||
notes: null,
|
||||
dayKey: night,
|
||||
// Drives the drill-down chevron: the stage segments behind the row,
|
||||
// the main night's and the naps', exactly what the drill-down returns.
|
||||
sampleCount: [main, ...naps].reduce(
|
||||
(sum, s) => sum + (s?.segments.length ?? 0),
|
||||
0,
|
||||
),
|
||||
sleepStages: (main?.stages ?? {}) as Record<string, number>,
|
||||
inBedMinutes: main?.inBedMinutes ?? null,
|
||||
awakeMinutes: main?.awakeMinutes ?? null,
|
||||
napCount: naps.length,
|
||||
napAsleepMinutes: naps.reduce((sum, n) => sum + n.asleepMinutes, 0),
|
||||
awakenings: main?.awakenings ?? 0,
|
||||
};
|
||||
});
|
||||
|
||||
annotate({
|
||||
action: { name: "measurement.list" },
|
||||
meta: {
|
||||
total: page.total,
|
||||
type: "SLEEP_DURATION",
|
||||
mode: "sleep-per-night",
|
||||
nights: measurements.length,
|
||||
rowsScanned: rows.length,
|
||||
},
|
||||
});
|
||||
return apiSuccess({
|
||||
measurements,
|
||||
meta: { total: page.total, limit, offset, groupBy: "night" },
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
/**
|
||||
* Sleep nights read from Postgres, a page or a night at a time.
|
||||
*
|
||||
* Sleep is stored one row per stage, so a long history is a large row set:
|
||||
* every reader here keeps the stage rows that reach the process to the
|
||||
* nights it was asked about, and the reconstruction itself stays the one in
|
||||
* `sleep-night.ts`. The SQL below only decides which nights exist and where
|
||||
* their sessions lie; what a night holds is always the JS reconstruction's.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { Prisma } from "@/generated/prisma/client";
|
||||
import type { MeasurementSource, SleepStage } from "@/generated/prisma/client";
|
||||
import {
|
||||
ASLEEP_STAGES,
|
||||
SESSION_GAP_MS,
|
||||
reconstructSleepNights,
|
||||
reconstructSleepSessions,
|
||||
type SleepNight,
|
||||
type SleepSession,
|
||||
} from "@/lib/analytics/sleep-night";
|
||||
import { localDayWindow } from "@/lib/tz/local-day";
|
||||
import { pageWithWindowTotal } from "./paging";
|
||||
|
||||
/** A stage row as every sleep reader selects it. */
|
||||
export interface SleepRow {
|
||||
id: string;
|
||||
value: number;
|
||||
measuredAt: Date;
|
||||
sleepStage: SleepStage | null;
|
||||
source: MeasurementSource;
|
||||
deviceType: string | null;
|
||||
}
|
||||
|
||||
/** The read bounds a list request passes through, both inclusive. */
|
||||
export interface SleepReadBounds {
|
||||
from?: Date;
|
||||
to?: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* How far a one-night read reaches either side of its day. Sessions split at
|
||||
* a three-hour gap, so a session ending on that day starts inside the reach
|
||||
* unless it lasted over a day and a half, and every session the cut
|
||||
* truncates ends on another day.
|
||||
*/
|
||||
export const NIGHT_READ_REACH_MS = 36 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* The asleep stages as a SQL list. A whitelist splice: each value is asserted
|
||||
* against the closed enum shape before it is joined, so nothing from a
|
||||
* request can reach it, and the list is `ASLEEP_STAGES`, the very set the
|
||||
* JS reconstruction counts as asleep.
|
||||
*/
|
||||
const ASLEEP_STAGE_SQL = Prisma.raw(
|
||||
ASLEEP_STAGES.map((stage) => {
|
||||
if (!/^[A-Z_]+$/.test(stage)) throw new Error(`bad sleep stage ${stage}`);
|
||||
return `'${stage}'`;
|
||||
}).join(", "),
|
||||
);
|
||||
|
||||
/**
|
||||
* Stage rows between two instants (inclusive), oldest first, narrowed by the
|
||||
* request's own bounds.
|
||||
*/
|
||||
export async function readSleepRows(
|
||||
userId: string,
|
||||
gte: Date,
|
||||
lte: Date,
|
||||
bounds: SleepReadBounds = {},
|
||||
): Promise<SleepRow[]> {
|
||||
return prisma.measurement.findMany({
|
||||
where: {
|
||||
userId,
|
||||
type: "SLEEP_DURATION",
|
||||
deletedAt: null,
|
||||
AND: [
|
||||
{ measuredAt: { gte, lte } },
|
||||
...(bounds.from ? [{ measuredAt: { gte: bounds.from } }] : []),
|
||||
...(bounds.to ? [{ measuredAt: { lte: bounds.to } }] : []),
|
||||
],
|
||||
},
|
||||
orderBy: [{ measuredAt: "asc" }, { id: "asc" }],
|
||||
select: {
|
||||
id: true,
|
||||
value: true,
|
||||
measuredAt: true,
|
||||
sleepStage: true,
|
||||
source: true,
|
||||
// Writer-level collapse: two HealthKit apps behind one source (watch
|
||||
// stages vs phone in-bed) must not blend into one night.
|
||||
deviceType: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The sessions of one wake day: the stage rows within `NIGHT_READ_REACH_MS`
|
||||
* of the day, reconstructed, and the sessions filed under it kept.
|
||||
*/
|
||||
export async function readNightSessions(
|
||||
userId: string,
|
||||
day: string,
|
||||
tz: string,
|
||||
priorityJson: unknown,
|
||||
bounds: SleepReadBounds = {},
|
||||
): Promise<{ sessions: SleepSession[]; rows: SleepRow[] }> {
|
||||
const { dayStart, dayEnd } = localDayWindow(day, tz);
|
||||
const rows = await readSleepRows(
|
||||
userId,
|
||||
new Date(dayStart.getTime() - NIGHT_READ_REACH_MS),
|
||||
new Date(dayEnd.getTime() + NIGHT_READ_REACH_MS),
|
||||
bounds,
|
||||
);
|
||||
const sessions = reconstructSleepSessions(rows, tz, priorityJson).filter(
|
||||
(s) => s.night === day,
|
||||
);
|
||||
return { sessions, rows };
|
||||
}
|
||||
|
||||
/**
|
||||
* The wake day of the most recent night that holds sleep, or null. Found
|
||||
* from the latest asleep row: the night holding it is the newest one, and
|
||||
* its sessions lie within the one-night reach of that row.
|
||||
*/
|
||||
export async function latestSleepNight(
|
||||
userId: string,
|
||||
tz: string,
|
||||
priorityJson: unknown,
|
||||
): Promise<string | null> {
|
||||
const latest = await prisma.measurement.findFirst({
|
||||
where: {
|
||||
userId,
|
||||
type: "SLEEP_DURATION",
|
||||
deletedAt: null,
|
||||
value: { gt: 0 },
|
||||
OR: [{ sleepStage: null }, { sleepStage: { in: [...ASLEEP_STAGES] } }],
|
||||
},
|
||||
orderBy: { measuredAt: "desc" },
|
||||
select: { measuredAt: true },
|
||||
});
|
||||
if (!latest) return null;
|
||||
const at = latest.measuredAt.getTime();
|
||||
const rows = await readSleepRows(
|
||||
userId,
|
||||
new Date(at - NIGHT_READ_REACH_MS),
|
||||
new Date(at + NIGHT_READ_REACH_MS),
|
||||
);
|
||||
const scorable = reconstructSleepSessions(rows, tz, priorityJson).filter(
|
||||
(s) => s.asleepMinutes > 0,
|
||||
);
|
||||
return scorable.at(-1)?.night ?? null;
|
||||
}
|
||||
|
||||
/** One night of a page: its wake day and the span its sessions cover. */
|
||||
export interface SleepNightSpan {
|
||||
night: string;
|
||||
spanStart: Date;
|
||||
spanEnd: Date;
|
||||
}
|
||||
|
||||
/**
|
||||
* One page of sleep nights, chosen in Postgres.
|
||||
*
|
||||
* The stage rows are cut into sessions the way `reconstructSleepSessions`
|
||||
* cuts them: ordered by start (`measured_at` minus the minutes), a new
|
||||
* session wherever a row starts more than `SESSION_GAP_MS` after every
|
||||
* earlier row has ended. A session counts when it holds an asleep row (an
|
||||
* `ASLEEP_STAGES` stage, or a stage-less row, with minutes), and its night is
|
||||
* the local day its last row ends on: the reconstruction's own rule, so a
|
||||
* page night and a reconstructed night name the same day. Nights are grouped,
|
||||
* sorted and cut by `limit` / `offset`; `total` counts the whole filtered set.
|
||||
* `spanStart` / `spanEnd` bound the night's counting sessions, so the caller
|
||||
* reads exactly their rows.
|
||||
*
|
||||
* Before the cut, rows are folded into hour buckets by their end: min start,
|
||||
* max end, any asleep. That cannot move a session boundary. Two rows whose
|
||||
* ends lie less than the session gap apart always share a session (the later
|
||||
* one starts before its own end, which is within the gap of the earlier end),
|
||||
* so a bucket narrower than the gap only ever holds rows of one session, and
|
||||
* its bounds are that group's bounds. It cuts the rows the window functions
|
||||
* sort from every stage to a handful per night.
|
||||
*/
|
||||
export async function sleepNightPage(
|
||||
userId: string,
|
||||
tz: string,
|
||||
opts: SleepReadBounds & {
|
||||
sortDir: "asc" | "desc";
|
||||
limit: number;
|
||||
offset: number;
|
||||
},
|
||||
): Promise<{ nights: SleepNightSpan[]; total: number }> {
|
||||
const gapSeconds = SESSION_GAP_MS / 1000;
|
||||
// Direction comes from the Zod enum, never from raw input.
|
||||
const direction = Prisma.raw(opts.sortDir === "asc" ? "ASC" : "DESC");
|
||||
const nightsCte = Prisma.sql`
|
||||
WITH r AS (
|
||||
SELECT
|
||||
MIN(m."measured_at" - m."value" * interval '1 minute') AS s,
|
||||
MAX(m."measured_at") AS e,
|
||||
BOOL_OR(m."value" > 0 AND (m."sleep_stage" IS NULL
|
||||
OR m."sleep_stage" IN (${ASLEEP_STAGE_SQL}))) AS asleep
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = ${userId}
|
||||
AND m."type" = 'SLEEP_DURATION'::measurement_type
|
||||
AND m."deleted_at" IS NULL
|
||||
${opts.from ? Prisma.sql`AND m."measured_at" >= ${opts.from}` : Prisma.empty}
|
||||
${opts.to ? Prisma.sql`AND m."measured_at" <= ${opts.to}` : Prisma.empty}
|
||||
GROUP BY date_bin(interval '1 hour', m."measured_at", TIMESTAMPTZ '2000-01-01 00:00:00+00')
|
||||
), o AS (
|
||||
SELECT r.*, MAX(e) OVER (
|
||||
ORDER BY s, e ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING
|
||||
) AS prev_end
|
||||
FROM r
|
||||
), g AS (
|
||||
SELECT o.*, SUM(
|
||||
CASE WHEN prev_end IS NULL
|
||||
OR s - prev_end > make_interval(secs => ${gapSeconds}::double precision) THEN 1 ELSE 0 END
|
||||
) OVER (ORDER BY s, e ROWS UNBOUNDED PRECEDING) AS sid
|
||||
FROM o
|
||||
), sess AS (
|
||||
SELECT MIN(s) AS s, MAX(e) AS e
|
||||
FROM g
|
||||
GROUP BY sid
|
||||
HAVING BOOL_OR(asleep)
|
||||
), nights AS (
|
||||
SELECT
|
||||
to_char((e AT TIME ZONE 'UTC') AT TIME ZONE ${tz}, 'YYYY-MM-DD') AS night,
|
||||
MIN(s) AS span_start,
|
||||
MAX(e) AS span_end
|
||||
FROM sess
|
||||
GROUP BY 1
|
||||
)
|
||||
`;
|
||||
const { rows, total } = await pageWithWindowTotal(
|
||||
await prisma.$queryRaw<
|
||||
Array<{ night: string; span_start: Date; span_end: Date; total: number }>
|
||||
>`
|
||||
${nightsCte}
|
||||
SELECT night, span_start, span_end, (COUNT(*) OVER ())::int AS total
|
||||
FROM nights
|
||||
ORDER BY night ${direction}
|
||||
LIMIT ${opts.limit} OFFSET ${opts.offset}
|
||||
`,
|
||||
opts.offset,
|
||||
async () =>
|
||||
Number(
|
||||
(
|
||||
await prisma.$queryRaw<Array<{ total: number }>>`
|
||||
${nightsCte}
|
||||
SELECT COUNT(*)::int AS total FROM nights
|
||||
`
|
||||
)[0]?.total ?? 0,
|
||||
),
|
||||
);
|
||||
return {
|
||||
nights: rows.map((r) => ({
|
||||
night: r.night,
|
||||
spanStart: r.span_start,
|
||||
spanEnd: r.span_end,
|
||||
})),
|
||||
total,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The stage rows of a set of page nights, read in one span. The nights are
|
||||
* consecutive in the sorted night set, so the span from the first one's start
|
||||
* to the last one's end holds their sessions whole; any other session in it
|
||||
* holds no sleep.
|
||||
*/
|
||||
export async function readNightSpanRows(
|
||||
userId: string,
|
||||
nights: readonly SleepNightSpan[],
|
||||
bounds: SleepReadBounds = {},
|
||||
): Promise<SleepRow[]> {
|
||||
if (nights.length === 0) return [];
|
||||
let spanStart = nights[0].spanStart;
|
||||
let spanEnd = nights[0].spanEnd;
|
||||
for (const n of nights) {
|
||||
if (n.spanStart < spanStart) spanStart = n.spanStart;
|
||||
if (n.spanEnd > spanEnd) spanEnd = n.spanEnd;
|
||||
}
|
||||
return readSleepRows(userId, spanStart, spanEnd, bounds);
|
||||
}
|
||||
|
||||
/** How many nights one chunked read reconstructs at a time. */
|
||||
const NIGHT_CHUNK = 90;
|
||||
|
||||
/**
|
||||
* Every night in a window as `reconstructSleepNights` builds it, oldest
|
||||
* first, without holding the window's stage rows at once: Postgres names the
|
||||
* nights and their spans, and the rows are read and reconstructed ninety
|
||||
* nights at a time. Any window length costs a bounded heap, so a reader never
|
||||
* has to cut a long range short to stay safe. `maxNights` keeps the oldest
|
||||
* nights of the window, or with `newest` the most recent ones.
|
||||
*/
|
||||
export async function readSleepNights(
|
||||
userId: string,
|
||||
tz: string,
|
||||
priorityJson: unknown,
|
||||
bounds: SleepReadBounds & { maxNights: number; newest?: boolean },
|
||||
): Promise<SleepNight[]> {
|
||||
const page = await sleepNightPage(userId, tz, {
|
||||
from: bounds.from,
|
||||
to: bounds.to,
|
||||
sortDir: bounds.newest ? "desc" : "asc",
|
||||
limit: bounds.maxNights,
|
||||
offset: 0,
|
||||
});
|
||||
const nights = bounds.newest ? page.nights.reverse() : page.nights;
|
||||
const out: SleepNight[] = [];
|
||||
for (let i = 0; i < nights.length; i += NIGHT_CHUNK) {
|
||||
const chunk = nights.slice(i, i + NIGHT_CHUNK);
|
||||
const keys = new Set(chunk.map((n) => n.night));
|
||||
const rows = await readNightSpanRows(userId, chunk, bounds);
|
||||
for (const night of reconstructSleepNights(rows, tz, priorityJson)) {
|
||||
if (keys.has(night.night)) out.push(night);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -23,7 +23,9 @@ import { walkSourceFiles } from "@/__tests__/helpers/source-files";
|
||||
|
||||
import {
|
||||
HEAP_CHECK_INTERVAL_MS,
|
||||
HEAP_CRITICAL_RATIO,
|
||||
HEAP_PRESSURE_COOLDOWN_MS,
|
||||
HEAP_PRESSURE_RATIO,
|
||||
HEAP_PRESSURE_LIST_MAX,
|
||||
HEAP_SAMPLE_INTERVAL_ELEVATED_MS,
|
||||
HEAP_SAMPLE_INTERVAL_MS,
|
||||
@@ -175,6 +177,41 @@ describe("heap pressure report", () => {
|
||||
expect(h.named(HEAP_TASK_NAMES.pressure)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("reports again at 95 % inside the cooldown, once per climb", () => {
|
||||
const h = harness();
|
||||
h.state.used = 900;
|
||||
h.run(60_000);
|
||||
// Ten minutes later, well inside the cooldown, the heap nears its limit.
|
||||
h.state.used = 800;
|
||||
h.run(10 * 60_000);
|
||||
h.state.used = 960;
|
||||
h.run(60_000);
|
||||
const pressure = h.named(HEAP_TASK_NAMES.pressure);
|
||||
expect(pressure.map((p) => p.meta.pressure_ratio)).toEqual([
|
||||
HEAP_PRESSURE_RATIO,
|
||||
HEAP_CRITICAL_RATIO,
|
||||
]);
|
||||
expect(pressure[1].meta).toHaveProperty("in_flight_oldest");
|
||||
|
||||
// Hovering at 95 % writes nothing more; back under 85 % re-arms it.
|
||||
h.run(5 * 60_000);
|
||||
expect(h.named(HEAP_TASK_NAMES.pressure)).toHaveLength(2);
|
||||
h.state.used = 840;
|
||||
h.run(HEAP_CHECK_INTERVAL_MS);
|
||||
h.state.used = 970;
|
||||
h.run(HEAP_CHECK_INTERVAL_MS);
|
||||
expect(h.named(HEAP_TASK_NAMES.pressure)).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("writes one report at the tier a sudden jump reaches", () => {
|
||||
const h = harness();
|
||||
h.state.used = 990;
|
||||
h.run(60_000);
|
||||
const pressure = h.named(HEAP_TASK_NAMES.pressure);
|
||||
expect(pressure).toHaveLength(1);
|
||||
expect(pressure[0].meta.pressure_ratio).toBe(HEAP_CRITICAL_RATIO);
|
||||
});
|
||||
|
||||
it("writes no snapshot unless opted in, and at most one when it is", () => {
|
||||
const off = harness();
|
||||
off.state.used = 900;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
import { envFlag, envOr } from "@/lib/env";
|
||||
import { WideEventBuilder } from "@/lib/logging/event-builder";
|
||||
import { emitIfSampled } from "@/lib/logging/transports";
|
||||
import { megabytes } from "@/lib/support/heap-limit";
|
||||
|
||||
import { inFlightRegistry, type InFlightRegistry } from "./in-flight";
|
||||
|
||||
@@ -32,7 +33,11 @@ import { inFlightRegistry, type InFlightRegistry } from "./in-flight";
|
||||
* flight and the routes most of them share (see `in-flight.ts`). It
|
||||
* re-arms only after the heap has fallen back under 70 % AND thirty minutes
|
||||
* have passed, so a heap hovering at the line writes one report, not one
|
||||
* every check.
|
||||
* every check. A second tier at 95 % writes the same report with no
|
||||
* cooldown at all, and re-arms as soon as the heap is back under 85 %:
|
||||
* a heap that climbs to its limit within the half hour after an 85 %
|
||||
* report would otherwise die with only that older report on record, the
|
||||
* requests that pushed it over missing from it.
|
||||
* - `runtime.heap.snapshot`, at `warn`, only with the opt-in below: where
|
||||
* the snapshot was written, or that writing it failed.
|
||||
*
|
||||
@@ -63,6 +68,7 @@ export const HEAP_SAMPLE_INTERVAL_MS = 5 * 60_000;
|
||||
export const HEAP_SAMPLE_INTERVAL_ELEVATED_MS = 60_000;
|
||||
export const HEAP_ELEVATED_RATIO = 0.7;
|
||||
export const HEAP_PRESSURE_RATIO = 0.85;
|
||||
export const HEAP_CRITICAL_RATIO = 0.95;
|
||||
export const HEAP_PRESSURE_COOLDOWN_MS = 30 * 60_000;
|
||||
export const HEAP_PRESSURE_LIST_MAX = 50;
|
||||
|
||||
@@ -101,10 +107,6 @@ export interface HeapMonitorDeps {
|
||||
writeSnapshot: () => SnapshotOutcome;
|
||||
}
|
||||
|
||||
function megabytes(bytes: number): number {
|
||||
return Math.round(bytes / 1024 / 1024);
|
||||
}
|
||||
|
||||
function defaultReadHeap(): { used: number; limit: number } {
|
||||
const stats = getHeapStatistics();
|
||||
return { used: stats.used_heap_size, limit: stats.heap_size_limit };
|
||||
@@ -205,6 +207,7 @@ export function writeDiagnosticHeapSnapshot(): SnapshotOutcome {
|
||||
export class HeapMonitor {
|
||||
private lastSampleAt: number | null = null;
|
||||
private pressureArmed = true;
|
||||
private criticalArmed = true;
|
||||
private lastPressureAt: number | null = null;
|
||||
private snapshotTaken = false;
|
||||
|
||||
@@ -225,11 +228,29 @@ export class HeapMonitor {
|
||||
this.pressureArmed = true;
|
||||
}
|
||||
|
||||
if (!this.criticalArmed && ratio < HEAP_PRESSURE_RATIO) {
|
||||
this.criticalArmed = true;
|
||||
}
|
||||
|
||||
if (this.pressureArmed && ratio >= HEAP_PRESSURE_RATIO) {
|
||||
this.pressureArmed = false;
|
||||
this.lastPressureAt = at;
|
||||
this.lastSampleAt = at;
|
||||
this.reportPressure(at);
|
||||
// A heap that jumps straight past 95 % gets one report, at the tier
|
||||
// it reached.
|
||||
const critical = ratio >= HEAP_CRITICAL_RATIO;
|
||||
if (critical) this.criticalArmed = false;
|
||||
this.reportPressure(
|
||||
at,
|
||||
critical ? HEAP_CRITICAL_RATIO : HEAP_PRESSURE_RATIO,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.criticalArmed && ratio >= HEAP_CRITICAL_RATIO) {
|
||||
this.criticalArmed = false;
|
||||
this.lastSampleAt = at;
|
||||
this.reportPressure(at, HEAP_CRITICAL_RATIO);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -245,7 +266,7 @@ export class HeapMonitor {
|
||||
}
|
||||
}
|
||||
|
||||
private reportPressure(at: number): void {
|
||||
private reportPressure(at: number, tier: number): void {
|
||||
const { registry, emit, readFigures } = this.deps;
|
||||
const inFlight = registry.report(at, HEAP_PRESSURE_LIST_MAX);
|
||||
const willSnapshot = this.deps.snapshotEnabled && !this.snapshotTaken;
|
||||
@@ -253,7 +274,7 @@ export class HeapMonitor {
|
||||
// process over, the report is already on stdout.
|
||||
emit(HEAP_TASK_NAMES.pressure, "warn", {
|
||||
...readFigures(),
|
||||
pressure_ratio: HEAP_PRESSURE_RATIO,
|
||||
pressure_ratio: tier,
|
||||
in_flight_oldest: inFlight.oldest,
|
||||
in_flight_by_route: inFlight.by_route,
|
||||
snapshot: willSnapshot ? "pending" : "disabled",
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
* to the derived contract cannot leave this document behind.
|
||||
*/
|
||||
import { z } from "zod/v4";
|
||||
import { recoveryEmptySchema } from "./recovery-empty";
|
||||
import type { ZodOpenApiObject } from "zod-openapi";
|
||||
|
||||
import { SCORE_PILLAR_IDS } from "@/lib/analytics/score/types";
|
||||
@@ -140,11 +141,12 @@ const analyticsSummariesSlice = z
|
||||
),
|
||||
lastSeenByType,
|
||||
sleepSourceDiscrepancy,
|
||||
recoveryEmpty: recoveryEmptySchema,
|
||||
})
|
||||
.meta({
|
||||
id: "AnalyticsSummariesSlice",
|
||||
description:
|
||||
"What `?slice=summaries` answers: the per-type summaries and their freshness, and nothing thick. Four keys, and the caller gets them from two SQL passes rather than the default slice's chain.",
|
||||
"What `?slice=summaries` answers: the per-type summaries and their freshness, the recovery page's empty reason, and nothing thick. Five keys, and the caller gets them from two SQL passes rather than the default slice's chain.",
|
||||
});
|
||||
|
||||
// ── Correlations ─────────────────────────────────────────────────────
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
* Schemas come from `src/lib/validations/*` where shared with the
|
||||
* runtime request parsing, so the wire contract stays single-source.
|
||||
*/
|
||||
import { recoveryEmptySchema } from "../recovery-empty";
|
||||
import { trendSentiment, trendVerdict } from "../trend-verdict";
|
||||
import { z } from "zod/v4";
|
||||
import { measurementSourceEnum, measurementTypeEnum } from "../shared";
|
||||
@@ -324,7 +325,7 @@ export const sleepScoreValue = z
|
||||
.enum(["few_nights", "no_main_sleep"])
|
||||
.optional()
|
||||
.describe(
|
||||
"Only on a null `consistency` / `timing`: why it is withheld. `few_nights` — fewer than three nights in the window have a main sleep on the clock (count in `rhythmNights`); `no_main_sleep` — the latest day had only a daytime nap. Absent otherwise.",
|
||||
"Only on a null `consistency` / `timing`: why it is withheld. `few_nights` — fewer than `rhythmMinNights` nights in the window have a main sleep on the clock (count in `rhythmNights`); `no_main_sleep` — the latest day had only a daytime nap. Absent otherwise.",
|
||||
),
|
||||
}),
|
||||
)
|
||||
@@ -336,6 +337,12 @@ export const sleepScoreValue = z
|
||||
.describe(
|
||||
"Nights in the window with a main sleep placed on the clock — what consistency and timing are computed from. A daytime nap is not one; a short night is.",
|
||||
),
|
||||
rhythmMinNights: z
|
||||
.number()
|
||||
.int()
|
||||
.describe(
|
||||
'How many `rhythmNights` consistency and timing need before they are scored; below it they carry `gap: "few_nights"`.',
|
||||
),
|
||||
series: z
|
||||
.array(z.number())
|
||||
.describe(
|
||||
@@ -2032,6 +2039,7 @@ export const analyticsSummariesSliceResponse = z
|
||||
),
|
||||
lastSeenByType: z.record(z.string(), lastSeenSlot),
|
||||
sleepSourceDiscrepancy,
|
||||
recoveryEmpty: recoveryEmptySchema,
|
||||
})
|
||||
.meta({
|
||||
id: "AnalyticsSummariesSlice",
|
||||
|
||||
@@ -465,7 +465,7 @@ const sleepNightResponse = z
|
||||
.meta({
|
||||
id: "SleepNightResource",
|
||||
description:
|
||||
"One reconstructed sleep night: the main session's hypnogram segments + breakdown, plus same-wake-day naps surfaced separately.",
|
||||
"One reconstructed sleep night: the main session's hypnogram segments + breakdown, plus same-wake-day naps surfaced separately. A session is a nap when it holds under three hours asleep centred between 10:00 and 20:00 local time; the main session is the longest session that is not a nap. A day whose only sleep is naps has `main: null` and its naps.",
|
||||
});
|
||||
|
||||
// ── Sleep rhythm (sleep-debt + chronotype, v1.17.0) ──────────────────
|
||||
@@ -804,7 +804,7 @@ export const measurementPaths: NonNullable<ZodOpenApiObject["paths"]> = {
|
||||
summary: "List measurements",
|
||||
description:
|
||||
"Filter by type + date range. Response is paged via offset/limit; default limit 100.\n\n" +
|
||||
"With `groupBy=day` on a cumulative type the rows are one sum per local day, and `limit` / `offset` page over those days; every returned day is complete. `type=SLEEP_DURATION` returns one row per night, paged the same way over nights. On both, `meta.total` counts the days or nights in the filtered window. A day-sum page takes `limit` as asked (up to the list's 5000); a sleep page holds at most 1000 nights, and `meta.limit` echoes the value applied.",
|
||||
"With `groupBy=day` on a cumulative type the rows are one sum per local day, and `limit` / `offset` page over those days; every returned day is complete, which is why `valueMin` / `valueMax` (a filter on single readings) are refused with 422 beside `groupBy=day`. `type=SLEEP_DURATION` returns one row per night, paged the same way over nights; every night of the page has a row, and a day whose only sleep is daytime naps carries `value` 0 with its naps in `napCount` / `napAsleepMinutes`. Its `dayKey` is the same day `/api/sleep/night?date=` answers for. On both, `meta.total` counts the days or nights in the filtered window. A day-sum page takes `limit` as asked (up to the list's 5000); a sleep page holds at most 1000 nights, and `meta.limit` echoes the value applied.",
|
||||
requestParams: {
|
||||
query: listMeasurementsSchema,
|
||||
},
|
||||
@@ -1118,7 +1118,7 @@ export const measurementPaths: NonNullable<ZodOpenApiObject["paths"]> = {
|
||||
tags: ["Measurements"],
|
||||
summary: "Reconstructed sleep night (hypnogram source, v1.11.5)",
|
||||
description:
|
||||
"Returns one night's reconstructed sleep session for the phase-progression (hypnogram) view: the canonical source's stage segments (each with an absolute start/end), the per-stage breakdown, asleep/in-bed/awake totals, the mid-sleep awakenings count, and same-wake-day naps surfaced separately. Session-clustered, keyed by the local wake day, collapsed to one source via the sleep priority ladder so two sources never overlay. `date` omitted returns the most recent night. A read-only view over existing per-stage SLEEP_DURATION rows — no schema, no new measurement type.",
|
||||
"Returns one night's reconstructed sleep session for the phase-progression (hypnogram) view: the canonical source's stage segments (each with an absolute start/end), the per-stage breakdown, asleep/in-bed/awake totals, the mid-sleep awakenings count, and same-wake-day naps surfaced separately. Session-clustered, keyed by the local wake day, collapsed to one source via the sleep priority ladder so two sources never overlay. `date` omitted returns the most recent night holding sleep, however old. Any past date is answered, with no window cut. A read-only view over existing per-stage SLEEP_DURATION rows — no schema, no new measurement type.",
|
||||
requestParams: {
|
||||
query: z.object({
|
||||
date: z
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import { z } from "zod/v4";
|
||||
import { RECOVERY_EMPTY_REASONS } from "@/lib/insights/recovery-empty";
|
||||
|
||||
/** Why the recovery page is empty; shared by both slim-slice schemas. */
|
||||
export const recoveryEmptySchema = z
|
||||
.enum(RECOVERY_EMPTY_REASONS)
|
||||
.nullable()
|
||||
.describe(
|
||||
"Why the recovery page has nothing to chart, resolved on the server; null when any recovery type (ANS_CHARGE, DAY_STRAIN, WORKOUT_STRAIN, CARDIO_LOAD, AVERAGE_HEART_RATE, MAX_HEART_RATE, ENERGY_EXPENDITURE_KJ, RESILIENCE) has a reading. `oura_permission`: Oura is connected and its grant leaves out the scope the resilience level needs, one reconnect fixes it. `oura_no_resilience`: Oura is connected and no level has arrived yet. `general`: anyone else. Clients render the empty note from this value and never work it out from the connection status.",
|
||||
);
|
||||
@@ -14,9 +14,14 @@ import { getHeapStatistics } from "node:v8";
|
||||
/** Below this, the heap is flagged. Between the 1g (524) and 1536m (792) heaps. */
|
||||
export const LOW_HEAP_LIMIT_MB = 700;
|
||||
|
||||
/** Bytes as whole megabytes (MiB), the unit every heap figure is shown in. */
|
||||
export function megabytes(bytes: number): number {
|
||||
return Math.round(bytes / 1024 / 1024);
|
||||
}
|
||||
|
||||
/** V8's heap limit for this process, in megabytes. */
|
||||
export function heapLimitMb(): number {
|
||||
return Math.round(getHeapStatistics().heap_size_limit / 1024 / 1024);
|
||||
return megabytes(getHeapStatistics().heap_size_limit);
|
||||
}
|
||||
|
||||
export function isHeapLimitLow(limitMb: number): boolean {
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
heapLimitMb,
|
||||
isHeapLimitLow,
|
||||
lowHeapLimitAdvice,
|
||||
megabytes,
|
||||
} from "@/lib/support/heap-limit";
|
||||
import { shouldEmitSecureCookie } from "@/lib/auth/secure-cookie";
|
||||
import { isOidcConfigured, isOidcOnly } from "@/lib/auth/oidc";
|
||||
@@ -692,8 +693,8 @@ function or(value: string | number | null | undefined, fallback = "unknown") {
|
||||
: String(value);
|
||||
}
|
||||
|
||||
function megabytes(bytes: number | null): string {
|
||||
return bytes === null ? "unknown" : `${Math.round(bytes / 1024 / 1024)} MB`;
|
||||
function sizeLine(bytes: number | null): string {
|
||||
return bytes === null ? "unknown" : `${megabytes(bytes)} MB`;
|
||||
}
|
||||
|
||||
function runLine(run: QueueRunFact | null): string {
|
||||
@@ -741,7 +742,7 @@ export function renderSupportReport(facts: SupportReportFacts): string {
|
||||
section("Database");
|
||||
row("PostgreSQL", facts.database.version);
|
||||
row("Server time zone", facts.database.timezone);
|
||||
row("Size", megabytes(facts.database.sizeBytes));
|
||||
row("Size", sizeLine(facts.database.sizeBytes));
|
||||
row(
|
||||
"Last migration",
|
||||
facts.database.lastMigration
|
||||
|
||||
@@ -589,7 +589,7 @@ export const listMeasurementsSchema = z
|
||||
// Omitting `aggregate` keeps the raw wire shape (iOS contract); the
|
||||
// chart-data client must opt in explicitly.
|
||||
aggregate: z.enum(["raw", "daily", "weekly", "monthly"]).optional(),
|
||||
// v1.4.36 W1 — opt-in source switch for daily-aggregate reads. When
|
||||
// v1.4.36 — opt-in source switch for daily-aggregate reads. When
|
||||
// `source=rollup` + `aggregate=daily`, the route reads from the
|
||||
// persistent `measurement_rollups` DAY buckets instead of running a
|
||||
// live `date_trunc` GROUP BY scan over the raw `measurements` table.
|
||||
@@ -612,7 +612,7 @@ export const listMeasurementsSchema = z
|
||||
// `value: { gte, lte }` range. Server-authoritative; validated min<=max.
|
||||
valueMin: z.coerce.number().finite().optional(),
|
||||
valueMax: z.coerce.number().finite().optional(),
|
||||
// v1.4.37 W7c — list-view "one row per day" mode for cumulative
|
||||
// v1.4.37 — list-view "one row per day" mode for cumulative
|
||||
// types (steps, active energy, distance, flights, daylight). When
|
||||
// `groupBy=day` is set and `type` is a cumulative HK type, the route
|
||||
// returns one synthesised row per user-TZ day with `value` = SUM
|
||||
@@ -623,18 +623,18 @@ export const listMeasurementsSchema = z
|
||||
// picks the page's local days in Postgres and reports the distinct-day
|
||||
// count as `meta.total` (#1245).
|
||||
groupBy: z.enum(["day"]).optional(),
|
||||
// v1.4.37 W7c — drill-down to per-sample rows for a single day in
|
||||
// v1.4.37 — drill-down to per-sample rows for a single day in
|
||||
// the user's IANA timezone. Format `YYYY-MM-DD`; the route resolves
|
||||
// the day boundary against the user's `User.timezone`. Used by the
|
||||
// expandable list row to reveal the chunks that contributed to the
|
||||
// collapsed daily total.
|
||||
//
|
||||
// v1.4.37 W10 — a drill-down returns one bounded page rather than a
|
||||
// v1.4.37 — a drill-down returns one bounded page rather than a
|
||||
// cursor, so it refuses a non-zero `offset`.
|
||||
dayKey: z
|
||||
.string()
|
||||
.regex(/^\d{4}-\d{2}-\d{2}$/, "dayKey must be YYYY-MM-DD")
|
||||
// v1.4.37 W10 — a `YYYY-MM-DD` string can satisfy the regex while
|
||||
// v1.4.37 — a `YYYY-MM-DD` string can satisfy the regex while
|
||||
// still being an impossible calendar date (`2026-02-30`,
|
||||
// `2026-13-01`). `new Date("2026-02-30T00:00:00Z")` silently
|
||||
// overflows to March 2, so the drill-down would return rows from
|
||||
@@ -665,6 +665,19 @@ export const listMeasurementsSchema = z
|
||||
"limit must be <= 1000 when dayKey is set; drill-down responses are capped at 1000 rows",
|
||||
path: ["limit"],
|
||||
})
|
||||
// A value range filters single readings, and a day-grouped row is the sum
|
||||
// of a whole day: filtering its readings would return partial days summed
|
||||
// as if they were whole. The combination has no honest answer, so it is
|
||||
// refused rather than served.
|
||||
.refine(
|
||||
({ groupBy, valueMin, valueMax }) =>
|
||||
groupBy !== "day" || (valueMin == null && valueMax == null),
|
||||
{
|
||||
message:
|
||||
"valueMin / valueMax filter single readings and cannot be combined with groupBy=day",
|
||||
path: ["groupBy"],
|
||||
},
|
||||
)
|
||||
// v1.18.5 — an inverted value range (`valueMin > valueMax`) is a client
|
||||
// bug, not an empty result set; reject it at the validator with a 422 so
|
||||
// the filter UI can surface the mismatch instead of silently returning
|
||||
|
||||
Reference in new issue
Block a user