Merge remote-tracking branch 'origin/main' into feat/coach-lookback-limit

# Conflicts:
#	src/lib/ai/coach/snapshot.ts
#	src/lib/ai/coach/tools/executor.ts
#	src/lib/insights/features.ts
This commit is contained in:
Marc-André Bombeck committed 2026-10-05 07:31:03 +02:00
commit 929c0ebaa4
59 files changed
+3433 -1470

No files matched your search

+40
View File
@@ -1,5 +1,45 @@
# Changelog
## [1.40.1] — 2026-10-04
Restore opens older encrypted values again; faster pages on a long-used
server; a day of pulse is the mean of its hours; Apple Health imports leave
out HealthLog's own samples; more memory for the app by default.
### Fixed
- **Restore key probe.** A legacy unprefixed ciphertext was read as a base64
string and refused a restore the key opens; legacy values open as strings,
probes come only from sections that count, binary probes use their AAD.
The drill checks every account of the newest date.
- **Daily pulse (#1107, migration 0372).** A day is the mean of its local
hours' means and a window the mean of its days, in every reader; stored
pulse rollups are recomputed. Count, min and max stay over every reading.
- **Apple Health import (#1109).** Records carrying HealthLog's origin
marker or a HealthLog id are left out and counted; cycle samples too.
- **Blood pressure series.** One source per day for both values.
- **Backups.** Heartbeat on both backup jobs; per-account attempts recorded
(migration 0373), interrupted accounts go last; the preview is built from
the writer's counts.
- **Document previews.** A permanent render failure is recorded (migration 0374) instead of re-queued at every start.
- **Umami.** Events from a tracker built with a custom endpoint are proxied
to the configured Umami host.
- **Display.** Rest Mode shows its start date; the favicon is transparent;
the sidebar no longer flashes scrollbars while it expands.
### Changed
- **Performance.** One Prisma client per process instead of one per bundle;
the rollup coverage probe uses a loose index scan and concurrent readers
share one probe; timezone offsets are memoised per hour; chart text is
measured outside `<body>`; Insights mounts lower sections near the
viewport; a nav prefetch is not re-sent on mount; MCP single-metric reads
build only the requested metric; the backup schema loads on first use.
- **Lab list (#1057).** The range bar and name use a wide card.
- **Compose.** `APP_MEMORY_LIMIT` defaults to `1536m` (792 MB heap).
- Dependencies: next 16.3.8, eslint-config-next 16.3.8, zod-openapi 6.0.2,
testcontainers 12.2.0, p-limit 7.3.3.
## [1.40.0] — 2026-10-04
Back up the encryption key and refuse a key that cannot open the data;
+1 -1
View File
@@ -1,7 +1,7 @@
openapi: 3.1.0
info:
title: HealthLog API
version: 1.40.0
version: 1.40.1
description: >-
Self-hosted personal-health-tracking PWA — public API surface for the iOS native client and external ingest.
+6 -1
View File
@@ -13,7 +13,12 @@ already installed, longer if a TLS-fronted public hostname is in scope
unchanged.
- **2 GB RAM, 10 GB free disk.** The Postgres data volume grows with
measurement history; the Apple Health import worker can briefly
hold a 1.5 GB upload in `/tmp` while it parses.
hold a 1.5 GB upload in `/tmp` while it parses. The app container is
capped at `APP_MEMORY_LIMIT`, `1536m` by default (Node then sizes its
heap at about 792 MB). On a 2 GB host that leaves too little for
Postgres and the system, so set `APP_MEMORY_LIMIT=1g` in `.env`
(a heap of about 524 MB); a long-used account then runs closer to
that heap, and the weekly backup with it.
- **Outbound HTTPS** to `ghcr.io` (image pulls) plus whichever
integration endpoints you plan to enable (Withings, OpenAI, etc.).
The opt-in environmental-context module additionally reaches
+5 -1
View File
@@ -163,7 +163,11 @@ the knob as `DB_CONNECTION_LIMIT`, baked into `DATABASE_URL` as
`connection_limit`; `DATABASE_POOL_MAX` remains a working explicit
override. Resolution order: `DB_CONNECTION_LIMIT`, then
`DATABASE_POOL_MAX`, then the `connection_limit` URL parameter, then
the default of 20.
the default of 20. In the bundled single-container setup
(`HEALTHLOG_PROCESS_TYPE=all`) the web requests and the background
worker run in one process and share that one Prisma pool, so a budget
sized for the web traffic alone leaves the worker's jobs queueing for
the same connections.
Plan total Postgres slots as **container_count × budget**.
A stock Postgres 16 container ships with `max_connections = 100`, so
+12 -5
View File
@@ -25,6 +25,7 @@ import {
seedStoredBriefing,
} from "./setup/ai-optional-fixture";
import { expect, test } from "./setup/test";
import { revealDeferredSectionsSettled } from "./utils/deferred-sections";
test.use({ storageState: AI_OPTIONAL_STORAGE_STATE_PATH });
test.describe.configure({ mode: "serial" });
@@ -36,12 +37,18 @@ test.afterAll(async () => {
await resetAiOptionalAccount();
});
/**
* The overview with every section mounted. Sections past the first three
* render as an empty sentinel until they near the viewport, so a count of
* zero taken before they mount would pass whatever they paint.
*/
async function openOverview(page: Page): Promise<void> {
await page.goto("/insights");
await expect(page.locator('[data-slot="insights-hero-strip"]')).toBeVisible({
timeout: 20_000,
});
await page.waitForLoadState("networkidle");
await revealDeferredSectionsSettled(page);
}
test("J5: withdrawing AI consent deletes the notes written under it", async ({
@@ -80,12 +87,12 @@ test("J4: the Coach memory stays readable and deletable with the Coach hidden",
const title = `Stored conversation ${Date.now()}`;
await seedHiddenCoachWithConversation(title);
// Hide Coach takes the Coach page and its launcher away...
await page.goto("/insights");
await expect(page.locator('[data-slot="insights-hero-strip"]')).toBeVisible({
timeout: 20_000,
});
// Hide Coach takes the Coach page, its launcher and every per-card
// hand-off away...
await openOverview(page);
await expect(page.locator('[data-slot="coach-fab"]')).toHaveCount(0);
await expect(page.locator('[data-slot="ask-coach-action"]')).toHaveCount(0);
await expect(page.locator('[data-slot="ask-coach-icon"]')).toHaveCount(0);
// ...and Settings, AI lists what the Coach stored.
await page.goto("/settings/ai");
+13
View File
@@ -21,9 +21,19 @@ import type { Page, Request, Response } from "@playwright/test";
import { STORAGE_STATE_PATH } from "./setup/global-setup";
import { aiBlockUnavailable, serveAiBlock } from "./setup/ai-capabilities";
import { expect, test } from "./setup/test";
import { revealDeferredSectionsSettled } from "./utils/deferred-sections";
test.use({ storageState: STORAGE_STATE_PATH });
/**
* Mount every overview section, then let their reads land. Sections past the
* first three render as an empty sentinel until they near the viewport, so a
* count of zero taken before this would pass whatever those sections paint.
*/
async function mountWholeOverview(page: Page): Promise<void> {
await revealDeferredSectionsSettled(page);
}
/** Routes that call a model or serve model-written text. */
const AI_ROUTE = new RegExp(
[
@@ -89,6 +99,7 @@ test.describe("J1: AI switched off by the operator", () => {
);
// Let every section's reads land before judging what painted.
await page.waitForLoadState("networkidle");
await mountWholeOverview(page);
// A read refused for want of AI paints one of these on the overview:
// the section boundary, the error card, or the dense error row.
@@ -137,6 +148,7 @@ test.describe("J3: no AI provider", () => {
timeout: 20_000,
});
await page.waitForLoadState("networkidle");
await mountWholeOverview(page);
await expect(page.locator('[data-slot="coach-fab"]')).toHaveCount(0);
await expect(page.locator('[data-slot="daily-briefing"]')).toHaveCount(0);
// The per-card Coach hand-offs paint only while the Coach is available.
@@ -163,6 +175,7 @@ test.describe("J3: no AI provider", () => {
{ timeout: 20_000 },
);
await page.waitForLoadState("networkidle");
await mountWholeOverview(page);
await expect(page.locator('[data-slot="ai-setup-hint"]')).toHaveCount(0);
expect(network.aiRequests).toEqual([]);
+37
View File
@@ -36,3 +36,40 @@ export async function revealDeferredSections(page: Page): Promise<void> {
.toBe(0);
await page.evaluate(() => document.querySelector("main")?.scrollTo(0, 0));
}
/**
* `revealDeferredSections`, then wait until the reads the newly mounted
* sections fired have landed: no request in flight for 750 ms. A
* `waitForLoadState("networkidle")` cannot do this: the page reached that
* state once already, so it resolves at once, and a count of zero taken then
* passes before a deferred section has painted anything.
*/
export async function revealDeferredSectionsSettled(page: Page): Promise<void> {
let inFlight = 0;
const started = () => {
inFlight += 1;
};
const ended = () => {
inFlight -= 1;
};
page.on("request", started);
page.on("requestfinished", ended);
page.on("requestfailed", ended);
try {
await revealDeferredSections(page);
let quietSince = Date.now();
await expect
.poll(
() => {
if (inFlight > 0) quietSince = Date.now();
return Date.now() - quietSince >= 750;
},
{ timeout: 20_000, intervals: [100] },
)
.toBe(true);
} finally {
page.off("request", started);
page.off("requestfinished", ended);
page.off("requestfailed", ended);
}
}
+6 -6
View File
@@ -1,6 +1,6 @@
{
"name": "healthlog",
"version": "1.40.0",
"version": "1.40.1",
"description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.",
"license": "PolyForm-Noncommercial-1.0.0",
"homepage": "https://healthlog.dev",
@@ -80,11 +80,11 @@
"jspdf-autotable": "^5.0.8",
"lucide-react": "^1.44.0",
"mmdb-lib": "^3.0.2",
"next": "16.3.6",
"next": "16.3.8",
"next-themes": "^0.4.6",
"nodemailer": "10.0.9",
"otpauth": "^9.5.1",
"p-limit": "^7.3.1",
"p-limit": "^7.3.3",
"pg": "^8.23.0",
"pg-boss": "^12.34.0",
"prisma": "^7.10.0",
@@ -104,7 +104,7 @@
"web-push": "^3.6.7",
"yaml": "^2.8.4",
"zod": "^4.4.3",
"zod-openapi": "^6.0.1",
"zod-openapi": "^6.0.2",
"zxcvbn-typescript": "^5.0.1"
},
"devDependencies": {
@@ -124,7 +124,7 @@
"@vitejs/plugin-react": "^6.1.1",
"@vitest/coverage-v8": "^4.1.5",
"eslint": "^9",
"eslint-config-next": "16.3.6",
"eslint-config-next": "16.3.8",
"knip": "^6.32.3",
"lefthook": "^2.1.12",
"pdf-parse": "^2.4.5",
@@ -133,7 +133,7 @@
"prettier-plugin-tailwindcss": "^0.8.1",
"shadcn": "^4.16.1",
"tailwindcss": "^4",
"testcontainers": "^12.1.0",
"testcontainers": "^12.2.0",
"tw-animate-css": "^1.4.0",
"typescript": "^6",
"vitest": "^4.1.11"
+380 -339
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -36,7 +36,7 @@ try {
// v1.4.38.4 → v1.4.42. Do not hand-edit; bump `package.json` and rebuild.
const CACHE_VERSION =
(typeof self !== "undefined" && self.__APP_VERSION__) ||
/* @sw-version-fallback */ "v1.40.0";
/* @sw-version-fallback */ "v1.40.1";
const STATIC_CACHE = `healthlog-static-${CACHE_VERSION}`;
const PAGE_CACHE = `healthlog-pages-${CACHE_VERSION}`;
// v1.18.6 — read-only data cache for a curated allowlist of safe GET `/api/*`
@@ -0,0 +1,75 @@
/**
* `backupPayloadSchema` is built when `src/lib/validations/backup.ts` is
* evaluated and holds about 12 MB of heap. The restore job, the stored-backup
* readers and the export writer sit on the boot path, so a static value import
* of that module from any of them puts the schema in every server process,
* whether or not it ever touches a backup. Runtime code reaches it with a
* dynamic `import()` at the moment a file is validated, takes only types
* statically, and takes the version and the summary from `backup-summary.ts`.
*
* The one static importer allowed is the OpenAPI registry, which only the
* spec generator loads.
*/
import { readdirSync, readFileSync, statSync } from "node:fs";
import { join, relative } from "node:path";
import { describe, expect, it } from "vitest";
const ROOT = join(__dirname, "..");
const ALLOWED_STATIC = new Set(["lib/openapi/routes/export.ts"]);
function sourceFiles(dir: string): string[] {
const out: string[] = [];
for (const name of readdirSync(dir)) {
const path = join(dir, name);
if (name === "__tests__" || name === "generated") continue;
if (statSync(path).isDirectory()) out.push(...sourceFiles(path));
else if (/\.(ts|tsx)$/.test(name) && !/\.test\.tsx?$/.test(name)) {
out.push(path);
}
}
return out;
}
// `import … from "@/lib/validations/backup"`, across lines, capturing what
// precedes the specifier. Every importer uses the alias (a relative
// `./backup` also names `src/lib/cycle/backup.ts`, so it is not matched).
const STATIC_IMPORT =
/^import\s+(type\s+)?([^;]*?)\s+from\s+["']@\/lib\/validations\/backup["']/gm;
const DYNAMIC_IMPORT = /import\(\s*["']@\/lib\/validations\/backup["']\s*\)/g;
describe("the backup payload schema loads on first use", () => {
const files = sourceFiles(ROOT);
const staticValueImporters: string[] = [];
let dynamicImports = 0;
let typeImports = 0;
for (const file of files) {
const rel = relative(ROOT, file);
if (rel === "lib/validations/backup.ts") continue;
const text = readFileSync(file, "utf8");
for (const match of text.matchAll(STATIC_IMPORT)) {
const typeOnly =
match[1] !== undefined ||
(/^\{[^}]*\}$/.test(match[2]) &&
match[2]
.slice(1, -1)
.split(",")
.map((s) => s.trim())
.filter(Boolean)
.every((s) => s.startsWith("type ")));
if (typeOnly) typeImports++;
else if (!ALLOWED_STATIC.has(rel)) staticValueImporters.push(rel);
}
dynamicImports += [...text.matchAll(DYNAMIC_IMPORT)].length;
}
it("has no static value import outside the OpenAPI registry", () => {
expect(staticValueImporters).toEqual([]);
});
it("finds the loaders it guards (an empty match set would pass vacuously)", () => {
// upload, restore, preview (store + summary route), download, streamed reader
expect(dynamicImports).toBeGreaterThanOrEqual(5);
expect(typeImports).toBeGreaterThan(0);
});
});
@@ -38,7 +38,6 @@ import {
type StreamedBackup,
} from "@/lib/export/streamed-backup";
import { annotate } from "@/lib/logging/context";
import { parseBackupPayload } from "@/lib/validations/backup";
export const dynamic = "force-dynamic";
@@ -108,6 +107,8 @@ export const GET = apiHandler(
let streamed: StreamedBackup;
try {
streamed = await readStreamedBackup(source);
// Loaded on first use, not at import (see `backup-summary.ts`).
const { parseBackupPayload } = await import("@/lib/validations/backup");
parseBackupPayload(streamed.raw);
} catch (err) {
const readFailure = isStoredBackupReadError(err);
@@ -67,14 +67,16 @@ const summarizeMock = vi.fn();
const compatibleMock = vi.fn((version: string) => version !== "99");
vi.mock("@/lib/validations/backup", () => ({
parseBackupPayload: (...a: unknown[]) => parseMock(...a),
summarizeBackup: (...a: unknown[]) => summarizeMock(...a),
isCompatibleSchemaVersion: (version: string) => compatibleMock(version),
// The measurements are read as a stream and counted one by one.
BACKUP_SCHEMA_VERSION: "2",
backupMeasurementSchema: {
safeParse: (value: unknown) => ({ success: true, data: value }),
},
}));
vi.mock("@/lib/validations/backup-summary", () => ({
summarizeBackup: (...a: unknown[]) => summarizeMock(...a),
isCompatibleSchemaVersion: (version: string) => compatibleMock(version),
BACKUP_SCHEMA_VERSION: "2",
}));
import { GET } from "../route";
import { storedCopyIdentity } from "@/lib/export/backup-preview";
@@ -59,9 +59,8 @@ import {
import { annotate } from "@/lib/logging/context";
import {
isCompatibleSchemaVersion,
parseBackupPayload,
summarizeBackup,
} from "@/lib/validations/backup";
} from "@/lib/validations/backup-summary";
export const dynamic = "force-dynamic";
@@ -123,6 +122,8 @@ async function readPreview(
try {
const streamed = await readStreamedBackup(source);
// Loaded on first use, not at import (see `backup-summary.ts`).
const { parseBackupPayload } = await import("@/lib/validations/backup");
const payload = parseBackupPayload(streamed.raw);
const summary = {
...summarizeBackup(payload),
+5 -4
View File
@@ -61,15 +61,13 @@ import { storeBackupBlob } from "@/lib/export/store-backup-blob";
import { StreamedBackupInvalidError } from "@/lib/export/streamed-backup";
import { annotate } from "@/lib/logging/context";
import { checkRateLimit } from "@/lib/rate-limit";
import type { BackupPayload } from "@/lib/validations/backup";
import {
BACKUP_SCHEMA_VERSION,
backupMeasurementSchema,
backupPayloadSchema,
isCompatibleSchemaVersion,
summarizeBackup,
type BackupPayload,
type BackupSummary,
} from "@/lib/validations/backup";
} from "@/lib/validations/backup-summary";
export const dynamic = "force-dynamic";
@@ -252,6 +250,9 @@ export const POST = apiHandler(async (request: NextRequest) => {
yield chunk;
}
}
// Loaded on first use, not at import (see `backup-summary.ts`).
const { backupMeasurementSchema, backupPayloadSchema } =
await import("@/lib/validations/backup");
let firstWithoutId: number | null = null;
let scanned;
try {
@@ -329,6 +329,7 @@ describe("every Coach tool honours the lookback limit", () => {
const ROOT = join(__dirname, "../../../../..");
const COACH_DIR = join(ROOT, "src/lib/ai/coach");
const MCP_DIR = join(ROOT, "src/lib/mcp");
const SOURCE_SNAPSHOT_FILE = "source-snapshot.ts";
function walk(dir: string): string[] {
const out: string[] = [];
@@ -387,11 +388,23 @@ describe("the source hands the limit on", () => {
});
it("passes the limit to every Coach snapshot build", () => {
const calls = walk(COACH_DIR).flatMap((file) =>
callArguments(readFileSync(file, "utf8"), "buildCoachSnapshot").map(
(args) => ({ file: file.slice(ROOT.length + 1), args }),
),
);
// `source-snapshot.ts` is the single-metric build only MCP asks for
// (`sourceSnapshot: true`); MCP reads without the Coach's limit, so that
// file is checked the other way round below.
const calls = walk(COACH_DIR)
.filter((file) => !file.endsWith(SOURCE_SNAPSHOT_FILE))
.flatMap((file) => {
const source = readFileSync(file, "utf8");
return ["buildCoachSnapshot", "resolveSnapshotPrelude"].flatMap((fn) =>
callArguments(source, fn).map((args) => ({
file: `${file.slice(ROOT.length + 1)} ${fn}`,
args,
})),
);
});
expect(
calls.filter((c) => c.file.endsWith("resolveSnapshotPrelude")).length,
).toBeGreaterThan(0);
expect(calls.length).toBeGreaterThan(0);
for (const call of calls) {
expect(call.args, `${call.file} builds without a limit`).toMatch(
@@ -400,6 +413,15 @@ describe("the source hands the limit on", () => {
}
});
it("keeps the MCP single-metric build unlimited", () => {
const source = readFileSync(join(COACH_DIR, SOURCE_SNAPSHOT_FILE), "utf8");
const calls = ["buildCoachSnapshot", "resolveSnapshotPrelude"].flatMap(
(fn) => callArguments(source, fn),
);
expect(calls.length).toBeGreaterThan(0);
for (const args of calls) expect(args).not.toMatch(/\breach\b/);
});
it("passes the limit from the chat loop and nowhere from MCP", () => {
const loop = callArguments(
readFileSync(join(COACH_DIR, "tools/loop.ts"), "utf8"),
@@ -223,6 +223,11 @@ const VALUE_BLOCKS: readonly ValueBlock[] = [
},
];
/** The sources a value-series block is built for. */
export const VALUE_SERIES_SOURCES: ReadonlySet<CoachScopeSource> = new Set(
VALUE_BLOCKS.map((block) => block.source),
);
const TYPE_TO_REFERENCE_METRIC: Readonly<Record<string, ReferenceMetric>> = {
RESTING_HEART_RATE: "RESTING_HEART_RATE",
OXYGEN_SATURATION: "OXYGEN_SATURATION",
+329
View File
@@ -0,0 +1,329 @@
/**
* The reads and resolutions every Coach snapshot build starts from, shared by
* the full builder (`snapshot.ts`) and the single-source builder
* (`source-snapshot.ts`). Both must agree on the prefs, the module gates, the
* admitted sources, the window cutoffs and the windowed row read, because the
* single-source build promises the same section the full build would carry;
* keeping these in one place is what makes that promise hold by construction
* instead of by two copies staying in step.
*/
import { prisma } from "@/lib/db";
import type { MeasurementType } from "@/generated/prisma/client";
import { parseCoachPrefs } from "@/lib/validations/coach-prefs";
import { DEFAULT_TIMEZONE } from "@/lib/tz/resolver";
import { locales, defaultLocale, type Locale } from "@/lib/i18n/config";
import { resolveUnitPreferences } from "@/lib/measurements/display-transform";
import { memoizePerRequest } from "@/lib/request-cache";
import { resolveModuleMap } from "@/lib/modules/gate";
import { admitCoachSources, coachExclusions } from "@/lib/ai/coach/scope-gate";
import type { CoachDataCluster } from "@/lib/validations/coach-prefs";
import { clusterSourcesFromPrefs, sourceCluster } from "./clusters";
import { resolveScope, windowToDays } from "./snapshot-series";
import {
UNBOUNDED_REACH,
clampWindow,
type CoachHistoryReach,
} from "./history-reach";
import type { CoachScope, CoachScopeSource, CoachScopeWindow } from "./types";
/**
* Day-level cap for the raw timeline. Days within this window are kept
* verbatim (one entry per day with weekday). Older days inside the
* snapshot window are folded into weekly means so a 90-day window
* lands at ~14 day-rows + ~11 week-rows ≈ 25 rows per metric — well
* under the 3 000-token Coach turn budget on a 5-metric snapshot.
*/
export const DAILY_TIMELINE_DAYS = 14;
/**
* v1.7.0 — assembled-snapshot soft char cap. After the snapshot is
* built we measure `JSON.stringify(snapshot).length` as a ~4-chars-per-
* token proxy and, if it exceeds this cap, progressively degrade the
* lowest-priority clusters (drop `timeline.recent`, then collapse the
* weekly buckets) until it fits. ~24 000 chars ≈ ~6 000 tokens, which
* sits comfortably inside every provider's context alongside the system
* prompt + history window. The daily token ledger (`budget.ts`) stays
* the per-day cost backstop; this is the per-prompt shape backstop.
*/
export const MAX_SNAPSHOT_CHARS = 24_000;
/**
* v1.18.10 (P-2) — newest-first cap on the single multi-type measurement read
* that feeds the Coach snapshot timelines. The window read can reach 365 days
* (`lastYear` / `allTime`) across high-frequency types (PULSE / glucose are
* 200k-row-class), but the prompt only renders ~21 daily + ~10 weekly buckets
* per metric, so an uncapped read loaded a year of rows to discard almost all
* of them. 6000 keeps the recent-daily + weekly window exact even with several
* dense types active (≈ a year of multi-daily readings on one type, or a
* handful of types at a few readings/day) while bounding the worst case; the
* coarse MONTH/YEAR tail comes from the rollup tier, not this read.
*/
const SNAPSHOT_MEASUREMENT_ROW_CAP = 6000;
/**
* v1.7.0 — when more than this many clusters are active, cap the
* additive (non-core) clusters' timeline window so a 10-cluster,
* allTime request can't fan the timeline out across every series at
* once. The core clinical clusters keep the user-chosen window.
*/
const MULTI_CLUSTER_THRESHOLD = 6;
const MULTI_CLUSTER_WINDOW_CAP: CoachScopeWindow = "last90days";
/**
* Clusters that keep the user-chosen window even under the multi-cluster
* cap — the high-signal clinical series.
*/
const CORE_CLUSTERS: ReadonlySet<CoachDataCluster> = new Set<CoachDataCluster>([
"medication",
"cardio",
"glucose",
]);
/**
* Resolve the prefs, module gates and scope a build reads under.
*
* v1.4.23 H4 — apply per-user `excludeMetrics` BEFORE we read any
* measurement rows so the model never sees data the user opted out
* of. The filter intersects with the resolved scope (the explicit
* `scope` argument from the request body still wins for the
* _maximum_ set; prefs only narrow further).
*
* v1.4.25 W7b — the same prefs read also returns the user's
* displayTimezone so the day-key and weekday labels below match the
* calendar the user is looking at. Reading both columns in one
* query keeps the snapshot's read budget the same as before.
*
* v1.7.0 — the prefs read now also drives the source default: when
* the request omits an explicit `scope.sources`, the resolved scope
* expands the user's saved `dataClusters` (legacy default when the
* key is absent). So the prefs read must precede `resolveScope`.
* v1.18.0 — resolve the per-user module map once at build start so a
* disabled data-domain module's data never enters the coach context.
* The map read is memoised per-request by the gate, and runs alongside
* the prefs read (both only need `userId`), so the cold path pays a
* single extra round-trip at most. Disabled modules fold into the same
* SYSTEM-side exclusion the user's `excludeMetrics` flow already drives.
*/
export async function resolveSnapshotPrelude(
userId: string,
scope: CoachScope | undefined,
/**
* The Coach's lookback limit, applied to the window here so every build
* that starts from the prelude reads under it. The single-source MCP build
* passes none and keeps its own window.
*/
reach: CoachHistoryReach = UNBOUNDED_REACH,
) {
const moduleMapPromise = resolveModuleMap(userId);
// v1.20.0 (H-1) — the F1 coach tools each rebuild a single-source snapshot
// with a distinct LRU key, so the 60s snapshot cache does not share this read
// across the fan-out. Memoise it per-request (the select shape is constant, so
// userId is the only key) so up to 6 concurrent tool builds collapse to one
// prefs round-trip instead of starving the shared Prisma pool.
const prefsRow = await memoizePerRequest(`coach-prefs:${userId}`, () =>
prisma.user.findUnique({
where: { id: userId },
select: {
coachPrefsJson: true,
timezone: true,
locale: true,
// v1.11.5 — needed to collapse a dual-source sleep night to one
// canonical source before reconstructing per-night asleep totals.
sourcePriorityJson: true,
// v1.15 — the cycle snapshot block is gated on the resolved cycle
// toggle (an explicit opt-in/out overrides the gender default). Read
// both columns here so a non-cycle account pays no extra round-trip.
gender: true,
cycleProfile: { select: { cycleTrackingEnabled: true } },
// v1.16.16 — the glucose block converts canonical mg/dL to the user's
// display unit so the Coach reads the same number every other surface
// shows. Read it on this existing prefs hop (no extra round-trip).
glucoseUnit: true,
// The metric/imperial choice: the grounding bands and the blocks
// that carry a mass, length or temperature are written in it.
unitPreference: true,
// v1.18.6 (W7) — the explicit, user-declared diabetes opt-in. Selects
// the tighter ADA glycemic GOAL band for the glucose reference-grounding
// line only; never inferred from a reading, never a diagnosis. Read on
// this existing prefs hop (no extra round-trip).
hasDiabetes: true,
},
}),
);
const prefs = parseCoachPrefs(prefsRow?.coachPrefsJson);
// Resolve the UI locale for the rolling-profile narrative recall. The
// narrative rows are keyed by the full UI locale union, so the stored value
// is carried through as-is; an unset or unknown value falls back to the app
// default (`en`), never to German. The former `=== "en" ? "en" : "de"`
// binary made a French account recall a German narrative row.
const coachLocale: Locale = locales.includes(prefsRow?.locale as Locale)
? (prefsRow?.locale as Locale)
: defaultLocale;
const clusterDefault = clusterSourcesFromPrefs(prefs.dataClusters);
const { sources: scopedSources, window: requestedWindow } = resolveScope(
scope,
clusterDefault,
);
const window = clampWindow(requestedWindow, reach);
const userTz = prefsRow?.timezone ?? DEFAULT_TIMEZONE;
const units = resolveUnitPreferences({
unitPreference: prefsRow?.unitPreference,
glucoseUnit: prefsRow?.glucoseUnit,
});
// v1.18.0 — fold disabled data-domain modules into the system exclusion.
// `moduleMap[key] === false` means the user turned that module off; the
// gate has already resolved every delegation (cycle/coach) so this map
// is authoritative. We union the disabled modules' owned sources into
// `excluded` so the existing source-narrowing path below removes them
// before any row is read — the model never sees a disabled domain.
const moduleMap = await moduleMapPromise;
const excluded = coachExclusions(prefs, moduleMap);
const sources = admitCoachSources(scopedSources, excluded);
return {
prefsRow,
coachLocale,
window,
userTz,
units,
recoveryDisabled: moduleMap.recovery === false,
excluded,
sources,
};
}
/**
* The window cutoffs a build reads under, and the per-source cutoff the
* additive (non-core) blocks use once many clusters are active.
*/
export function resolveSnapshotCutoffs(
sources: ReadonlySet<CoachScopeSource>,
window: CoachScopeWindow,
now: Date,
) {
const windowDays = windowToDays(window);
const cutoff = new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000);
const recentCutoff = new Date(
now.getTime() - DAILY_TIMELINE_DAYS * 24 * 60 * 60 * 1000,
);
// v1.7.0 — when many clusters are active, cap the timeline read
// window for the ADDITIVE (non-core) clusters so a 10-cluster /
// allTime request cannot fan a dense timeline across every series at
// once. The core clinical clusters keep the user-chosen window.
const activeClusters = new Set<CoachDataCluster>();
for (const src of sources) {
const c = sourceCluster(src);
if (c) activeClusters.add(c);
}
const multiClusterCapActive = activeClusters.size > MULTI_CLUSTER_THRESHOLD;
const additiveCapDays = windowToDays(MULTI_CLUSTER_WINDOW_CAP);
const additiveCapCutoff = new Date(
now.getTime() - additiveCapDays * 24 * 60 * 60 * 1000,
);
// Effective `cutoff` for an additive block under the multi-cluster
// cap — the later of the window cutoff and the cap cutoff. Core
// clusters always use the full window cutoff.
const additiveCutoff = (source: CoachScopeSource): Date => {
const cluster = sourceCluster(source);
if (
multiClusterCapActive &&
cluster !== null &&
!CORE_CLUSTERS.has(cluster) &&
additiveCapCutoff > cutoff
) {
return additiveCapCutoff;
}
return cutoff;
};
return {
windowDays,
cutoff,
recentCutoff,
activeClusters,
multiClusterCapActive,
additiveCutoff,
};
}
/**
* The windowed measurement read the timeline blocks share.
*
* Single fetch for all measurement types — Prisma's filter pushes
* the type list into one SQL `WHERE type IN (…)` so we don't pay
* per-metric round-trips.
*/
export function readSnapshotMeasurementRows(
userId: string,
wantedTypes: readonly MeasurementType[],
cutoff: Date,
) {
if (wantedTypes.length === 0) return Promise.resolve([]);
return (
prisma.measurement
.findMany({
where: {
userId,
type: { in: [...wantedTypes] },
measuredAt: { gte: cutoff },
deletedAt: null,
},
// v1.18.10 (P-2) — read NEWEST-first + cap. PULSE / glucose are
// 200k-row-class types and the window can reach 365 days
// (lastYear / allTime), so an uncapped read pulled the entire
// year of high-frequency rows into memory just to fold them into
// ~21 daily + ~10 weekly buckets the prompt actually shows. The
// newest-first cap keeps the recent-daily timeline exact and only
// sheds the deepest weekly buckets on an extreme-volume account;
// the coarse MONTH/YEAR tail is read separately from the rollup
// tier (`buildCoarseTimelineTail`), so deep history survives.
orderBy: { measuredAt: "desc" },
take: SNAPSHOT_MEASUREMENT_ROW_CAP,
// v1.7.0 — `glucoseContext` rides along so the glucose block
// can split fasting / postprandial / random / bedtime without
// a second query. NULL on every non-glucose row.
select: {
type: true,
value: true,
measuredAt: true,
glucoseContext: true,
},
})
// Downstream bucketers re-sort/group internally, but restore
// ascending order so any order-sensitive consumer sees the same
// shape as before the cap.
.then((rows) => rows.reverse())
);
}
/**
* Sleep-stage rows from `since` on, with the writer columns the canonical
* dedup needs. Writer-level collapse: two HealthKit apps behind one source
* (watch stages vs phone in-bed) must not blend into one night.
*/
export function readSleepStageRows(userId: string, since: Date) {
return prisma.measurement.findMany({
where: {
userId,
type: "SLEEP_DURATION" as never,
measuredAt: { gte: since },
deletedAt: null,
},
orderBy: { measuredAt: "asc" },
select: {
value: true,
measuredAt: true,
sleepStage: true,
source: true,
deviceType: true,
},
});
}
/** The mood entries the mood timeline folds, tombstones excluded (v1.7.0). */
export function readSnapshotMoodRows(userId: string, since: Date) {
return prisma.moodEntry.findMany({
where: { userId, deletedAt: null, moodLoggedAt: { gte: since } },
orderBy: { moodLoggedAt: "asc" },
select: { moodLoggedAt: true, score: true },
});
}
+50 -289
View File
@@ -17,20 +17,21 @@
*/
import { prisma } from "@/lib/db";
import { extractFeatures } from "@/lib/insights/features";
import {
parseCoachPrefs,
type CoachDataCluster,
} from "@/lib/validations/coach-prefs";
import { DEFAULT_TIMEZONE } from "@/lib/tz/resolver";
import { locales, defaultLocale, type Locale } from "@/lib/i18n/config";
import {
resolveUnitPreferences,
type UnitPreferences,
} from "@/lib/measurements/display-transform";
import type { CoachDataCluster } from "@/lib/validations/coach-prefs";
import type { UnitPreferences } from "@/lib/measurements/display-transform";
import type { SleepStageRow } from "@/lib/analytics/sleep-night";
import { compactSections } from "@/lib/ai/prompts/compact-sections";
import { annotate } from "@/lib/logging/context";
import { memoizePerRequest } from "@/lib/request-cache";
import {
DAILY_TIMELINE_DAYS,
MAX_SNAPSHOT_CHARS,
readSleepStageRows,
readSnapshotMeasurementRows,
readSnapshotMoodRows,
resolveSnapshotCutoffs,
resolveSnapshotPrelude,
} from "./snapshot-prelude";
import { annotateSnapshotFreshness } from "./snapshot-freshness";
import { buildGlp1SnapshotBlock } from "./glp1-snapshot";
import { buildDerivedSnapshotBlock } from "./derived-snapshot";
@@ -54,25 +55,14 @@ import type { MeasurementType } from "@/generated/prisma/client";
import { COACH_SOURCE_MEASUREMENT_TYPES } from "@/lib/ai/coach/source-measurement-types";
import type { ReferenceMetric } from "@/lib/reference-ranges";
import { isCycleAvailableForUser } from "@/lib/cycle/gate";
import { resolveModuleMap } from "@/lib/modules/gate";
import { admitCoachSources, coachExclusions } from "@/lib/ai/coach/scope-gate";
import {
COURSES_COMPLIANCE_SELECT,
SCHEDULE_COMPLIANCE_SELECT,
} from "@/lib/analytics/compliance";
import type { BaselineProfile } from "@/lib/insights/derived";
import { toProfileSex } from "@/lib/profile/sex";
import {
CLUSTER_PRIORITY,
clusterSourcesFromPrefs,
sourceCluster,
} from "./clusters";
import type {
CoachProvenance,
CoachScope,
CoachScopeSource,
CoachScopeWindow,
} from "./types";
import { CLUSTER_PRIORITY, sourceCluster } from "./clusters";
import type { CoachProvenance, CoachScope, CoachScopeSource } from "./types";
import {
readSnapshotCache,
snapshotCacheKey,
@@ -80,8 +70,6 @@ import {
} from "./snapshot-cache";
import {
buildCoarseTimelineTail,
resolveScope,
windowToDays,
type CoarseTimelineTail,
} from "./snapshot-series";
import { buildWorkoutsBlock } from "./snapshot-blocks/workouts-block";
@@ -103,7 +91,6 @@ import {
import { TRACKED_INTAKE_WHERE } from "@/lib/medications/intake-tracking";
import {
UNBOUNDED_REACH,
clampWindow,
fitsReach,
isBounded,
reachFloor,
@@ -151,28 +138,6 @@ export interface CoachSnapshotResult {
units: UnitPreferences;
}
/**
* Day-level cap for the raw timeline. Days within this window are kept
* verbatim (one entry per day with weekday). Older days inside the
* snapshot window are folded into weekly means so a 90-day window
* lands at ~14 day-rows + ~11 week-rows ≈ 25 rows per metric — well
* under the 3 000-token Coach turn budget on a 5-metric snapshot.
*/
const DAILY_TIMELINE_DAYS = 14;
/**
* v1.18.10 (P-2) — newest-first cap on the single multi-type measurement read
* that feeds the Coach snapshot timelines. The window read can reach 365 days
* (`lastYear` / `allTime`) across high-frequency types (PULSE / glucose are
* 200k-row-class), but the prompt only renders ~21 daily + ~10 weekly buckets
* per metric, so an uncapped read loaded a year of rows to discard almost all
* of them. 6000 keeps the recent-daily + weekly window exact even with several
* dense types active (≈ a year of multi-daily readings on one type, or a
* handful of types at a few readings/day) while bounding the worst case; the
* coarse MONTH/YEAR tail comes from the rollup tier, not this read.
*/
const SNAPSHOT_MEASUREMENT_ROW_CAP = 6000;
/**
* v1.17.0 — the sleep-rhythm read (sleep-debt + chronotype) is a fixed
* trailing-window artifact, identical across the Sleep page, the dashboard
@@ -192,37 +157,6 @@ const SLEEP_RHYTHM_WINDOW_DAYS = 42;
*/
const COARSE_TAIL_START_DAYS = 90;
/**
* v1.7.0 — assembled-snapshot soft char cap. After the snapshot is
* built we measure `JSON.stringify(snapshot).length` as a ~4-chars-per-
* token proxy and, if it exceeds this cap, progressively degrade the
* lowest-priority clusters (drop `timeline.recent`, then collapse the
* weekly buckets) until it fits. ~24 000 chars ≈ ~6 000 tokens, which
* sits comfortably inside every provider's context alongside the system
* prompt + history window. The daily token ledger (`budget.ts`) stays
* the per-day cost backstop; this is the per-prompt shape backstop.
*/
const MAX_SNAPSHOT_CHARS = 24_000;
/**
* v1.7.0 — when more than this many clusters are active, cap the
* additive (non-core) clusters' timeline window so a 10-cluster,
* allTime request can't fan the timeline out across every series at
* once. The core clinical clusters keep the user-chosen window.
*/
const MULTI_CLUSTER_THRESHOLD = 6;
const MULTI_CLUSTER_WINDOW_CAP: CoachScopeWindow = "last90days";
/**
* Clusters that keep the user-chosen window even under the multi-cluster
* cap — the high-signal clinical series.
*/
const CORE_CLUSTERS: ReadonlySet<CoachDataCluster> = new Set<CoachDataCluster>([
"medication",
"cardio",
"glucose",
]);
/**
* Build the Coach prompt snapshot for `userId`. Always uses
* `includeRaw=false` because the Coach replies are conversational and
@@ -297,105 +231,40 @@ async function buildCoachSnapshotImpl(
scope: CoachScope | undefined,
reach: CoachHistoryReach,
): Promise<CoachSnapshotResult> {
// v1.4.23 H4 — apply per-user `excludeMetrics` BEFORE we read any
// measurement rows so the model never sees data the user opted out
// of. The filter intersects with the resolved scope (the explicit
// `scope` argument from the request body still wins for the
// _maximum_ set; prefs only narrow further).
//
// v1.4.25 W7b — the same prefs read also returns the user's
// displayTimezone so the day-key and weekday labels below match the
// calendar the user is looking at. Reading both columns in one
// query keeps the snapshot's read budget the same as before.
//
// v1.7.0 — the prefs read now also drives the source default: when
// the request omits an explicit `scope.sources`, the resolved scope
// expands the user's saved `dataClusters` (legacy default when the
// key is absent). So the prefs read must precede `resolveScope`.
// v1.18.0 — resolve the per-user module map once at build start so a
// disabled data-domain module's data never enters the coach context.
// The map read is memoised per-request by the gate, and runs alongside
// the prefs read (both only need `userId`), so the cold path pays a
// single extra round-trip at most. Disabled modules fold into the same
// SYSTEM-side exclusion the user's `excludeMetrics` flow already drives.
const moduleMapPromise = resolveModuleMap(userId);
// v1.20.0 (H-1) — the F1 coach tools each rebuild a single-source snapshot
// with a distinct LRU key, so the 60s snapshot cache does not share this read
// across the fan-out. Memoise it per-request (the select shape is constant, so
// userId is the only key) so up to 6 concurrent tool builds collapse to one
// prefs round-trip instead of starving the shared Prisma pool.
const prefsRow = await memoizePerRequest(`coach-prefs:${userId}`, () =>
prisma.user.findUnique({
where: { id: userId },
select: {
coachPrefsJson: true,
timezone: true,
locale: true,
// v1.11.5 — needed to collapse a dual-source sleep night to one
// canonical source before reconstructing per-night asleep totals.
sourcePriorityJson: true,
// v1.15 — the cycle snapshot block is gated on the resolved cycle
// toggle (an explicit opt-in/out overrides the gender default). Read
// both columns here so a non-cycle account pays no extra round-trip.
gender: true,
cycleProfile: { select: { cycleTrackingEnabled: true } },
// v1.16.16 — the glucose block converts canonical mg/dL to the user's
// display unit so the Coach reads the same number every other surface
// shows. Read it on this existing prefs hop (no extra round-trip).
glucoseUnit: true,
// The metric/imperial choice: the grounding bands and the blocks
// that carry a mass, length or temperature are written in it.
unitPreference: true,
// v1.18.6 (W7) — the explicit, user-declared diabetes opt-in. Selects
// the tighter ADA glycemic GOAL band for the glucose reference-grounding
// line only; never inferred from a reading, never a diagnosis. Read on
// this existing prefs hop (no extra round-trip).
hasDiabetes: true,
},
}),
);
const prefs = parseCoachPrefs(prefsRow?.coachPrefsJson);
// Resolve the UI locale for the rolling-profile narrative recall. The
// narrative rows are keyed by the full UI locale union, so the stored value
// is carried through as-is; an unset or unknown value falls back to the app
// default (`en`), never to German. The former `=== "en" ? "en" : "de"`
// binary made a French account recall a German narrative row.
const coachLocale: Locale = locales.includes(prefsRow?.locale as Locale)
? (prefsRow?.locale as Locale)
: defaultLocale;
const clusterDefault = clusterSourcesFromPrefs(prefs.dataClusters);
const { sources: scopedSources, window: requestedWindow } = resolveScope(
scope,
clusterDefault,
);
// The lookback limit caps the narration window, whatever the caller asked
// for. Every fixed-window block below answers to the same limit.
const window = clampWindow(requestedWindow, reach);
// Prefs, module gates and the admitted sources — see `snapshot-prelude.ts`.
// The lookback limit caps the narration window there, whatever the caller
// asked for; every fixed-window block below answers to the same limit.
const {
prefsRow,
coachLocale,
window,
userTz,
units,
recoveryDisabled,
excluded,
sources,
} = await resolveSnapshotPrelude(userId, scope, reach);
const bounded = isBounded(reach);
const userTz = prefsRow?.timezone ?? DEFAULT_TIMEZONE;
const units = resolveUnitPreferences({
unitPreference: prefsRow?.unitPreference,
glucoseUnit: prefsRow?.glucoseUnit,
});
const glucoseUnit = units.glucoseUnit;
// v1.18.0 — fold disabled data-domain modules into the system exclusion.
// `moduleMap[key] === false` means the user turned that module off; the
// gate has already resolved every delegation (cycle/coach) so this map
// is authoritative. We union the disabled modules' owned sources into
// `excluded` so the existing source-narrowing path below removes them
// before any row is read — the model never sees a disabled domain.
const moduleMap = await moduleMapPromise;
const recoveryDisabled = moduleMap.recovery === false;
const excluded = coachExclusions(prefs, moduleMap);
// v1.4.36 W3 T2 — `medications` and `anthropometrics` are
// exclude-only toggles (not in `CoachScopeSource`); they gate the
// GLP-1 weeklyContext / compliance branch and the anthropometrics
// block respectively.
const excludesMedications = excluded.has("medications");
const excludesAnthropometrics = excluded.has("anthropometrics");
const sources = admitCoachSources(scopedSources, excluded);
const windowDays = windowToDays(window);
// Pull raw measurement rows once for the configured window so day
// and week buckets share a single I/O hop. Mood + compliance live in
// separate tables and are loaded conditionally below.
const now = new Date();
const {
windowDays,
cutoff,
recentCutoff,
activeClusters,
multiClusterCapActive,
additiveCutoff,
} = resolveSnapshotCutoffs(sources, window, now);
// v1.11.3 — kick the feature extraction off as a promise now and await
// it alongside the shared measurement read below. `extractFeatures`
// and the measurement `findMany` are independent (each only needs
@@ -461,25 +330,6 @@ async function buildCoachSnapshotImpl(
if (cluster) blockClusters.set(key, cluster);
};
// Pull raw measurement rows once for the configured window so day
// and week buckets share a single I/O hop. Mood + compliance live in
// separate tables and are loaded conditionally below.
const now = new Date();
const cutoff = new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000);
const recentCutoff = new Date(
now.getTime() - DAILY_TIMELINE_DAYS * 24 * 60 * 60 * 1000,
);
// v1.7.0 — when many clusters are active, cap the timeline read
// window for the ADDITIVE (non-core) clusters so a 10-cluster /
// allTime request cannot fan a dense timeline across every series at
// once. The core clinical clusters keep the user-chosen window.
const activeClusters = new Set<CoachDataCluster>();
for (const src of sources) {
const c = sourceCluster(src);
if (c) activeClusters.add(c);
}
const multiClusterCapActive = activeClusters.size > MULTI_CLUSTER_THRESHOLD;
// v1.7.0 — record which clusters resolved active for this build so
// the observability dashboards can track cluster adoption + the
// multi-cluster cap firing rate.
@@ -491,25 +341,6 @@ async function buildCoachSnapshotImpl(
multiClusterCap: multiClusterCapActive,
},
});
const additiveCapDays = windowToDays(MULTI_CLUSTER_WINDOW_CAP);
const additiveCapCutoff = new Date(
now.getTime() - additiveCapDays * 24 * 60 * 60 * 1000,
);
// Effective `cutoff` for an additive block under the multi-cluster
// cap — the later of the window cutoff and the cap cutoff. Core
// clusters always use the full window cutoff.
const additiveCutoff = (source: CoachScopeSource): Date => {
const cluster = sourceCluster(source);
if (
multiClusterCapActive &&
cluster !== null &&
!CORE_CLUSTERS.has(cluster) &&
additiveCapCutoff > cutoff
) {
return additiveCapCutoff;
}
return cutoff;
};
const wantsBp = sources.has("bp");
const wantsWeight = sources.has("weight");
@@ -579,42 +410,11 @@ async function buildCoachSnapshotImpl(
(source) => COACH_SOURCE_MEASUREMENT_TYPES[source] ?? [],
);
const measurementRowsPromise =
wantedTypes.length > 0
? prisma.measurement
.findMany({
where: {
userId,
type: { in: wantedTypes },
measuredAt: { gte: cutoff },
deletedAt: null,
},
// v1.18.10 (P-2) — read NEWEST-first + cap. PULSE / glucose are
// 200k-row-class types and the window can reach 365 days
// (lastYear / allTime), so an uncapped read pulled the entire
// year of high-frequency rows into memory just to fold them into
// ~21 daily + ~10 weekly buckets the prompt actually shows. The
// newest-first cap keeps the recent-daily timeline exact and only
// sheds the deepest weekly buckets on an extreme-volume account;
// the coarse MONTH/YEAR tail is read separately from the rollup
// tier (`buildCoarseTimelineTail`), so deep history survives.
orderBy: { measuredAt: "desc" },
take: SNAPSHOT_MEASUREMENT_ROW_CAP,
// v1.7.0 — `glucoseContext` rides along so the glucose block
// can split fasting / postprandial / random / bedtime without
// a second query. NULL on every non-glucose row.
select: {
type: true,
value: true,
measuredAt: true,
glucoseContext: true,
},
})
// Downstream bucketers re-sort/group internally, but restore
// ascending order so any order-sensitive consumer sees the same
// shape as before the cap.
.then((rows) => rows.reverse())
: Promise.resolve([]);
const measurementRowsPromise = readSnapshotMeasurementRows(
userId,
wantedTypes,
cutoff,
);
// v1.11.3 — `extractFeatures` and the shared measurement read are
// mutually independent and both gate the blocks below (every aggregate
@@ -661,14 +461,7 @@ async function buildCoachSnapshotImpl(
!recoveryDisabled && derivedSources.some((s) => sources.has(s));
const moodRowsPromise =
wantsMood && features.mood
? prisma.moodEntry.findMany({
// v1.7.0 sync — exclude tombstoned rows from the Coach snapshot.
where: { userId, deletedAt: null, moodLoggedAt: { gte: cutoff } },
orderBy: { moodLoggedAt: "asc" },
select: { moodLoggedAt: true, score: true },
})
: null;
wantsMood && features.mood ? readSnapshotMoodRows(userId, cutoff) : null;
// v1.16.9 — the adherence timeline derives from the LEDGER tally (the
// same band engine the compliance % + dose history consume), not from a
// raw intake-row count. The raw count read a worker-minted pending row
@@ -718,24 +511,7 @@ async function buildCoachSnapshotImpl(
})
: null;
const sleepRowsPromise = sources.has("sleep")
? prisma.measurement.findMany({
where: {
userId,
type: "SLEEP_DURATION" as never,
measuredAt: { gte: additiveCutoff("sleep") },
deletedAt: null,
},
orderBy: { measuredAt: "asc" },
// Writer-level collapse: two HealthKit apps behind one source
// (watch stages vs phone in-bed) must not blend into one night.
select: {
value: true,
measuredAt: true,
sleepStage: true,
source: true,
deviceType: true,
},
})
? readSleepStageRows(userId, additiveCutoff("sleep"))
: null;
// v1.17.0 — sleep-rhythm rows. The sleep-debt + chronotype DTO is a fixed
// trailing-42-day artifact (the Sleep page + dashboard read the same window),
@@ -748,28 +524,13 @@ async function buildCoachSnapshotImpl(
const sleepRhythmCutoff = new Date(
now.getTime() - SLEEP_RHYTHM_WINDOW_DAYS * 24 * 60 * 60 * 1000,
);
// A limit shorter than the rhythm's fixed 42 days leaves the block out: a
// debt or chronotype band over fewer nights is not the one the page shows.
// source + deviceType feed the canonical writer-dedup so a multi-source
// night is counted ONCE, matching every other sleep surface. A limit
// shorter than the rhythm's fixed 42 days leaves the block out: a debt or
// chronotype band over fewer nights is not the one the page shows.
const sleepRhythmRowsPromise =
sources.has("sleep") && fitsReach(SLEEP_RHYTHM_WINDOW_DAYS, reach)
? prisma.measurement.findMany({
where: {
userId,
type: "SLEEP_DURATION" as never,
measuredAt: { gte: sleepRhythmCutoff },
deletedAt: null,
},
orderBy: { measuredAt: "asc" },
// source + deviceType feed the canonical writer-dedup so a multi-source
// night is counted ONCE, matching every other sleep surface.
select: {
value: true,
measuredAt: true,
sleepStage: true,
source: true,
deviceType: true,
},
})
? readSleepStageRows(userId, sleepRhythmCutoff)
: null;
const workoutRowsPromise = sources.has("workouts")
? prisma.workout.findMany({
+265
View File
@@ -0,0 +1,265 @@
/**
* The snapshot one metric's read needs, built from that metric's rows.
*
* `get_metric_series` answers with one section of a Coach snapshot. Built by
* the full builder, that one section paid for everything a Coach turn
* carries: the all-type feature extraction, the derived wellness layer, the
* trajectory and correlation engines, memory, GLP-1, cycle, illness, labs and
* visits — a few hundred SQL statements and the event-loop time to fold them,
* for a result that keeps none of it. An MCP client polling one metric every
* few minutes paid that on every call.
*
* This builder runs the same prelude (prefs, module gates, admitted sources,
* cutoffs) and the same block builders as `snapshot.ts`, over the requested
* source's own reads only, and hands back the same section and the same
* reference grounding. Each block reads exactly what the full build reads for
* it: the core clinical aggregates come from `extractFeatures` scoped to the
* one block (whose bulk read keeps its all-type cap, so the rows a block sees
* are unchanged), the timelines from the same windowed read, the coarse tail
* from the same rollup reader.
*
* Two cases go to the full builder instead:
*
* - a source whose section this module does not build (glucose, workouts,
* compliance have their own tools; the environment cluster is ordered
* beside the memory / trajectory / correlation blocks in the budget pass,
* see below);
* - a section large enough that the full build's prompt-budget pass could
* plausibly trim it.
*
* The budget pass is the one thing a single-source build cannot see. The full
* builder trims blocks, lowest cluster first, once the WHOLE snapshot passes
* `MAX_SNAPSHOT_CHARS`; the requested section is only touched after every
* lower-priority block is already collapsed, so what decides it is the section
* plus the context that is never shed (labs, illness, anthropometrics) or
* ranks above it. A section below `SOURCE_SNAPSHOT_FALLBACK_CHARS` leaves more
* than `MAX_SNAPSHOT_CHARS - SOURCE_SNAPSHOT_FALLBACK_CHARS` of room for that
* context, which a single-source build does not reach on any account we know
* of; above it, the full builder answers and the result is the old one by
* construction. Should an account ever carry more context than that room, the
* difference is that this build returns the section untrimmed — the trim
* exists for a prompt this read never sends.
*/
import { annotate } from "@/lib/logging/context";
import {
extractFeatures,
type ScopedFeatureBlock,
} from "@/lib/insights/features";
import { compactSections } from "@/lib/ai/prompts/compact-sections";
import { memoizePerRequest } from "@/lib/request-cache";
import type { MeasurementType } from "@/generated/prisma/client";
import type { ReferenceMetric } from "@/lib/reference-ranges";
import type { SleepStageRow } from "@/lib/analytics/sleep-night";
import { COACH_SOURCE_MEASUREMENT_TYPES } from "./source-measurement-types";
import { sourceCluster } from "./clusters";
import { buildCoachSnapshot, type CoachSnapshotResult } from "./snapshot";
import {
DAILY_TIMELINE_DAYS,
MAX_SNAPSHOT_CHARS,
readSleepStageRows,
readSnapshotMeasurementRows,
readSnapshotMoodRows,
resolveSnapshotCutoffs,
resolveSnapshotPrelude,
} from "./snapshot-prelude";
import { annotateSnapshotFreshness } from "./snapshot-freshness";
import { buildReferenceGroundingBlock } from "./reference-grounding";
import { buildCoarseTimelineTail } from "./snapshot-series";
import { buildCoreMetricsBlocks } from "./snapshot-blocks/core-metrics-block";
import {
buildValueSeriesBlocks,
VALUE_SERIES_SOURCES,
} from "./snapshot-blocks/value-series-blocks";
import { buildSleepTimelineBlock } from "./snapshot-blocks/sleep-block";
import type {
CoachProvenance,
CoachScopeSource,
CoachScopeWindow,
} from "./types";
/**
* Above this pretty-printed size the section goes to the full builder, whose
* budget pass may trim it (see the module comment). A 30-day section is a few
* thousand characters; a year of blood pressure with its coarse tail is the
* kind of section that crosses it.
*/
export const SOURCE_SNAPSHOT_FALLBACK_CHARS = MAX_SNAPSHOT_CHARS / 2;
/** Core clinical sources: their aggregate is a feature block. */
const CORE_FEATURE_BLOCK: Partial<
Record<CoachScopeSource, ScopedFeatureBlock>
> = {
bp: "bloodPressure",
weight: "weight",
pulse: "pulse",
mood: "mood",
};
/** Core clinical sources with a coarse MONTH/YEAR tail, by its anchor type. */
const COARSE_TAIL_TYPE: Partial<Record<CoachScopeSource, MeasurementType>> = {
bp: "BLOOD_PRESSURE_SYS",
weight: "WEIGHT",
pulse: "PULSE",
};
/** Whether this module builds `source`'s section itself. */
export function buildsSourceSection(source: CoachScopeSource): boolean {
if (sourceCluster(source) === "environment") return false;
return (
source in CORE_FEATURE_BLOCK ||
source === "sleep" ||
VALUE_SERIES_SOURCES.has(source)
);
}
export type CoachSourceSnapshot = Pick<
CoachSnapshotResult,
"sections" | "referenceGrounding"
>;
/**
* Build the snapshot sections and reference grounding a single-source read
* slices, identical to what `buildCoachSnapshot(userId, { sources: [source],
* window })` returns for that source's section.
*/
export async function buildCoachSourceSnapshot(
userId: string,
source: CoachScopeSource,
window: CoachScopeWindow | undefined,
): Promise<CoachSourceSnapshot> {
const scope = { sources: [source], window };
if (!buildsSourceSection(source)) {
annotate({
action: { name: "coach.source_snapshot.full" },
meta: { source, reason: "unsupported_source" },
});
return buildCoachSnapshot(userId, scope);
}
const prelude = await resolveSnapshotPrelude(userId, scope);
const { prefsRow, userTz, units, sources } = prelude;
const now = new Date();
const { windowDays, cutoff, recentCutoff, additiveCutoff } =
resolveSnapshotCutoffs(sources, prelude.window, now);
// An excluded source leaves `sources` empty: nothing is read, the section
// stays absent, and the caller resolves the miss the way it always has.
const admitted = sources.has(source);
const featureBlock = admitted ? CORE_FEATURE_BLOCK[source] : undefined;
const coarseType = admitted ? COARSE_TAIL_TYPE[source] : undefined;
const wantedTypes =
admitted && source !== "sleep" && source !== "mood"
? [...(COACH_SOURCE_MEASUREMENT_TYPES[source] ?? [])]
: [];
const [features, coarseTail, measurementRows] = await Promise.all([
featureBlock
? memoizePerRequest(
`coach-source-features:${userId}:${windowDays}:${featureBlock}`,
() =>
extractFeatures(userId, false, {
sinceDays: windowDays,
only: new Set([featureBlock]),
}),
)
: undefined,
coarseType
? buildCoarseTimelineTail(
userId,
coarseType,
now,
userTz,
prefsRow?.sourcePriorityJson ?? null,
)
: undefined,
readSnapshotMeasurementRows(userId, wantedTypes, cutoff),
]);
const moodRows =
source === "mood" && admitted && features?.mood
? await readSnapshotMoodRows(userId, cutoff)
: null;
// Provenance and the degrade registry belong to the full builder's prompt;
// the block builders still want somewhere to write them.
const snapshot: Record<string, unknown> = {};
const windows = new Set<CoachProvenance["windows"][number]>();
const metrics = new Set<CoachProvenance["metrics"][number]>();
const counts: NonNullable<CoachProvenance["counts"]> = {};
const registerBlock = () => {};
const groundingValues = new Map<ReferenceMetric, number>();
buildCoreMetricsBlocks({
sources,
features: features ?? {},
measurementRows,
moodRows,
recentCutoff,
userTz,
coarseTails: coarseType ? { [source]: coarseTail } : {},
snapshot,
windows,
metrics,
counts,
registerBlock,
groundingValues,
units,
});
buildValueSeriesBlocks({
sources,
measurementRows,
additiveCutoff,
recentCutoff,
userTz,
snapshot,
metrics,
counts,
registerBlock,
groundingValues,
units,
});
if (admitted && source === "sleep") {
const sleepRows = await readSleepStageRows(userId, additiveCutoff("sleep"));
buildSleepTimelineBlock({
sleepRows: sleepRows as SleepStageRow[],
sourcePriorityJson: prefsRow?.sourcePriorityJson ?? null,
userTz,
recentCutoff,
snapshot,
metrics,
counts,
registerBlock,
groundingValues,
});
}
snapshot.scope = {
window: prelude.window,
sources: Array.from(sources),
timelineRecentDays: DAILY_TIMELINE_DAYS,
};
const sections = compactSections(snapshot);
annotateSnapshotFreshness(sections);
const size = JSON.stringify(sections, null, 2).length;
if (size > SOURCE_SNAPSHOT_FALLBACK_CHARS) {
annotate({
action: { name: "coach.source_snapshot.full" },
meta: { source, reason: "section_size", chars: size },
});
return buildCoachSnapshot(userId, scope);
}
const referenceGrounding = buildReferenceGroundingBlock({
metrics: Array.from(groundingValues.entries()).map(([metric, value]) => ({
metric,
value,
})),
hasDiabetes: prefsRow?.hasDiabetes ?? false,
units,
});
annotate({
action: { name: "coach.source_snapshot.built" },
meta: { source, chars: size },
});
return { sections, referenceGrounding };
}
+34 -7
View File
@@ -47,6 +47,7 @@ import type { Locale } from "@/lib/i18n/config";
import { resolveUserTimezone } from "@/lib/tz/resolver";
import { resolveModuleMap } from "@/lib/modules/gate";
import { buildCoachSnapshot } from "@/lib/ai/coach/snapshot";
import { buildCoachSourceSnapshot } from "@/lib/ai/coach/source-snapshot";
import { admitCoachSources, coachExclusions } from "@/lib/ai/coach/scope-gate";
import type { CoachPrefs } from "@/lib/validations/coach-prefs";
import { readMessageResults } from "@/lib/ai/coach/persistence";
@@ -271,6 +272,13 @@ export async function executeCoachTool(args: {
* already-built cache entry instead of rebuilding a single-source snapshot.
*/
sharedScope?: CoachScope;
/**
* Build a single-metric read from that source's own rows
* (`source-snapshot.ts`) rather than from a Coach snapshot. Set by the MCP
* surface, whose reads have no turn snapshot to land on and keep only the
* one section; the section is the same either way.
*/
sourceSnapshot?: boolean;
/** v1.39.4 — the chat turn, when the call runs inside one. */
turn?: CoachToolTurnContext;
/**
@@ -281,8 +289,15 @@ export async function executeCoachTool(args: {
*/
reach?: CoachHistoryReach;
}): Promise<CoachToolResult> {
const { userId, name, rawArguments, fallbackWindow, sharedScope, turn } =
args;
const {
userId,
name,
rawArguments,
fallbackWindow,
sharedScope,
sourceSnapshot = false,
turn,
} = args;
const reach = args.reach ?? UNBOUNDED_REACH;
if (!isCoachToolName(name) && name !== SHOW_RESULT_TOOL_NAME) {
@@ -316,6 +331,7 @@ export async function executeCoachTool(args: {
sharedScope,
turn,
reach,
sourceSnapshot,
);
annotate({
action: { name: "coach.tool.executed" },
@@ -344,6 +360,7 @@ async function dispatch(
sharedScope: CoachScope | undefined,
turn: CoachToolTurnContext | undefined,
reach: CoachHistoryReach,
sourceSnapshot: boolean,
): Promise<CoachToolResult> {
const result = await dispatchRead(
name,
@@ -353,6 +370,7 @@ async function dispatch(
sharedScope,
turn,
reach,
sourceSnapshot,
);
if (!turn || !result.present || result.table) return result;
return withProjectedTable(
@@ -374,6 +392,7 @@ async function dispatchRead(
sharedScope: CoachScope | undefined,
turn: CoachToolTurnContext | undefined,
reach: CoachHistoryReach,
sourceSnapshot: boolean,
): Promise<CoachToolResult> {
// Every branch takes the lookback limit; `coach-history-reach-guard.test.ts`
// fails when one does not.
@@ -385,6 +404,7 @@ async function dispatchRead(
fallbackWindow,
sharedScope,
reach,
sourceSnapshot,
);
case "get_glucose_panel":
return getGlucosePanel(
@@ -455,6 +475,7 @@ async function getMetricSeries(
fallbackWindow: CoachScopeWindow | undefined,
sharedScope: CoachScope | undefined,
reach: CoachHistoryReach,
sourceSnapshot: boolean,
): Promise<CoachToolResult> {
const parsed = getMetricSeriesArgsSchema.safeParse(rawArgs);
if (!parsed.success) return badArgs("get_metric_series", parsed.error);
@@ -480,11 +501,17 @@ async function getMetricSeries(
return { present: false, reason: "unsupported_metric" };
}
const snapshot = await buildCoachSnapshot(
userId,
scopeFor([metric], window, fallbackWindow, sharedScope, reach),
{ reach },
);
// An MCP read needs one section and nothing else, so it is built from that
// source's own rows: the same section, without the context blocks a Coach
// turn carries. See `source-snapshot.ts`. Only MCP asks for it, and MCP
// reads without the Coach's lookback limit, so it takes no reach.
const snapshot = sourceSnapshot
? await buildCoachSourceSnapshot(userId, metric, window ?? fallbackWindow)
: await buildCoachSnapshot(
userId,
scopeFor([metric], window, fallbackWindow, sharedScope, reach),
{ reach },
);
const section = pickSection(snapshot.sections, sectionKey);
if (section === undefined) {
return emptyRead(
+7 -3
View File
@@ -4,9 +4,13 @@
* from the app, and any Withings, import or manual row the server-to-Health
* mirror writes. Builds from before the marker existed wrote the id alone.
*
* Such a sample is already in the database under its own source. When the same
* sample comes back through an Apple Health export it is a second copy of that
* reading, so the export import leaves it out.
* Such a sample is usually already in the database under its own source. When
* it comes back through an Apple Health export it is a second copy of that
* reading, so the export import leaves it out, but only once its
* `HKExternalUUID` names a row of the importing account (a cycle sample: once
* that account holds a day-log on its day). The marker alone says HealthLog
* wrote the sample, not that this instance still holds it: after a move to a
* new instance without a backup, the export is how those values come back.
*
* This is not the predicate the app's own sync uses. The app decides "our own
* echo" from `HKExternalUUID` plus the sample's authoring source being the app
@@ -20,7 +20,8 @@ import {
storedPreviewFor,
} from "@/lib/export/backup-preview";
import type { FullBackupCounts } from "@/lib/export/full-backup-payload";
import { backupPayloadSchema, summarizeBackup } from "@/lib/validations/backup";
import { backupPayloadSchema } from "@/lib/validations/backup";
import { summarizeBackup } from "@/lib/validations/backup-summary";
const ts = "2026-07-19T07:00:00.000Z";
/** A bytes-column value holding the string codec under key id `v3`. */
@@ -100,8 +101,8 @@ describe("createBackupPreviewCollector", () => {
const expectedKeys = new BackupKeyIdCollector();
expectedKeys.visit(FILE);
it("says what a full read of the copy says", () => {
const collector = createBackupPreviewCollector();
it("says what a full read of the copy says", async () => {
const collector = await createBackupPreviewCollector();
observeAsTheWriterDoes(FILE, collector.observe);
const result = collector.finish(countsOf(FILE));
expect(result).not.toBeNull();
@@ -111,8 +112,8 @@ describe("createBackupPreviewCollector", () => {
expect(result!.keys.keyIds()).toEqual(["v3"]);
});
it("answers null for a copy that does not name its version, owner and date", () => {
const collector = createBackupPreviewCollector();
it("answers null for a copy that does not name its version, owner and date", async () => {
const collector = await createBackupPreviewCollector();
const { userId, ...rest } = FILE;
void userId;
observeAsTheWriterDoes(rest, collector.observe);
@@ -120,6 +121,64 @@ describe("createBackupPreviewCollector", () => {
});
});
describe("createBackupPreviewCollector against the schema", () => {
// A copy the schema refuses must not be described as restorable: a null
// preview sends the preview route to read the copy, which says why not.
async function previewOf(file: Record<string, unknown>) {
const collector = await createBackupPreviewCollector();
observeAsTheWriterDoes(file, collector.observe);
return collector.finish(countsOf(FILE));
}
it("answers null when a bulk row violates the schema", async () => {
const measurements = FILE.measurements.map((m, i) =>
i === 1 ? { ...m, measuredAt: "not a date" } : m,
);
expect(await previewOf({ ...FILE, measurements })).toBeNull();
});
it("answers null when a whole section violates the schema", async () => {
const labResults = [{ ...FILE.labResults[0], takenAt: 42 }];
expect(await previewOf({ ...FILE, labResults })).toBeNull();
});
it("answers null when the header does not parse", async () => {
expect(
await previewOf({ ...FILE, exportedAt: "yesterday-ish" }),
).toBeNull();
});
it("answers null for a canonical copy with a measurement without an id", async () => {
const measurements = FILE.measurements.map((m, i) =>
i === 0 ? { ...m, id: undefined } : m,
);
expect(await previewOf({ ...FILE, measurements })).toBeNull();
});
it("agrees with the whole-document parse on every case above", () => {
// The oracle: each refused file is refused by the full schema too.
const bad = [
{
...FILE,
measurements: FILE.measurements.map((m, i) =>
i === 1 ? { ...m, measuredAt: "not a date" } : m,
),
},
{ ...FILE, labResults: [{ ...FILE.labResults[0], takenAt: 42 }] },
{ ...FILE, exportedAt: "yesterday-ish" },
{
...FILE,
measurements: FILE.measurements.map((m, i) =>
i === 0 ? { ...m, id: undefined } : m,
),
},
];
for (const file of bad) {
expect(backupPayloadSchema.safeParse(file).success).toBe(false);
}
});
});
describe("storedPreviewFor", () => {
const chunked = { data: null, chunkCount: 4, chunkStreamId: "s1" };
const preview = () =>
@@ -31,11 +31,13 @@ vi.mock("@/lib/crypto", () => ({
vi.mock("@/lib/crypto/note-cipher", () => ({ encryptNote: vi.fn() }));
vi.mock("@/lib/validations/backup", () => ({
parseBackupPayload: vi.fn(),
// The streamed reader checks measurements one at a time; this file has none.
backupMeasurementSchema: { safeParse: vi.fn() },
}));
vi.mock("@/lib/validations/backup-summary", () => ({
isCompatibleSchemaVersion: vi.fn(() => true),
summarizeBackup: vi.fn(() => ({})),
// The streamed reader checks measurements one at a time; this file has none.
BACKUP_SCHEMA_VERSION: "2",
backupMeasurementSchema: { safeParse: vi.fn() },
}));
vi.mock("@/lib/auth/audit", () => ({ auditLog: vi.fn() }));
vi.mock("@/lib/logging/context", () => ({ annotate: vi.fn() }));
+62 -6
View File
@@ -35,7 +35,10 @@ import {
} from "@/lib/export/backup-key-ids";
import type { FullBackupCounts } from "@/lib/export/full-backup-payload";
import type { StoredBackupRef } from "@/lib/export/stored-backup";
import type { BackupSummary } from "@/lib/validations/backup";
import {
BACKUP_SCHEMA_VERSION,
type BackupSummary,
} from "@/lib/validations/backup-summary";
export interface BackupPreview {
version: 1;
@@ -207,7 +210,9 @@ export interface BackupPreviewCollector {
observe(member: string, value: unknown): void;
/**
* The counts and key uses, once the writer has finished; null when the
* copy did not name its schema version, owner and date.
* copy did not name its schema version, owner and date, or when a section
* did not pass the backup schema. A null preview sends the preview route
* to read the copy itself, which then says why it cannot be restored.
*/
finish(counts: FullBackupCounts): {
summary: BackupSummary;
@@ -215,19 +220,64 @@ export interface BackupPreviewCollector {
} | null;
}
type SchemaModule = typeof import("@/lib/validations/backup");
/**
* A check of one member of the payload against `backupPayloadSchema`, as the
* writer shows it: a whole section, or one row of a bulk table. An array
* section is checked an element at a time, so the check holds one element's
* copy at most, never a section's. Each value is checked in its JSON form,
* exactly as it lands in the copy. A member the schema does not name passes,
* as it does under the schema's passthrough.
*/
function sectionChecker(
mod: SchemaModule,
): (member: string, value: unknown) => boolean {
const shape = mod.backupPayloadSchema.shape as Record<string, z.ZodType>;
const asWritten = (value: unknown): unknown =>
value === undefined ? undefined : JSON.parse(JSON.stringify(value));
const unwrap = (schema: z.ZodType): z.ZodType =>
schema instanceof z.ZodDefault
? unwrap(schema.unwrap() as z.ZodType)
: schema;
return (member, value) => {
const declared = shape[member];
if (!declared) return true;
const section = unwrap(declared);
if (section instanceof z.ZodArray) {
const element = section.element as z.ZodType;
const rows = Array.isArray(value) ? value : [value];
return rows.every((row) => element.safeParse(asWritten(row)).success);
}
return declared.safeParse(asWritten(value)).success;
};
}
/**
* Work out a weekly copy's preview from what its writer already has in hand:
* each section as it is written, for the key uses, and the writer's own
* counts. Nothing is parsed and nothing is kept but the key samples, so the
* preview costs the pass no memory that grows with the record.
* each section as it is written, for the key uses and the schema check, and
* the writer's own counts. Nothing is parsed back and nothing is kept but the
* key samples, so the preview costs the pass no memory that grows with the
* record. The check loads the backup schema (`backup-summary.ts` says why it
* is loaded on first use), a fixed cost of the module, not of the record.
*/
export function createBackupPreviewCollector(): BackupPreviewCollector {
export async function createBackupPreviewCollector(): Promise<BackupPreviewCollector> {
const check = sectionChecker(await import("@/lib/validations/backup"));
const keys = new BackupKeyIdCollector();
const header: Partial<
Record<"schemaVersion" | "userId" | "exportedAt", string>
> = {};
let valid = true;
let measurementWithoutId = false;
return {
observe(member, value) {
if (valid && !check(member, value)) valid = false;
if (member === "measurements") {
const rows: unknown[] = Array.isArray(value) ? value : [value];
if (rows.some((row) => !(row as { id?: unknown } | null)?.id)) {
measurementWithoutId = true;
}
}
if (
(member === "schemaVersion" ||
member === "userId" ||
@@ -242,6 +292,12 @@ export function createBackupPreviewCollector(): BackupPreviewCollector {
finish(counts) {
const { schemaVersion, userId, exportedAt } = header;
if (!schemaVersion || !userId || !exportedAt) return null;
if (!valid) return null;
// The schema's one whole-document rule: a canonical copy names every
// measurement by a stable id.
if (schemaVersion === BACKUP_SCHEMA_VERSION && measurementWithoutId) {
return null;
}
const summary = { schemaVersion, userId, exportedAt } as BackupSummary;
for (const field of SUMMARY_COUNTS) summary[field] = counts[field];
return { summary, keys };
+1 -1
View File
@@ -27,7 +27,7 @@ import type { Prisma, PrismaClient } from "@/generated/prisma/client";
import { readNote } from "@/lib/crypto/note-cipher";
import { openSealedForExport } from "@/lib/export/sealed-text";
import { iterateMeasurementPages } from "@/lib/export/paged-measurements";
import { BACKUP_SCHEMA_VERSION } from "@/lib/validations/backup";
import { BACKUP_SCHEMA_VERSION } from "@/lib/validations/backup-summary";
import {
buildCycleBackupSection,
type CycleBackupSection,
+5 -3
View File
@@ -71,13 +71,12 @@ import { encryptToBytes } from "@/lib/ai/coach/bytes-codec";
import { encryptContextToBytes } from "@/lib/labs/biomarker-store";
import { encryptNoteToBytes } from "@/lib/labs/store";
import { annotate } from "@/lib/logging/context";
import type { BackupMeasurement } from "@/lib/validations/backup";
import {
parseBackupPayload,
isCompatibleSchemaVersion,
summarizeBackup,
type BackupMeasurement,
type BackupSummary,
} from "@/lib/validations/backup";
} from "@/lib/validations/backup-summary";
import { recomputeUserMoodRollups } from "@/lib/rollups/mood-rollups";
import {
recomputeUserMedicationCompliance,
@@ -431,6 +430,9 @@ export async function restoreBackup(
progress.measurementsTotal = streamed.measurementCount;
report("validating", true);
raw = streamed.raw;
// Loaded here, not at import: the schema is ~12 MB of heap and a server
// that never validates a backup should not hold it (`backup-summary.ts`).
const { parseBackupPayload } = await import("@/lib/validations/backup");
payload = parseBackupPayload(raw);
} catch (err) {
const readFailure = isStoredBackupReadError(err);
+1 -1
View File
@@ -48,7 +48,7 @@ import {
newChunkStreamId,
} from "@/lib/export/backup-chunks";
import { BackupFormsConflictError } from "@/lib/export/stored-backup";
import type { BackupSummary } from "@/lib/validations/backup";
import type { BackupSummary } from "@/lib/validations/backup-summary";
/**
* How long the storing transaction may stay open. The job around it expires
+23 -7
View File
@@ -23,11 +23,22 @@
*/
import { BackupKeyIdCollector } from "@/lib/export/backup-key-ids";
import { scanBackupJson } from "@/lib/export/backup-json-scan";
import {
BACKUP_SCHEMA_VERSION,
import type {
BackupMeasurement,
backupMeasurementSchema,
type BackupMeasurement,
} from "@/lib/validations/backup";
import { BACKUP_SCHEMA_VERSION } from "@/lib/validations/backup-summary";
type MeasurementSchema = typeof backupMeasurementSchema;
/**
* The element schema, loaded with the rest of `backup.ts` on the first read
* rather than at import: the module sits on the boot path of the restore job
* and the schema is ~12 MB of heap (`backup-summary.ts`).
*/
async function loadMeasurementSchema(): Promise<MeasurementSchema> {
return (await import("@/lib/validations/backup")).backupMeasurementSchema;
}
/** The file's bytes, from the start, every time it is called. */
export type BackupSource = () => AsyncIterable<Uint8Array>;
@@ -69,8 +80,12 @@ export interface StreamedBackup {
): Promise<void>;
}
function parseMeasurement(element: unknown, index: number): BackupMeasurement {
const parsed = backupMeasurementSchema.safeParse(element);
function parseMeasurement(
schema: MeasurementSchema,
element: unknown,
index: number,
): BackupMeasurement {
const parsed = schema.safeParse(element);
if (!parsed.success) {
const issue = parsed.error.issues[0];
throw new StreamedBackupInvalidError(
@@ -98,6 +113,7 @@ export async function readStreamedBackup(
source: BackupSource,
options: ReadStreamedBackupOptions = {},
): Promise<StreamedBackup> {
const schema = await loadMeasurementSchema();
let firstWithoutId: number | null = null;
let bytes = 0;
const keys = new BackupKeyIdCollector();
@@ -110,7 +126,7 @@ export async function readStreamedBackup(
const { document, streamedCounts } = await scanBackupJson(counted(), {
streamKeys: STREAMED,
onElement: (_key, element, index) => {
const row = parseMeasurement(element, index);
const row = parseMeasurement(schema, element, index);
if (!row.id && firstWithoutId === null) firstWithoutId = index;
keys.visit(element, MEASUREMENTS);
options.onMeasurementChecked?.(index + 1);
@@ -146,7 +162,7 @@ export async function readStreamedBackup(
streamKeys: STREAMED,
skipKeys,
onElement: async (_key, element, index) => {
batch.push(parseMeasurement(element, index));
batch.push(parseMeasurement(schema, element, index));
if (batch.length >= size) {
const full = batch;
batch = [];
+501 -433
View File
@@ -54,7 +54,10 @@ import {
computeSignalsOfDay,
type SignalOfDay,
} from "@/lib/insights/signals-of-day";
import type { RollupGranularity } from "@/generated/prisma/client";
import type {
MeasurementType,
RollupGranularity,
} from "@/generated/prisma/client";
import { resolveWeightTargetOverride } from "@/lib/analytics/effective-range";
import {
@@ -977,439 +980,19 @@ export function avgInWindow(
const INTEGRATION_BLOCK_MIN_WINDOW_DAYS = 180;
/**
* v1.25 — newest-first row cap on the bulk measurement read in
* `extractFeatures`. Mirrors the Coach snapshot's `SNAPSHOT_MEASUREMENT_ROW_CAP`
* (6000): the prompt / insights aggregates only ever fold this read into a
* bounded set of summaries, so a year of dense PULSE / glucose rows is wasted
* I/O on the shared pool for a heavy-data tenant. Full-history extremes come
* from `readAllTimeExtremes`, not this read, so the cap stays correct.
* The mood feature block and the per-day points it was built from. Split out of
* `extractFeatures` so a caller that needs only this block (the single-metric
* Coach read) runs exactly the same reads without the rest of the function.
*/
const FEATURE_MEASUREMENT_ROW_CAP = 6000;
export async function extractFeatures(
async function readMoodFeature(
userId: string,
includeRaw: boolean,
options: {
sinceDays?: number;
/**
* The Coach's lookback floor. Given, nothing older than it is read: the
* `allTime*` figures cover the history from the floor on, the mood
* history and its sub-blocks start at it, and the mood forecast, a model
* fitted on the whole history, is left out.
*/
historyFloor?: Date;
} = {},
): Promise<AggregatedFeatures | RawFeatures> {
const historyFloor = options.historyFloor ?? null;
const user = await prisma.user.findUnique({
where: { id: userId },
select: {
heightCm: true,
dateOfBirth: true,
gender: true,
thresholdsJson: true,
},
});
const now = Date.now();
// Fetch measurements. Default = ALL (full temporal context for the
// dashboard / insights generator). Callers that only consume the
// ≤90-day windows (Coach snapshot per turn) pass `sinceDays: 90` so
// the per-turn I/O stays bounded — `findMany({ where: { userId } })`
// is unbounded by user-history size and gets paid once per Coach turn
// for power users with multi-year Withings imports.
const sinceDays = options.sinceDays;
const sinceCutoff =
typeof sinceDays === "number" && sinceDays > 0
? new Date(now - sinceDays * 24 * 60 * 60 * 1000)
: null;
// v1.25 — newest-first cap on the bulk feature read, mirroring the Coach
// snapshot (`SNAPSHOT_MEASUREMENT_ROW_CAP`). Callers pass windows of 365-400
// days, and PULSE / glucose are 200k-row-class types, so an uncapped read
// pulled hundreds of thousands of rows per Coach turn and nightly briefing
// only to fold them into a handful of summaries. Read newest-first, cap, then
// reverse so every downstream consumer (`byType`, `summarize`, BP pairing,
// sleep reconstruction, oldest/newest span) still sees ascending order. The
// cap is safe even unbounded: the genuine all-time extremes are sourced
// separately via `readAllTimeExtremes` on the bounded branch, so capping only
// sheds the deepest rows of the bulk read and never relabels an extreme.
const measurements = await prisma.measurement
.findMany({
where: sinceCutoff
? { userId, measuredAt: { gte: sinceCutoff }, deletedAt: null }
: { userId, deletedAt: null },
orderBy: { measuredAt: "desc" },
take: FEATURE_MEASUREMENT_ROW_CAP,
// Project only the columns every downstream consumer reads (`byType`,
// `summarize`, BP pairing, and `reconstructSleepNights`'s `SleepStageRow`).
// The PULSE / glucose windows are 200k-row-class; pulling every column
// (notes, externalId, …) is pure wasted I/O on the shared Prisma pool.
select: {
type: true,
value: true,
measuredAt: true,
sleepStage: true,
source: true,
deviceType: true,
},
})
// Restore ascending order so order-sensitive consumers (oldest/newest span,
// BP pairing, sleep reconstruction) see the same shape as before the cap.
.then((rows) => rows.reverse());
// v1.18.11 P1 — when the bulk read is bounded to a recent window, the
// windowed `summarize()` no longer covers the full history, so the `allTime*`
// fields would silently become "last `sinceDays` days" extremes. Source the
// genuine full-history min / max / mean from one grouped aggregation (no row
// materialisation) so the labels stay honest. Unbounded reads (sinceCutoff
// null) already see the whole history and skip the extra query.
const allTimeExtremes = sinceCutoff
? await readAllTimeExtremes(
userId,
["WEIGHT", "BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA", "PULSE"],
historyFloor,
)
: null;
const byType = (type: string) => measurements.filter((m) => m.type === type);
// The zone pulse days are read in; resolved only when there is pulse to read.
const pulseTz = measurements.some((m) => m.type === "PULSE")
? await resolveUserTimezone(userId)
: DEFAULT_TIMEZONE;
const bpTargets = getBpTargets(user?.dateOfBirth ?? null);
// Compute overall data span
const oldestMeasurement =
measurements.length > 0 ? measurements[0].measuredAt : null;
const newestMeasurement =
measurements.length > 0
? measurements[measurements.length - 1].measuredAt
: null;
const overallSpanDays =
oldestMeasurement && newestMeasurement
? Math.round(
(newestMeasurement.getTime() - oldestMeasurement.getTime()) /
(24 * 60 * 60 * 1000),
)
: 0;
// Compute age
let ageYears: number | null = null;
if (user?.dateOfBirth) {
const dob = user.dateOfBirth;
const today = new Date();
ageYears = today.getFullYear() - dob.getFullYear();
const monthDiff = today.getMonth() - dob.getMonth();
if (monthDiff < 0 || (monthDiff === 0 && today.getDate() < dob.getDate())) {
ageYears--;
}
}
const features: AggregatedFeatures = {
context: {
heightCm: user?.heightCm ?? null,
hasBpTargets: !!bpTargets,
totalMeasurements: measurements.length,
dataSpanDays: overallSpanDays,
oldestMeasurementDaysAgo: oldestMeasurement
? Math.round(
(now - oldestMeasurement.getTime()) / (24 * 60 * 60 * 1000),
)
: null,
newestMeasurementDaysAgo: newestMeasurement
? Math.round(
(now - newestMeasurement.getTime()) / (24 * 60 * 60 * 1000),
)
: null,
ageYears,
gender: user?.gender ?? null,
},
};
// Weight
const weightData = byType("WEIGHT");
if (weightData.length > 0) {
const summary = summarize(toDataPoints(weightData));
const bmi =
user?.heightCm && summary.latest
? parseFloat((summary.latest / (user.heightCm / 100) ** 2).toFixed(1))
: null;
features.weight = {
latest: summary.latest!,
avg7: summary.avg7,
avg30: summary.avg30,
avg90: avgInWindow(weightData, now, 90),
// v1.18.11 P1 — full-history extremes from the grouped aggregation when
// the bulk read is windowed; the in-window summary otherwise.
allTimeAvg: allTimeExtremes
? roundMean(allTimeExtremes.get("WEIGHT")?.mean ?? null)
: summary.count > 0
? summary.mean
: null,
allTimeMin: allTimeExtremes
? (allTimeExtremes.get("WEIGHT")?.min ?? null)
: summary.count > 0
? summary.min
: null,
allTimeMax: allTimeExtremes
? (allTimeExtremes.get("WEIGHT")?.max ?? null)
: summary.count > 0
? summary.max
: null,
slope30: summary.slope30?.slope ?? null,
outlierCount: summary.anomalyCount ?? 0,
bmi,
coverage: computeCoverage(weightData, now),
};
const weightTarget = buildWeightTargetFeature(
resolveWeightTargetOverride(user?.thresholdsJson),
{ avg7: summary.avg7, latest: summary.latest },
);
if (weightTarget) features.weight.target = weightTarget;
}
// Blood Pressure
const sysData = byType("BLOOD_PRESSURE_SYS");
const diaData = byType("BLOOD_PRESSURE_DIA");
if (sysData.length > 0 || diaData.length > 0) {
const sysSummary =
sysData.length > 0 ? summarize(toDataPoints(sysData)) : null;
const diaSummary =
diaData.length > 0 ? summarize(toDataPoints(diaData)) : null;
let pctInTarget: number | null = null;
if (bpTargets) {
const sysByTime = new Map(
sysData.map((m) => [m.measuredAt.getTime(), m.value]),
);
let inTargetCount = 0;
let pairedCount = 0;
for (const dia of diaData) {
const sysVal = sysByTime.get(dia.measuredAt.getTime());
if (sysVal === undefined) continue;
pairedCount++;
// v1.4.16 A2 — one-sided ceiling semantics with hypotension
// floor. See lib/analytics/bp-in-target.ts.
if (isBpReadingInTarget(sysVal, dia.value, bpTargets)) {
inTargetCount++;
}
}
pctInTarget =
pairedCount > 0
? Math.round((inTargetCount / pairedCount) * 100)
: null;
}
features.bloodPressure = {
avgSys30: sysSummary?.avg30 ?? null,
avgDia30: diaSummary?.avg30 ?? null,
avgSys90: sysData.length > 0 ? avgInWindow(sysData, now, 90) : null,
avgDia90: diaData.length > 0 ? avgInWindow(diaData, now, 90) : null,
// v1.18.11 P1 — full-history extremes when the bulk read is windowed.
allTimeAvgSys: allTimeExtremes
? roundMean(allTimeExtremes.get("BLOOD_PRESSURE_SYS")?.mean ?? null)
: sysSummary?.count
? sysSummary.mean
: null,
allTimeAvgDia: allTimeExtremes
? roundMean(allTimeExtremes.get("BLOOD_PRESSURE_DIA")?.mean ?? null)
: diaSummary?.count
? diaSummary.mean
: null,
allTimeMinSys: allTimeExtremes
? (allTimeExtremes.get("BLOOD_PRESSURE_SYS")?.min ?? null)
: sysSummary?.count
? sysSummary.min
: null,
allTimeMaxSys: allTimeExtremes
? (allTimeExtremes.get("BLOOD_PRESSURE_SYS")?.max ?? null)
: sysSummary?.count
? sysSummary.max
: null,
allTimeMinDia: allTimeExtremes
? (allTimeExtremes.get("BLOOD_PRESSURE_DIA")?.min ?? null)
: diaSummary?.count
? diaSummary.min
: null,
allTimeMaxDia: allTimeExtremes
? (allTimeExtremes.get("BLOOD_PRESSURE_DIA")?.max ?? null)
: diaSummary?.count
? diaSummary.max
: null,
slopeSys30: sysSummary?.slope30?.slope ?? null,
slopeDia30: diaSummary?.slope30?.slope ?? null,
sdSys30: (() => {
const fromMs = now - 30 * 24 * 60 * 60 * 1000;
const vals = sysData
.filter((m) => m.measuredAt.getTime() >= fromMs)
.map((m) => m.value);
return stdDev(vals);
})(),
sdDia30: (() => {
const fromMs = now - 30 * 24 * 60 * 60 * 1000;
const vals = diaData
.filter((m) => m.measuredAt.getTime() >= fromMs)
.map((m) => m.value);
return stdDev(vals);
})(),
pulsePressure30: (() => {
const avgSys = sysSummary?.avg30 ?? null;
const avgDia = diaSummary?.avg30 ?? null;
if (avgSys === null || avgDia === null) return null;
return Math.round((avgSys - avgDia) * 10) / 10;
})(),
pctInTarget,
coverage: computeCoverage(
[...sysData, ...diaData].sort(
(a, b) => a.measuredAt.getTime() - b.measuredAt.getTime(),
),
now,
),
};
}
// Pulse
const pulseData = byType("PULSE");
if (pulseData.length > 0) {
const summary = summarize(toDataPoints(pulseData));
// A day of pulse is the mean of its local hours' means and a window the
// mean of its days (see `day-mean.ts`): `summarize` means every reading,
// so the means come from the shared helper over the same windows.
const pulseDays = { type: "PULSE", tz: pulseTz };
const pulseMean = (fromDaysAgo: number) =>
avgInWindow(pulseData, now, fromDaysAgo, 0, pulseDays);
const allPulse = readingsMean("PULSE", pulseData, pulseTz);
features.pulse = {
avg7: pulseMean(7),
avg30: pulseMean(30),
avg90: pulseMean(90),
// v1.18.11 P1 — full-history extremes when the bulk read is windowed.
allTimeAvg: allTimeExtremes
? roundMean(allTimeExtremes.get("PULSE")?.mean ?? null)
: allPulse === null
? null
: Math.round(allPulse * 100) / 100,
allTimeMin: allTimeExtremes
? (allTimeExtremes.get("PULSE")?.min ?? null)
: summary.count > 0
? summary.min
: null,
allTimeMax: allTimeExtremes
? (allTimeExtremes.get("PULSE")?.max ?? null)
: summary.count > 0
? summary.max
: null,
slope30: summary.slope30?.slope ?? null,
anomalyCount: summary.anomalyCount ?? 0,
coverage: computeCoverage(pulseData, now),
};
}
// Body Fat
const fatData = byType("BODY_FAT");
if (fatData.length > 0) {
const summary = summarize(toDataPoints(fatData));
features.bodyFat = {
latest: summary.latest,
avg30: summary.avg30,
slope30: summary.slope30?.slope ?? null,
coverage: computeCoverage(fatData, now),
};
}
// Grip strength (kg) — clinical-depth signal; the briefing narrates the
// trajectory, the sex-aware EWGSOP2 floor stays at the display edge.
const gripData = byType("GRIP_STRENGTH");
if (gripData.length > 0) {
const summary = summarize(toDataPoints(gripData));
features.gripStrength = {
latest: summary.latest,
avg30: summary.avg30,
slope30: summary.slope30?.slope ?? null,
coverage: computeCoverage(gripData, now),
};
}
// Waist circumference (cm) + waist-to-height ratio. WHtR is computed from the
// freshest circumference and the user's height (the same canonical derivation
// the detail page uses); omitted when height is unknown.
const waistData = byType("WAIST_CIRCUMFERENCE");
if (waistData.length > 0) {
const summary = summarize(toDataPoints(waistData));
const whtrLatest =
user?.heightCm && summary.latest
? Math.round((summary.latest / user.heightCm) * 100) / 100
: null;
features.waist = {
latest: summary.latest,
avg30: summary.avg30,
slope30: summary.slope30?.slope ?? null,
whtrLatest,
coverage: computeCoverage(waistData, now),
};
}
// Pain (0–10 NRS, lower-better) — surface a sustained or rising pain burden.
const painData = byType("PAIN_NRS");
if (painData.length > 0) {
const summary = summarize(toDataPoints(painData));
features.pain = {
latest: summary.latest,
avg7: summary.avg7,
avg30: summary.avg30,
slope30: summary.slope30?.slope ?? null,
coverage: computeCoverage(painData, now),
};
}
// Sleep Duration
//
// SLEEP_DURATION is stored ONE ROW PER STAGE per night, so summarising the
// raw stage rows would average individual stages (and double-count a bare
// ASLEEP aggregate against its granular CORE/DEEP/REM twin). Route the
// feature block through the per-night dedup reconstruction — the same helper
// the dashboard / series / status path use — so `avg7` / `avg30` / `latest`
// are per-night TIME-ASLEEP totals in minutes, never stage averages.
const sleepData = byType("SLEEP_DURATION");
if (sleepData.length > 0) {
const [sleepTz, sleepPriority] = await Promise.all([
resolveUserTimezone(userId),
loadUserSourcePriority(userId),
]);
const sleepNights = reconstructSleepNights(
sleepData as unknown as SleepStageRow[],
sleepTz,
sleepPriority,
).filter((n) => n.asleepMinutes > 0);
const summary = summarize(
sleepNights.map((n) => ({ date: n.measuredAt, value: n.asleepMinutes })),
);
features.sleep = {
avg7: summary.avg7,
avg30: summary.avg30,
latest: summary.latest,
coverage: computeCoverage(
sleepNights.map((n) => ({ measuredAt: n.measuredAt })),
now,
),
};
}
// Activity Steps
const activityData = byType("ACTIVITY_STEPS");
if (activityData.length > 0) {
const summary = summarize(toDataPoints(activityData));
features.activity = {
avg7: summary.avg7,
avg30: summary.avg30,
latest: summary.latest,
coverage: computeCoverage(activityData, now),
};
}
/** The Coach's lookback floor, or null: see `extractFeatures`. */
historyFloor: Date | null,
): Promise<{
feature: AggregatedFeatures["mood"];
dailyPoints: Array<{ measuredAt: Date; value: number; count: number }>;
}> {
let feature: AggregatedFeatures["mood"];
// Mood
//
// v1.4.40 — swap the unbounded `prisma.moodEntry.findMany` for the
@@ -1553,7 +1136,7 @@ export async function extractFeatures(
? Math.round((spanDays / (moodTotalEntries - 1)) * 10) / 10
: null;
features.mood = {
feature = {
scale: "1=LAUSIG, 2=SCHLECHT, 3=OKAY, 4=GUT, 5=SUPER_GUT",
avg7: avg(last7),
avg30: avg(last30),
@@ -1577,6 +1160,491 @@ export async function extractFeatures(
};
}
return { feature, dailyPoints: moodDailyPoints };
}
/**
* v1.25 — newest-first row cap on the bulk measurement read in
* `extractFeatures`. Mirrors the Coach snapshot's `SNAPSHOT_MEASUREMENT_ROW_CAP`
* (6000): the prompt / insights aggregates only ever fold this read into a
* bounded set of summaries, so a year of dense PULSE / glucose rows is wasted
* I/O on the shared pool for a heavy-data tenant. Full-history extremes come
* from `readAllTimeExtremes`, not this read, so the cap stays correct.
*/
const FEATURE_MEASUREMENT_ROW_CAP = 6000;
/**
* The feature blocks the single-metric Coach read can ask for on their own.
* Each is built from exactly the reads the full extraction runs for it.
*/
export type ScopedFeatureBlock = "weight" | "bloodPressure" | "pulse" | "mood";
const SCOPED_BLOCK_TYPES: Readonly<
Record<ScopedFeatureBlock, readonly MeasurementType[]>
> = {
weight: ["WEIGHT"],
bloodPressure: ["BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA"],
pulse: ["PULSE"],
mood: [],
};
export async function extractFeatures(
userId: string,
includeRaw: boolean,
options: {
sinceDays?: number;
/**
* Build only these blocks (plus `context`) and return. Each block is
* identical to the one the full extraction builds: the bulk read keeps its
* all-type newest-first cap, so the rows a block sees do not change. Used
* by the single-metric Coach read, which reads one block and nothing else.
*/
only?: ReadonlySet<ScopedFeatureBlock>;
/**
* The Coach's lookback floor. Given, nothing older than it is read: the
* `allTime*` figures cover the history from the floor on, the mood
* history and its sub-blocks start at it, and the mood forecast, a model
* fitted on the whole history, is left out.
*/
historyFloor?: Date;
} = {},
): Promise<AggregatedFeatures | RawFeatures> {
const only = options.only;
const historyFloor = options.historyFloor ?? null;
const wants = (block: ScopedFeatureBlock) => !only || only.has(block);
const allTimeTypes = (["weight", "bloodPressure", "pulse"] as const).flatMap(
(block) => (wants(block) ? SCOPED_BLOCK_TYPES[block] : []),
);
// The bulk read feeds every block except mood; a mood-only read skips it.
const needsMeasurementRows = !only || allTimeTypes.length > 0;
const user = await prisma.user.findUnique({
where: { id: userId },
select: {
heightCm: true,
dateOfBirth: true,
gender: true,
thresholdsJson: true,
},
});
const now = Date.now();
// Fetch measurements. Default = ALL (full temporal context for the
// dashboard / insights generator). Callers that only consume the
// ≤90-day windows (Coach snapshot per turn) pass `sinceDays: 90` so
// the per-turn I/O stays bounded — `findMany({ where: { userId } })`
// is unbounded by user-history size and gets paid once per Coach turn
// for power users with multi-year Withings imports.
const sinceDays = options.sinceDays;
const sinceCutoff =
typeof sinceDays === "number" && sinceDays > 0
? new Date(now - sinceDays * 24 * 60 * 60 * 1000)
: null;
// v1.25 — newest-first cap on the bulk feature read, mirroring the Coach
// snapshot (`SNAPSHOT_MEASUREMENT_ROW_CAP`). Callers pass windows of 365-400
// days, and PULSE / glucose are 200k-row-class types, so an uncapped read
// pulled hundreds of thousands of rows per Coach turn and nightly briefing
// only to fold them into a handful of summaries. Read newest-first, cap, then
// reverse so every downstream consumer (`byType`, `summarize`, BP pairing,
// sleep reconstruction, oldest/newest span) still sees ascending order. The
// cap is safe even unbounded: the genuine all-time extremes are sourced
// separately via `readAllTimeExtremes` on the bounded branch, so capping only
// sheds the deepest rows of the bulk read and never relabels an extreme.
const measurements = !needsMeasurementRows
? []
: await prisma.measurement
.findMany({
where: sinceCutoff
? { userId, measuredAt: { gte: sinceCutoff }, deletedAt: null }
: { userId, deletedAt: null },
orderBy: { measuredAt: "desc" },
take: FEATURE_MEASUREMENT_ROW_CAP,
// Project only the columns every downstream consumer reads (`byType`,
// `summarize`, BP pairing, and `reconstructSleepNights`'s `SleepStageRow`).
// The PULSE / glucose windows are 200k-row-class; pulling every column
// (notes, externalId, …) is pure wasted I/O on the shared Prisma pool.
select: {
type: true,
value: true,
measuredAt: true,
sleepStage: true,
source: true,
deviceType: true,
},
})
// Restore ascending order so order-sensitive consumers (oldest/newest span,
// BP pairing, sleep reconstruction) see the same shape as before the cap.
.then((rows) => rows.reverse());
// v1.18.11 P1 — when the bulk read is bounded to a recent window, the
// windowed `summarize()` no longer covers the full history, so the `allTime*`
// fields would silently become "last `sinceDays` days" extremes. Source the
// genuine full-history min / max / mean from one grouped aggregation (no row
// materialisation) so the labels stay honest. Unbounded reads (sinceCutoff
// null) already see the whole history and skip the extra query.
// Grouped by type, so asking for fewer types leaves each one's figures as
// they were.
const allTimeExtremes = sinceCutoff
? await readAllTimeExtremes(
userId,
only
? allTimeTypes
: ["WEIGHT", "BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA", "PULSE"],
historyFloor,
)
: null;
const byType = (type: string) => measurements.filter((m) => m.type === type);
// The zone pulse days are read in; resolved only when there is pulse to read.
const pulseTz = measurements.some((m) => m.type === "PULSE")
? await resolveUserTimezone(userId)
: DEFAULT_TIMEZONE;
const bpTargets = getBpTargets(user?.dateOfBirth ?? null);
// Compute overall data span
const oldestMeasurement =
measurements.length > 0 ? measurements[0].measuredAt : null;
const newestMeasurement =
measurements.length > 0
? measurements[measurements.length - 1].measuredAt
: null;
const overallSpanDays =
oldestMeasurement && newestMeasurement
? Math.round(
(newestMeasurement.getTime() - oldestMeasurement.getTime()) /
(24 * 60 * 60 * 1000),
)
: 0;
// Compute age
let ageYears: number | null = null;
if (user?.dateOfBirth) {
const dob = user.dateOfBirth;
const today = new Date();
ageYears = today.getFullYear() - dob.getFullYear();
const monthDiff = today.getMonth() - dob.getMonth();
if (monthDiff < 0 || (monthDiff === 0 && today.getDate() < dob.getDate())) {
ageYears--;
}
}
const features: AggregatedFeatures = {
context: {
heightCm: user?.heightCm ?? null,
hasBpTargets: !!bpTargets,
totalMeasurements: measurements.length,
dataSpanDays: overallSpanDays,
oldestMeasurementDaysAgo: oldestMeasurement
? Math.round(
(now - oldestMeasurement.getTime()) / (24 * 60 * 60 * 1000),
)
: null,
newestMeasurementDaysAgo: newestMeasurement
? Math.round(
(now - newestMeasurement.getTime()) / (24 * 60 * 60 * 1000),
)
: null,
ageYears,
gender: user?.gender ?? null,
},
};
// Weight
const weightData = byType("WEIGHT");
if (wants("weight") && weightData.length > 0) {
const summary = summarize(toDataPoints(weightData));
const bmi =
user?.heightCm && summary.latest
? parseFloat((summary.latest / (user.heightCm / 100) ** 2).toFixed(1))
: null;
features.weight = {
latest: summary.latest!,
avg7: summary.avg7,
avg30: summary.avg30,
avg90: avgInWindow(weightData, now, 90),
// v1.18.11 P1 — full-history extremes from the grouped aggregation when
// the bulk read is windowed; the in-window summary otherwise.
allTimeAvg: allTimeExtremes
? roundMean(allTimeExtremes.get("WEIGHT")?.mean ?? null)
: summary.count > 0
? summary.mean
: null,
allTimeMin: allTimeExtremes
? (allTimeExtremes.get("WEIGHT")?.min ?? null)
: summary.count > 0
? summary.min
: null,
allTimeMax: allTimeExtremes
? (allTimeExtremes.get("WEIGHT")?.max ?? null)
: summary.count > 0
? summary.max
: null,
slope30: summary.slope30?.slope ?? null,
outlierCount: summary.anomalyCount ?? 0,
bmi,
coverage: computeCoverage(weightData, now),
};
const weightTarget = buildWeightTargetFeature(
resolveWeightTargetOverride(user?.thresholdsJson),
{ avg7: summary.avg7, latest: summary.latest },
);
if (weightTarget) features.weight.target = weightTarget;
}
// Blood Pressure
const sysData = byType("BLOOD_PRESSURE_SYS");
const diaData = byType("BLOOD_PRESSURE_DIA");
if (wants("bloodPressure") && (sysData.length > 0 || diaData.length > 0)) {
const sysSummary =
sysData.length > 0 ? summarize(toDataPoints(sysData)) : null;
const diaSummary =
diaData.length > 0 ? summarize(toDataPoints(diaData)) : null;
let pctInTarget: number | null = null;
if (bpTargets) {
const sysByTime = new Map(
sysData.map((m) => [m.measuredAt.getTime(), m.value]),
);
let inTargetCount = 0;
let pairedCount = 0;
for (const dia of diaData) {
const sysVal = sysByTime.get(dia.measuredAt.getTime());
if (sysVal === undefined) continue;
pairedCount++;
// v1.4.16 A2 — one-sided ceiling semantics with hypotension
// floor. See lib/analytics/bp-in-target.ts.
if (isBpReadingInTarget(sysVal, dia.value, bpTargets)) {
inTargetCount++;
}
}
pctInTarget =
pairedCount > 0
? Math.round((inTargetCount / pairedCount) * 100)
: null;
}
features.bloodPressure = {
avgSys30: sysSummary?.avg30 ?? null,
avgDia30: diaSummary?.avg30 ?? null,
avgSys90: sysData.length > 0 ? avgInWindow(sysData, now, 90) : null,
avgDia90: diaData.length > 0 ? avgInWindow(diaData, now, 90) : null,
// v1.18.11 P1 — full-history extremes when the bulk read is windowed.
allTimeAvgSys: allTimeExtremes
? roundMean(allTimeExtremes.get("BLOOD_PRESSURE_SYS")?.mean ?? null)
: sysSummary?.count
? sysSummary.mean
: null,
allTimeAvgDia: allTimeExtremes
? roundMean(allTimeExtremes.get("BLOOD_PRESSURE_DIA")?.mean ?? null)
: diaSummary?.count
? diaSummary.mean
: null,
allTimeMinSys: allTimeExtremes
? (allTimeExtremes.get("BLOOD_PRESSURE_SYS")?.min ?? null)
: sysSummary?.count
? sysSummary.min
: null,
allTimeMaxSys: allTimeExtremes
? (allTimeExtremes.get("BLOOD_PRESSURE_SYS")?.max ?? null)
: sysSummary?.count
? sysSummary.max
: null,
allTimeMinDia: allTimeExtremes
? (allTimeExtremes.get("BLOOD_PRESSURE_DIA")?.min ?? null)
: diaSummary?.count
? diaSummary.min
: null,
allTimeMaxDia: allTimeExtremes
? (allTimeExtremes.get("BLOOD_PRESSURE_DIA")?.max ?? null)
: diaSummary?.count
? diaSummary.max
: null,
slopeSys30: sysSummary?.slope30?.slope ?? null,
slopeDia30: diaSummary?.slope30?.slope ?? null,
sdSys30: (() => {
const fromMs = now - 30 * 24 * 60 * 60 * 1000;
const vals = sysData
.filter((m) => m.measuredAt.getTime() >= fromMs)
.map((m) => m.value);
return stdDev(vals);
})(),
sdDia30: (() => {
const fromMs = now - 30 * 24 * 60 * 60 * 1000;
const vals = diaData
.filter((m) => m.measuredAt.getTime() >= fromMs)
.map((m) => m.value);
return stdDev(vals);
})(),
pulsePressure30: (() => {
const avgSys = sysSummary?.avg30 ?? null;
const avgDia = diaSummary?.avg30 ?? null;
if (avgSys === null || avgDia === null) return null;
return Math.round((avgSys - avgDia) * 10) / 10;
})(),
pctInTarget,
coverage: computeCoverage(
[...sysData, ...diaData].sort(
(a, b) => a.measuredAt.getTime() - b.measuredAt.getTime(),
),
now,
),
};
}
// Pulse
const pulseData = byType("PULSE");
if (wants("pulse") && pulseData.length > 0) {
const summary = summarize(toDataPoints(pulseData));
// A day of pulse is the mean of its local hours' means and a window the
// mean of its days (see `day-mean.ts`): `summarize` means every reading,
// so the means come from the shared helper over the same windows.
const pulseDays = { type: "PULSE", tz: pulseTz };
const pulseMean = (fromDaysAgo: number) =>
avgInWindow(pulseData, now, fromDaysAgo, 0, pulseDays);
const allPulse = readingsMean("PULSE", pulseData, pulseTz);
features.pulse = {
avg7: pulseMean(7),
avg30: pulseMean(30),
avg90: pulseMean(90),
// v1.18.11 P1 — full-history extremes when the bulk read is windowed.
allTimeAvg: allTimeExtremes
? roundMean(allTimeExtremes.get("PULSE")?.mean ?? null)
: allPulse === null
? null
: Math.round(allPulse * 100) / 100,
allTimeMin: allTimeExtremes
? (allTimeExtremes.get("PULSE")?.min ?? null)
: summary.count > 0
? summary.min
: null,
allTimeMax: allTimeExtremes
? (allTimeExtremes.get("PULSE")?.max ?? null)
: summary.count > 0
? summary.max
: null,
slope30: summary.slope30?.slope ?? null,
anomalyCount: summary.anomalyCount ?? 0,
coverage: computeCoverage(pulseData, now),
};
}
if (only) {
if (only.has("mood")) {
const moodRead = await readMoodFeature(userId, historyFloor);
if (moodRead.feature) features.mood = moodRead.feature;
}
return features;
}
// Body Fat
const fatData = byType("BODY_FAT");
if (fatData.length > 0) {
const summary = summarize(toDataPoints(fatData));
features.bodyFat = {
latest: summary.latest,
avg30: summary.avg30,
slope30: summary.slope30?.slope ?? null,
coverage: computeCoverage(fatData, now),
};
}
// Grip strength (kg) — clinical-depth signal; the briefing narrates the
// trajectory, the sex-aware EWGSOP2 floor stays at the display edge.
const gripData = byType("GRIP_STRENGTH");
if (gripData.length > 0) {
const summary = summarize(toDataPoints(gripData));
features.gripStrength = {
latest: summary.latest,
avg30: summary.avg30,
slope30: summary.slope30?.slope ?? null,
coverage: computeCoverage(gripData, now),
};
}
// Waist circumference (cm) + waist-to-height ratio. WHtR is computed from the
// freshest circumference and the user's height (the same canonical derivation
// the detail page uses); omitted when height is unknown.
const waistData = byType("WAIST_CIRCUMFERENCE");
if (waistData.length > 0) {
const summary = summarize(toDataPoints(waistData));
const whtrLatest =
user?.heightCm && summary.latest
? Math.round((summary.latest / user.heightCm) * 100) / 100
: null;
features.waist = {
latest: summary.latest,
avg30: summary.avg30,
slope30: summary.slope30?.slope ?? null,
whtrLatest,
coverage: computeCoverage(waistData, now),
};
}
// Pain (0–10 NRS, lower-better) — surface a sustained or rising pain burden.
const painData = byType("PAIN_NRS");
if (painData.length > 0) {
const summary = summarize(toDataPoints(painData));
features.pain = {
latest: summary.latest,
avg7: summary.avg7,
avg30: summary.avg30,
slope30: summary.slope30?.slope ?? null,
coverage: computeCoverage(painData, now),
};
}
// Sleep Duration
//
// SLEEP_DURATION is stored ONE ROW PER STAGE per night, so summarising the
// raw stage rows would average individual stages (and double-count a bare
// ASLEEP aggregate against its granular CORE/DEEP/REM twin). Route the
// feature block through the per-night dedup reconstruction — the same helper
// the dashboard / series / status path use — so `avg7` / `avg30` / `latest`
// are per-night TIME-ASLEEP totals in minutes, never stage averages.
const sleepData = byType("SLEEP_DURATION");
if (sleepData.length > 0) {
const [sleepTz, sleepPriority] = await Promise.all([
resolveUserTimezone(userId),
loadUserSourcePriority(userId),
]);
const sleepNights = reconstructSleepNights(
sleepData as unknown as SleepStageRow[],
sleepTz,
sleepPriority,
).filter((n) => n.asleepMinutes > 0);
const summary = summarize(
sleepNights.map((n) => ({ date: n.measuredAt, value: n.asleepMinutes })),
);
features.sleep = {
avg7: summary.avg7,
avg30: summary.avg30,
latest: summary.latest,
coverage: computeCoverage(
sleepNights.map((n) => ({ measuredAt: n.measuredAt })),
now,
),
};
}
// Activity Steps
const activityData = byType("ACTIVITY_STEPS");
if (activityData.length > 0) {
const summary = summarize(toDataPoints(activityData));
features.activity = {
avg7: summary.avg7,
avg30: summary.avg30,
latest: summary.latest,
coverage: computeCoverage(activityData, now),
};
}
// Mood — see `readMoodFeature`.
const moodRead = await readMoodFeature(userId, historyFloor);
const moodDailyPoints = moodRead.dailyPoints;
if (moodRead.feature) features.mood = moodRead.feature;
// Cross-metric correlations
const weightPoints = toDataPoints(weightData);
const sysPoints = toDataPoints(sysData);
@@ -0,0 +1,118 @@
import { describe, expect, it } from "vitest";
import {
heldBackAccounts,
orderInterruptedLast,
} from "../backup-pass-attempts";
type Row = { userId: string; startedAt: Date; finishedAt: Date | null };
/**
* A weekly-style pass over `ids`: oldest copy first, held-back accounts last,
* at most `budget` accounts per run, `kills` the accounts whose record kills
* the process. Returns the accounts copied in each run.
*/
function simulate(opts: {
ids: string[];
budget: number;
kills: Set<string>;
runs: number;
interruptedFirst?: string;
}): string[][] {
let clock = 0;
const attempts = new Map<string, Row>();
const copiedAt = new Map<string, number>();
if (opts.interruptedFirst) {
attempts.set(opts.interruptedFirst, {
userId: opts.interruptedFirst,
startedAt: new Date(++clock),
finishedAt: null,
});
}
const copied: string[][] = [];
for (let run = 0; run < opts.runs; run += 1) {
const order = orderInterruptedLast(
[...opts.ids]
.map((id) => ({ id }))
.sort(
(a, b) => (copiedAt.get(a.id) ?? -1) - (copiedAt.get(b.id) ?? -1),
),
heldBackAccounts([...attempts.values()]),
);
const thisRun: string[] = [];
for (const { id } of order.slice(0, opts.budget)) {
const row = attempts.get(id) ?? {
userId: id,
startedAt: new Date(0),
finishedAt: null,
};
row.startedAt = new Date(++clock);
attempts.set(id, row);
if (opts.kills.has(id)) break; // the process dies; the run ends here
row.finishedAt = new Date(++clock);
copiedAt.set(id, clock);
thisRun.push(id);
}
copied.push(thisRun);
}
return copied;
}
describe("heldBackAccounts", () => {
const at = (ms: number) => new Date(ms);
it("holds back an interrupted attempt that is the newest of the pass", () => {
const held = heldBackAccounts([
{ userId: "x", startedAt: at(30), finishedAt: at(10) },
{ userId: "a", startedAt: at(20), finishedAt: at(21) },
]);
expect([...held.keys()]).toEqual(["x"]);
});
it("releases it once a later run has started on another account", () => {
const held = heldBackAccounts([
{ userId: "x", startedAt: at(30), finishedAt: null },
{ userId: "a", startedAt: at(40), finishedAt: at(41) },
]);
expect(held.size).toBe(0);
});
it("ignores finished attempts", () => {
const held = heldBackAccounts([
{ userId: "a", startedAt: at(40), finishedAt: at(41) },
]);
expect(held.size).toBe(0);
});
});
describe("the order across runs", () => {
const ids = ["x", "a", "b", "c", "d"];
it("keeps a record that kills the process from costing the others their copies", () => {
// `x` has the oldest copy and kills every run that reaches it.
const runs = simulate({ ids, budget: 99, kills: new Set(["x"]), runs: 6 });
// The first run meets it unwarned; from then on it goes last every run,
// because each crash makes its attempt the newest again.
expect(runs[0]).toEqual([]);
for (const run of runs.slice(1)) {
expect([...run].sort()).toEqual(["a", "b", "c", "d"]);
}
});
it("still copies an interrupted account when every run stops on its budget", () => {
// `x` was interrupted once (a restart, not its record). Every run stops
// after three accounts, so kept last for good it would never be reached.
const runs = simulate({
ids,
budget: 3,
kills: new Set(),
runs: 4,
interruptedFirst: "x",
});
const firstCopy = runs.findIndex((run) => run.includes("x"));
expect(firstCopy).toBeGreaterThanOrEqual(0);
expect(firstCopy).toBeLessThanOrEqual(1);
// And every account gets a copy within the four runs.
expect(new Set(runs.flat())).toEqual(new Set(ids));
});
});
@@ -2,10 +2,8 @@ import { createCipheriv, randomBytes } from "node:crypto";
import type { Readable } from "node:stream";
import { describe, it, expect, beforeEach, vi } from "vitest";
import {
BACKUP_SCHEMA_VERSION,
backupPayloadSchema,
} from "@/lib/validations/backup";
import { backupPayloadSchema } from "@/lib/validations/backup";
import { BACKUP_SCHEMA_VERSION } from "@/lib/validations/backup-summary";
const mocks = vi.hoisted(() => ({
buildFullBackupPayload: vi.fn(),
@@ -7,7 +7,9 @@ import {
runRestoreDrill,
RESTORE_DRILL_CRON,
RESTORE_DRILL_QUEUE,
RESTORE_DRILL_SEND_OPTIONS,
} from "../restore-drill";
import { QUEUE_RUNTIME } from "../queue-runtime";
vi.mock("../report-worker-error", () => ({
reportWorkerError: vi.fn().mockResolvedValue(undefined),
@@ -86,6 +88,36 @@ describe("runRestoreDrill", () => {
expect(RESTORE_DRILL_CRON).toBe("11 4 1 * *");
});
it("runs as a locked long pass that pages each account once per run", () => {
// Every account's object is read, so the pg-boss default of fifteen
// minutes is no ceiling for it; no retry, so a failed account is paged
// once per run instead of three times.
expect(RESTORE_DRILL_SEND_OPTIONS.expireInSeconds).toBeGreaterThan(15 * 60);
expect(RESTORE_DRILL_SEND_OPTIONS.retryLimit).toBe(0);
expect(QUEUE_RUNTIME[RESTORE_DRILL_QUEUE]).toMatchObject({
runtime: "long",
exclusive: "lockedPass",
});
});
it("stops between accounts on its budget and counts the ones it did not reach", async () => {
const s3 = makeS3Mock({
"2026-06-01/user-aaa.json.enc": backupObject({ userId: "user-aaa" }),
"2026-06-01/user-mmm.json.enc": backupObject({ userId: "user-mmm" }),
"2026-06-01/user-zzz.json.enc": backupObject({ userId: "user-zzz" }),
});
let asked = 0;
const report = await runRestoreDrill(
s3,
new Date("2026-06-02T04:11:00Z"),
() => ++asked > 1,
);
expect(report.accounts.map((a) => a.objectKey)).toEqual([
"2026-06-01/user-aaa.json.enc",
]);
expect(report.unchecked).toBe(2);
});
it("fetches, decrypts, and parses the newest backup object", async () => {
const s3 = makeS3Mock({
"2026-05-30/user-old.json.enc": backupObject(),
+55 -1
View File
@@ -16,6 +16,16 @@
* takes that account last, after everybody else has their copy, and the admin
* backups page names it. Every write and read here is best effort: the marker
* is bookkeeping about a backup, and must never be the reason one fails.
*
* Last only once. A pass that regularly stops on its time budget never reaches
* the end of its list, so an account kept last for as long as its attempt
* stays open would never be attempted again, and would never get a copy. An
* interrupted account is therefore held back only until a later run has
* started on another account (`accountsToTakeLast`): from then on it takes its
* ordinary place again. If it kills the process again, its new attempt is the
* newest one and it goes last once more. So a crash loop costs the other
* accounts at most every second run, and the account itself is attempted at
* least every second run.
*/
import type { PrismaClient } from "@/generated/prisma/client";
@@ -83,7 +93,51 @@ export async function readInterruptedBackupAttempts(
}
/**
* `accounts` in their order, with every interrupted one moved behind the
* The interrupted accounts a pass should take last: those whose interrupted
* attempt is still the newest attempt of the pass, i.e. no later run has
* started on another account since. Rows are one per account. Pure, so the
* rule can be tested apart from the database.
*/
export function heldBackAccounts(
rows: ReadonlyArray<{
userId: string;
startedAt: Date;
finishedAt: Date | null;
}>,
): Map<string, Date> {
const out = new Map<string, Date>();
for (const row of rows) {
const open = row.finishedAt === null || row.finishedAt < row.startedAt;
if (!open) continue;
const passedOver = rows.some(
(other) => other.userId !== row.userId && other.startedAt > row.startedAt,
);
if (!passedOver) out.set(row.userId, row.startedAt);
}
return out;
}
/**
* What a pass orders by: the interrupted accounts it takes last
* (`heldBackAccounts`). Empty when the table cannot be read.
*/
export async function readAccountsToTakeLast(
prisma: Db,
pass: BackupPass,
): Promise<Map<string, Date>> {
try {
const rows = await prisma.backupPassAttempt.findMany({
where: { pass },
select: { userId: true, startedAt: true, finishedAt: true },
});
return heldBackAccounts(rows);
} catch {
return new Map();
}
}
/**
* `accounts` in their order, with every held-back one moved behind the
* rest; among those, the one interrupted longest ago first.
*/
export function orderInterruptedLast<T extends { id: string }>(
+1
View File
@@ -200,6 +200,7 @@ export const JOB_FACT_ALLOWLIST: ReadonlySet<string> = new Set([
"restore_drill_medications",
"restore_drill_mood_entries",
"restore_drill_plaintext_bytes",
"restore_drill_unchecked",
"restore_drill_stale",
"restore_fail_cause",
"restore_failed",
+5 -4
View File
@@ -39,7 +39,7 @@ import {
markBackupAttemptFinished,
markBackupAttemptStarted,
orderInterruptedLast,
readInterruptedBackupAttempts,
readAccountsToTakeLast,
} from "@/lib/jobs/backup-pass-attempts";
import { BACKUP_HEARTBEAT_SECONDS } from "@/lib/jobs/data-backup-policy";
import { annotate, getEvent } from "@/lib/logging/context";
@@ -930,14 +930,15 @@ export async function runOffhostBackup(
// By id, with an account whose last walk never came back behind the rest:
// a record that killed the process once would otherwise lead every retry
// and keep every account after it out of the bucket
// (`backup-pass-attempts.ts`).
// and keep every account after it out of the bucket. Behind them for one
// run only, so a pass that stops on its budget still reaches it the run
// after (`backup-pass-attempts.ts`).
const users = orderInterruptedLast(
await prisma.user.findMany({
select: { id: true },
orderBy: { id: "asc" },
}),
await readInterruptedBackupAttempts(prisma, OFFHOST_BACKUP_QUEUE),
await readAccountsToTakeLast(prisma, OFFHOST_BACKUP_QUEUE),
);
const doneThisRun = new Set<string>();
if (runStartedAt) {
+9 -1
View File
@@ -36,6 +36,7 @@ import { DOCUMENT_AI_RUN_EXPIRE_SECONDS } from "@/lib/jobs/document-ai-run";
import { DATA_BACKUP_SEND_OPTIONS } from "@/lib/jobs/data-backup-policy";
import { OFFHOST_BACKUP_EXPIRE_SECONDS } from "@/lib/jobs/offhost-backup";
import { BACKUP_RESTORE_EXPIRE_SECONDS } from "@/lib/jobs/backup-restore";
import { RESTORE_DRILL_EXPIRE_SECONDS } from "@/lib/jobs/restore-drill";
import { APPLE_HEALTH_IMPORT_SEND_OPTIONS } from "@/lib/jobs/apple-health-import-worker";
import { MEDICATION_INTAKE_IMPORT_SEND_OPTIONS } from "@/lib/jobs/medication-intake-import";
@@ -267,7 +268,14 @@ export const QUEUE_RUNTIME: Readonly<Record<string, QueueRuntime>> = {
exclusive: "lockedPass",
why: "Uploads every account's record off-host.",
},
"data-restore-drill": short("Fetches and stream-parses one off-host object."),
"data-restore-drill": {
runtime: "long",
expireInSeconds: RESTORE_DRILL_EXPIRE_SECONDS,
expiryVia: ["RESTORE_DRILL_SEND_OPTIONS"],
stop: { file: "lib/jobs/restore-drill.ts", fn: "handleRestoreDrill" },
exclusive: "lockedPass",
why: "Fetches and stream-parses every account's newest off-host object.",
},
"offhost-backup-purge": short("At most 200 delete requests per run."),
"backup-restore": {
runtime: "long",
+6 -4
View File
@@ -27,7 +27,7 @@ import {
markBackupAttemptFinished,
markBackupAttemptStarted,
orderInterruptedLast,
readInterruptedBackupAttempts,
readAccountsToTakeLast,
} from "@/lib/jobs/backup-pass-attempts";
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
@@ -154,7 +154,7 @@ export async function handleDataBackup(
where: { type: "WEEKLY_AUTO" },
select: { userId: true, createdAt: true },
}),
readInterruptedBackupAttempts(prisma, DATA_BACKUP_QUEUE),
readAccountsToTakeLast(prisma, DATA_BACKUP_QUEUE),
]);
const copiedAt = new Map(
copies.map((copy) => [copy.userId, copy.createdAt.getTime()]),
@@ -163,7 +163,9 @@ export async function handleDataBackup(
// attempt never came back goes behind all of them: it is the one with
// the oldest copy, so it would lead every retry, and a record that
// killed the process once would keep every account after it from a
// copy (`backup-pass-attempts.ts`).
// copy. Only until a later run has started on another account, so a
// pass that stops on its budget still reaches it the run after
// (`backup-pass-attempts.ts`).
const users = orderInterruptedLast(
[...accounts].sort(
(a, b) => (copiedAt.get(a.id) ?? -1) - (copiedAt.get(b.id) ?? -1),
@@ -205,7 +207,7 @@ export async function handleDataBackup(
// so its size does not depend on this process's memory (#1031).
// The restore preview is taken from the writer's own counts and
// the sections as they are written, never by parsing the copy.
const preview = createBackupPreviewCollector();
const preview = await createBackupPreviewCollector();
let counts: FullBackupCounts | undefined;
const { bytes: storedBytes } = await storeBackupBlob(
prisma,
@@ -83,6 +83,7 @@ import {
handleRestoreDrill,
RESTORE_DRILL_CRON,
RESTORE_DRILL_QUEUE,
RESTORE_DRILL_SEND_OPTIONS,
} from "@/lib/jobs/restore-drill";
import {
BACKUP_RESTORE_QUEUE,
@@ -551,8 +552,10 @@ const allQueues = [
// a bulk DELETE or UPDATE that timed out will time out again on an immediate
// retry, so pg-boss's default two would run the same doomed statement three
// times a night. The queues without it are the ones whose failure is
// transient — the TLS probe, the off-host upload, the restore drill, the
// per-user environment fetch — and those still want the default retries.
// transient — the TLS probe, the off-host upload, the per-user environment
// fetch — and those still want the default retries. The restore drill sends
// `retryLimit: 0` in its own options for the other reason: it pages each
// failed account itself, and a retry would page the same accounts again.
const schedules: ScheduleEntry[] = [
// v1.25 (W-ENV) — daily 02:10 Europe/Berlin discovery tick (empty payload)
// that fans out one per-user environment fetch per opted-in account.
@@ -571,7 +574,7 @@ const schedules: ScheduleEntry[] = [
cronIsTheRetry,
],
[OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON, OFFHOST_BACKUP_SEND_OPTIONS],
[RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON],
[RESTORE_DRILL_QUEUE, RESTORE_DRILL_CRON, RESTORE_DRILL_SEND_OPTIONS],
// Nightly backstop for the purge the deletion kicks straight away: a
// request the bucket refused, or one written while the queue was down.
[OFFHOST_PURGE_QUEUE, OFFHOST_PURGE_CRON, cronIsTheRetry],
@@ -832,11 +835,13 @@ export async function registerMaintenanceQueues(
{ localConcurrency: 1, includeMetadata: true },
lockedPass(OFFHOST_BACKUP_QUEUE, () => WHOLE_PASS, handleOffhostBackup),
);
// Locked: a drill that outlives a restart or a manual run never reads the
// bucket beside another one.
await createAndWork(
boss,
RESTORE_DRILL_QUEUE,
{ localConcurrency: 1 },
handleRestoreDrill,
lockedPass(RESTORE_DRILL_QUEUE, () => WHOLE_PASS, handleRestoreDrill),
);
await createAndWork(
boss,
+43 -4
View File
@@ -21,6 +21,13 @@
* Schedule: 04:11 on the 1st of each month (`11 4 1 * *`), after the
* 02:30 nightly upload and the 03:xx cleanup window, on a minute slot
* no other cron uses.
*
* The drill reads every account's object, so it runs as a long pass: an
* explicit expiry (`RESTORE_DRILL_EXPIRE_SECONDS`), a stop between accounts
* on the job's budget, a lock so two drills never overlap (`lockedPass` at
* the binding), and no retry. A retry would re-read the same objects and
* page the same failed accounts a second and third time; each account is
* reported once per run, and the next drill is the retry.
*/
import type { Job } from "pg-boss";
import { BackupJsonError, scanBackupJson } from "@/lib/export/backup-json-scan";
@@ -38,12 +45,28 @@ import {
type S3Like,
} from "@/lib/jobs/offhost-backup";
import { reportWorkerError } from "@/lib/jobs/report-worker-error";
import { jobBudget } from "@/lib/jobs/job-budget";
import { jobDone, jobFailed, type JobOutcome } from "@/lib/jobs/job-outcome";
import { withBackgroundEvent } from "@/lib/logging/background";
export const RESTORE_DRILL_QUEUE = "data-restore-drill";
export const RESTORE_DRILL_CRON = "11 4 1 * *";
/**
* Four hours, the off-host upload's own window: the drill downloads and
* stream-parses every object that upload wrote in one night.
*/
export const RESTORE_DRILL_EXPIRE_SECONDS = 4 * 60 * 60;
/**
* What the monthly schedule sends with: the expiry above and no retry
* (`retryLimit: 0`, the maintenance schedules' `cronIsTheRetry`).
*/
export const RESTORE_DRILL_SEND_OPTIONS = {
expireInSeconds: RESTORE_DRILL_EXPIRE_SECONDS,
retryLimit: 0,
} as const;
/**
* A drill run that finds the newest backup older than this is reported
* as a failure even when decrypt + parse succeed: the nightly uploader
@@ -80,6 +103,8 @@ export interface RestoreDrillReport {
accounts: RestoreDrillAccount[];
/** The accounts whose object did not pass. */
failed: RestoreDrillAccount[];
/** Objects of the newest date the run stopped before, on its budget. */
unchecked: number;
}
function countArray(value: unknown): number {
@@ -178,6 +203,7 @@ async function drillObject(
export async function runRestoreDrill(
s3Override?: S3Like,
now: Date = new Date(),
shouldStop: () => boolean = () => false,
): Promise<RestoreDrillReport> {
const cfg = loadOffhostConfig();
if (!cfg) {
@@ -208,6 +234,8 @@ export async function runRestoreDrill(
const accounts: RestoreDrillAccount[] = [];
for (const objectKey of newest) {
// Between accounts, never inside one: the job's budget is spent.
if (shouldStop()) break;
const account: RestoreDrillAccount = {
objectKey,
ok: true,
@@ -239,6 +267,7 @@ export async function runRestoreDrill(
stale: ageDays > MAX_BACKUP_AGE_DAYS,
accounts,
failed: accounts.filter((a) => !a.ok),
unchecked: newest.length - accounts.length,
};
}
@@ -252,10 +281,10 @@ export async function runRestoreDrill(
export async function handleRestoreDrill(
jobs: Job<object>[],
): Promise<JobOutcome> {
void jobs;
const shouldStop = jobBudget(jobs);
return withBackgroundEvent("job.restore_drill", async (evt) => {
try {
const report = await runRestoreDrill();
const report = await runRestoreDrill(undefined, new Date(), shouldStop);
const total = (pick: (a: RestoreDrillAccount) => number) =>
report.accounts.reduce((sum, a) => sum + pick(a), 0);
const meta = {
@@ -270,6 +299,7 @@ export async function handleRestoreDrill(
restore_drill_intake_events: total((a) => a.recordCounts.intakeEvents),
restore_drill_mood_entries: total((a) => a.recordCounts.moodEntries),
restore_drill_stale: report.stale,
restore_drill_unchecked: report.unchecked,
};
for (const [key, value] of Object.entries(meta)) evt.addMeta(key, value);
evt.addMeta(
@@ -301,9 +331,18 @@ export async function handleRestoreDrill(
{ dateKey: report.dateKey, ageDays: report.ageDays },
);
}
if (report.unchecked > 0) {
await reportWorkerError(
RESTORE_DRILL_QUEUE,
new Error(
`Restore drill stopped on its time budget with ${report.unchecked} of ${report.accounts.length + report.unchecked} backup objects unchecked.`,
),
{ dateKey: report.dateKey, unchecked: report.unchecked },
);
}
if (report.failed.length > 0) {
// Already paged above, per account. A retry reads the same objects
// and cannot change the verdict before the next upload.
// Already paged above, per account. The queue does not retry: a
// retry reads the same objects and would page the same accounts again.
return jobFailed(
`restore drill: ${report.failed.length} of ${report.accounts.length} accounts failed`,
new Error(report.failed.map((a) => a.objectKey).join(", ")),
+1
View File
@@ -344,6 +344,7 @@ describe("get_metric_series", () => {
expect(executeCoachTool).toHaveBeenCalledWith({
userId: "user-1",
sourceSnapshot: true,
name: "get_metric_series",
rawArguments: JSON.stringify({ metric: "bp", window: "last90days" }),
});
+1
View File
@@ -157,6 +157,7 @@ async function read(
): Promise<unknown> {
return executeCoachTool({
userId: ctx.userId,
sourceSnapshot: true,
name,
rawArguments: JSON.stringify(args),
});
+2
View File
@@ -460,6 +460,7 @@ export const MCP_RESOURCE_TEMPLATES: McpResourceTemplateDefinition[] = [
const metric = firstVar(variables.type);
const result = await executeCoachTool({
userId: ctx.userId,
sourceSnapshot: true,
name: "get_metric_series",
rawArguments: JSON.stringify({ metric }),
});
@@ -483,6 +484,7 @@ export const MCP_RESOURCE_TEMPLATES: McpResourceTemplateDefinition[] = [
const window = firstVar(variables.window);
const result = await executeCoachTool({
userId: ctx.userId,
sourceSnapshot: true,
name: "get_metric_series",
rawArguments: JSON.stringify(window ? { metric, window } : { metric }),
});
+5
View File
@@ -144,6 +144,9 @@ async function runCoachTool(
): Promise<unknown> {
const result = await executeCoachTool({
userId: ctx.userId,
// The single-metric read keeps one section, so it is built from that
// source's rows (see `source-snapshot.ts`).
...(name === "get_metric_series" ? { sourceSnapshot: true } : {}),
name,
rawArguments: JSON.stringify(args ?? {}),
});
@@ -659,6 +662,7 @@ function searchAndFetchTools(): McpToolDefinition[] {
}
const result = await executeCoachTool({
userId: ctx.userId,
sourceSnapshot: true,
name: "get_metric_series",
rawArguments: JSON.stringify({ metric: rid }),
});
@@ -1571,6 +1575,7 @@ export const MCP_TOOLS: McpToolDefinition[] = [
page.map(async (metric) => {
const r = (await executeCoachTool({
userId: ctx.userId,
sourceSnapshot: true,
name: "get_metric_series",
rawArguments: JSON.stringify({
metric,
+22 -24
View File
@@ -245,34 +245,32 @@ function weightedRowsSql(
if (!/^[a-z_][a-z0-9_]*$/.test(source)) {
throw new Error(`source must be a CTE name: ${source}`);
}
const hourOf = (alias: string) =>
`date_trunc('hour', ${frameTs(frame, `${alias}."measured_at"`)})`;
const dayOf = (alias: string) =>
`date_trunc('day', ${frameTs(frame, `${alias}."measured_at"`)})`;
const src = bySource ? `, t."source"` : "";
const part = bySource ? `, x."source"` : "";
const join = bySource ? ` AND hw."source" = s."source"` : "";
// One scan of the CTE and window functions only. The earlier shape joined
// the CTE to an hour grouping of itself; with stale or missing statistics
// (a fresh import below the autoanalyze threshold) the planner ran that
// join as a nested loop and a year of pulse ran into the statement timeout.
//
// `dm_hour_n` counts the readings in a row's hour and `dm_hour_first` marks
// one row per hour, so the sum of `dm_hour_first` over a day is the day's
// hour count. Both columns ride along on the row; every caller aggregates.
const partition = (alias: string, unit: "hour" | "day") => {
const keys = bySource
? `${alias}."type", ${alias}."source"`
: `${alias}."type"`;
const ts = frameTs(frame, `${alias}."measured_at"`);
return `PARTITION BY ${keys}, date_trunc('${unit}', ${ts})`;
};
return `(
SELECT s.*,
(CASE WHEN ${isHourlyMeanTypeSql('s."type"')}
THEN 1.0 / (hw.n * hw.hours)
THEN 1.0 / (s.dm_hour_n * SUM(s.dm_hour_first) OVER (${partition("s", "day")}))
ELSE 1.0 END)::double precision AS day_weight
FROM ${source} s
LEFT JOIN (
SELECT x."type"${part}, x.local_hour, x.n,
COUNT(*) OVER (PARTITION BY x."type"${part}, x.local_day) AS hours
FROM (
SELECT t."type"${src},
${hourOf("t")} AS local_hour,
${dayOf("t")} AS local_day,
COUNT(*) AS n
FROM ${source} t
WHERE ${isHourlyMeanTypeSql('t."type"')}
GROUP BY ${bySource ? "1, 2, 3, 4" : "1, 2, 3"}
) x
) hw
ON hw."type" = s."type"${join}
AND hw.local_hour = ${hourOf("s")}
FROM (
SELECT t.*,
COUNT(*) OVER (${partition("t", "hour")}) AS dm_hour_n,
(ROW_NUMBER() OVER (${partition("t", "hour")}) = 1)::int AS dm_hour_first
FROM ${source} t
) s
)`;
}
@@ -122,11 +122,14 @@ export interface ImportJobResult {
clinical: { skipped: number };
/**
* Records left out because HealthLog wrote them into Apple Health itself and
* already holds them under their own source, each counted once however often
* the export repeats it: `byMarker` carry the app's origin marker,
* `byExternalId` carry a HealthLog row id as `HKExternalUUID` (samples from
* before the marker existed), `matchedManual` match a manual entry (same type
* and value, within 2 s) without either.
* this account already holds them under their own source, each counted once
* however often the export repeats it: `byMarker` carry the app's origin
* marker and a HealthLog row id of this account (a cycle sample: a day-log of
* this account on its day), `byExternalId` carry such a row id without the
* marker (samples from before the marker existed), `matchedManual` match a
* manual entry (same type and value, within 2 s). A marked sample this
* account does not hold (a move to a new instance without a backup) is
* imported like any other.
*/
writtenByHealthLog: {
byMarker: number;
@@ -186,8 +189,11 @@ interface PreparedMeasurement {
* at the close tag, once its `<MetadataEntry>` children have been read.
*/
interface PendingRecord {
/** Queue the record; `healthLogId` is the `HKExternalUUID` it carried. */
commit: (healthLogId: string | null) => void;
/**
* Queue the record; `healthLogId` is the `HKExternalUUID` it carried and
* `marked` whether it (or its correlation) carried the origin marker.
*/
commit: (healthLogId: string | null, marked: boolean) => void;
ownOrigin: boolean;
healthLogId: string | null;
/** The record's sample identity, computed only when it is left out. */
@@ -369,7 +375,11 @@ export interface StreamParseInput {
/** Prisma client to flush rows through. */
prisma: Pick<
PrismaClient,
"measurement" | "workout" | "$transaction" | "$queryRawUnsafe"
| "measurement"
| "workout"
| "cycleDayLog"
| "$transaction"
| "$queryRawUnsafe"
>;
/**
* Live progress hook. Called every `PROGRESS_TICK_RECORDS` records
@@ -449,6 +459,15 @@ export async function streamParseExportXml(
sampleKey: string;
ownOrigin: boolean;
} | null = null;
// Marked cycle samples, held to the end of the parse: one counts as written
// by HealthLog only when this account holds a day-log on its day.
const markedCycle: Array<{
hkType: string;
dayKey: string;
rawValue: string | undefined;
protectionUsed?: boolean;
sampleKey: string;
}> = [];
// Cumulative fold: type -> local day -> hashed source identity -> subtotal.
const cumulativeBucket = new Map<
@@ -471,13 +490,25 @@ export async function streamParseExportXml(
countedOwn.add(sampleKey);
writtenByHealthLog[reason] += 1;
};
// The `HKExternalUUID` a queued spot row carried, checked at flush.
// The `HKExternalUUID` a queued spot row carried, checked at flush, and the
// queued rows that carried the origin marker.
const healthLogIdOf = new Map<PreparedMeasurement, string>();
const markedRows = new Set<PreparedMeasurement>();
// Marked cumulative samples that named a HealthLog row id, held to the end
// of the parse and added to their day only when the id is not this
// account's. Small: one per mirrored total.
const markedCumulative: Array<{
type: MeasurementType;
healthLogId: string;
sampleKey: string;
add: () => void;
}> = [];
// A `<Correlation>` (a blood pressure reading) wraps its records, and the
// marker may sit on the correlation itself, before or after them. Its records
// are held until it closes, then all kept or all left out together.
let currentCorrelation: {
ownOrigin: boolean;
healthLogId: string | null;
held: PendingRecord[];
} | null = null;
// Spot-row batch awaiting flush.
@@ -536,12 +567,29 @@ export async function streamParseExportXml(
};
// ── Flush helpers ──────────────────────────────────────────
// A sample the app wrote before it stamped its origin marker still carries
// the HealthLog row id as `HKExternalUUID`. A hit on one of this account's
// rows (deleted ones included: the sample is ours either way) of the same
// kind is an exact match. Blood pressure is one row per half, while the app
// stamps both samples of a reading with one id (the systolic row's, the
// diastolic row's in older builds), so either half matches either id.
// A sample the app wrote carries the HealthLog row id as `HKExternalUUID`,
// with the origin marker or (before the marker existed) without it. A hit on
// one of this account's rows (deleted ones included: the sample is ours
// either way) of the same kind is an exact match. The marker alone proves
// nothing about THIS account: after a move to a new instance without a
// backup, re-importing the export must bring the mirrored values back.
// Blood pressure is one row per half, while the app stamps both samples of a
// reading with one id (the systolic row's, the diastolic row's in older
// builds), so either half matches either id.
const ownRowKinds = async (
ids: ReadonlySet<string>,
): Promise<Map<string, string>> => {
const out = new Map<string, string>();
const list = [...ids];
for (let i = 0; i < list.length; i += 1000) {
const known = await prisma.measurement.findMany({
where: { userId, id: { in: list.slice(i, i + 1000) } },
select: { id: true, type: true },
});
for (const row of known) out.set(row.id, kindOf(row.type));
}
return out;
};
const withoutHealthLogIds = async (
rows: PreparedMeasurement[],
): Promise<PreparedMeasurement[]> => {
@@ -551,17 +599,16 @@ export async function streamParseExportXml(
if (id) ids.add(id);
}
if (ids.size === 0) return rows;
const known = await prisma.measurement.findMany({
where: { userId, id: { in: [...ids] } },
select: { id: true, type: true },
});
if (known.length === 0) return rows;
const typeOf = new Map(known.map((row) => [row.id, row.type]));
const kindOfId = await ownRowKinds(ids);
if (kindOfId.size === 0) return rows;
return rows.filter((row) => {
const id = healthLogIdOf.get(row);
const rowType = id ? typeOf.get(id) : undefined;
if (!rowType || kindOf(rowType) !== kindOf(row.type)) return true;
countOwn("byExternalId", row.externalId);
const rowKind = id ? kindOfId.get(id) : undefined;
if (!rowKind || rowKind !== kindOf(row.type)) return true;
countOwn(
markedRows.has(row) ? "byMarker" : "byExternalId",
row.externalId,
);
return false;
});
};
@@ -663,7 +710,10 @@ export async function streamParseExportXml(
const chunk = await withoutManualMirrors(
await withoutHealthLogIds(incoming),
);
for (const row of incoming) healthLogIdOf.delete(row);
for (const row of incoming) {
healthLogIdOf.delete(row);
markedRows.delete(row);
}
for (const row of chunk) widenSpan(row.measuredAt);
const insertedArrivals: Array<{
id: string;
@@ -993,6 +1043,61 @@ export async function streamParseExportXml(
}
};
const consumeCycle = (rec: {
hkType: string;
dayKey: string;
rawValue: string | undefined;
protectionUsed?: boolean;
}): void => {
const consumed = cycleAccumulator.consume(
rec.hkType,
rec.dayKey,
rec.rawValue,
rec.protectionUsed,
);
if (!consumed) {
// Recognised identifier but unrecognised value — count it under
// `unknown` with the reason tag so operators can spot it.
unknown[`${rec.hkType}::cycle_unmapped`] =
(unknown[`${rec.hkType}::cycle_unmapped`] ?? 0) + 1;
}
};
// Marked cumulative samples: left out when their id is a row of this
// account of the same kind, added to their day otherwise.
const settleMarkedCumulative = async (): Promise<void> => {
if (markedCumulative.length === 0) return;
const kindOfId = await ownRowKinds(
new Set(markedCumulative.map((m) => m.healthLogId)),
);
for (const m of markedCumulative.splice(0, markedCumulative.length)) {
if (kindOfId.get(m.healthLogId) === kindOf(m.type)) {
countOwn("byMarker", m.sampleKey);
} else {
m.add();
}
}
};
// Marked cycle samples: left out when this account holds a day-log on
// their day (deleted ones included), folded in otherwise.
const settleMarkedCycle = async (): Promise<void> => {
if (markedCycle.length === 0) return;
const days = [...new Set(markedCycle.map((rec) => rec.dayKey))];
const held = new Set<string>();
for (let i = 0; i < days.length; i += 1000) {
const logs = await prisma.cycleDayLog.findMany({
where: { userId, date: { in: days.slice(i, i + 1000) } },
select: { date: true },
});
for (const log of logs) held.add(log.date);
}
for (const rec of markedCycle.splice(0, markedCycle.length)) {
if (held.has(rec.dayKey)) countOwn("byMarker", rec.sampleKey);
else consumeCycle(rec);
}
};
// ── SAX parser configuration ────────────────────────────────
const parser = sax.parser(true, { trim: true });
let pendingError: Error | null = null;
@@ -1121,27 +1226,44 @@ export async function streamParseExportXml(
attrs.startDate,
attrs.endDate,
),
commit: (healthLogId) => {
commit: (healthLogId, marked) => {
if (CUMULATIVE_HK_TYPES.has(mapped.type)) {
const dayKey = dayKeyForUserTz(mapped.takenAt, userTimezone);
const sourceHash = hashCumulativeSourceIdentity(
attrs.sourceName,
attrs.device,
);
let byDay = cumulativeBucket.get(mapped.type);
if (!byDay) {
byDay = new Map();
cumulativeBucket.set(mapped.type, byDay);
const add = () => {
let byDay = cumulativeBucket.get(mapped.type);
if (!byDay) {
byDay = new Map();
cumulativeBucket.set(mapped.type, byDay);
}
let bySource = byDay.get(dayKey);
if (!bySource) {
bySource = new Map();
byDay.set(dayKey, bySource);
}
bySource.set(
sourceHash,
(bySource.get(sourceHash) ?? 0) + mapped.value,
);
};
if (marked && healthLogId) {
markedCumulative.push({
type: mapped.type,
healthLogId,
sampleKey: hashSampleKey(
hkType,
attrs.value ?? "",
attrs.startDate,
attrs.endDate,
),
add,
});
} else {
add();
}
let bySource = byDay.get(dayKey);
if (!bySource) {
bySource = new Map();
byDay.set(dayKey, bySource);
}
bySource.set(
sourceHash,
(bySource.get(sourceHash) ?? 0) + mapped.value,
);
} else {
// Spot row: derive a stable externalId, queue for flush.
const externalId = hashSampleKey(
@@ -1163,6 +1285,7 @@ export async function streamParseExportXml(
};
spotBatch.push(row);
if (healthLogId) healthLogIdOf.set(row, healthLogId);
if (marked) markedRows.add(row);
}
},
};
@@ -1244,7 +1367,7 @@ export async function streamParseExportXml(
}
if (name === "Correlation") {
currentCorrelation = { ownOrigin: false, held: [] };
currentCorrelation = { ownOrigin: false, healthLogId: null, held: [] };
return;
}
@@ -1255,8 +1378,12 @@ export async function streamParseExportXml(
else if (currentCycleRecord) currentCycleRecord.ownOrigin = true;
else if (currentCorrelation) currentCorrelation.ownOrigin = true;
}
if (pendingRecord && attrs.key === HK_EXTERNAL_UUID_METADATA_KEY) {
pendingRecord.healthLogId = attrs.value || null;
if (attrs.key === HK_EXTERNAL_UUID_METADATA_KEY) {
// On a record, or on the correlation that wraps it.
if (pendingRecord) pendingRecord.healthLogId = attrs.value || null;
else if (currentCorrelation && !currentCycleRecord) {
currentCorrelation.healthLogId = attrs.value || null;
}
}
// Attach the SexualActivity protection flag to the open cycle record.
// Apple writes `HKMetadataKeySexualActivityProtectionUsed` with a
@@ -1306,18 +1433,16 @@ export async function streamParseExportXml(
const rec = pendingRecord;
pendingRecord = null;
if (currentCorrelation) currentCorrelation.held.push(rec);
else if (rec.ownOrigin) countOwn("byMarker", rec.sampleKey());
else rec.commit(rec.healthLogId);
else rec.commit(rec.healthLogId, rec.ownOrigin);
}
if (tagName === "Correlation" && currentCorrelation) {
const correlation = currentCorrelation;
currentCorrelation = null;
for (const rec of correlation.held) {
if (correlation.ownOrigin || rec.ownOrigin) {
countOwn("byMarker", rec.sampleKey());
} else {
rec.commit(rec.healthLogId);
}
rec.commit(
rec.healthLogId ?? correlation.healthLogId,
correlation.ownOrigin || rec.ownOrigin,
);
}
}
if (tagName === "Workout" && currentWorkout) {
@@ -1328,22 +1453,12 @@ export async function streamParseExportXml(
const rec = currentCycleRecord;
currentCycleRecord = null;
if (rec.ownOrigin) {
// A day HealthLog mirrored into Apple Health from its own day-log.
countOwn("byMarker", rec.sampleKey);
// A day HealthLog mirrored into Apple Health from a day-log; decided
// at the end of the parse against this account's day-logs.
markedCycle.push(rec);
return;
}
const consumed = cycleAccumulator.consume(
rec.hkType,
rec.dayKey,
rec.rawValue,
rec.protectionUsed,
);
if (!consumed) {
// Recognised identifier but unrecognised value — count it under
// `unknown` with the reason tag so operators can spot it.
unknown[`${rec.hkType}::cycle_unmapped`] =
(unknown[`${rec.hkType}::cycle_unmapped`] ?? 0) + 1;
}
consumeCycle(rec);
}
};
@@ -1421,7 +1536,9 @@ export async function streamParseExportXml(
// threshold during the parse.
await flushSpotBatch();
await flushWorkoutBatch();
await settleMarkedCumulative();
await flushCumulativeBuckets();
await settleMarkedCycle();
// v1.15.0 — fold the accumulated reproductive samples into CYCLE
// day-logs. Gated on cycle-tracking being enabled for the account so a
@@ -125,6 +125,28 @@ describe("refetchMedicationsOnMount", () => {
unsubscribe();
});
it("refetches a prefetched read that was invalidated before the mount", async () => {
// Hover the nav link, then log a dose elsewhere before the page mounts:
// the dose invalidates the medication reads, and the prefetched answer
// no longer holds the compliance the user just changed.
const { client, calls } = setup();
prefetchMedicationsList(client);
await vi.waitFor(() => expect(calls("/api/medications")).toBe(1));
await vi.waitFor(() =>
expect(client.getQueryState(queryKeys.medications())?.status).toBe(
"success",
),
);
await client.invalidateQueries({ queryKey: queryKeys.medications() });
expect(client.getQueryState(queryKeys.medications())?.isInvalidated).toBe(
true,
);
const { refetched, unsubscribe } = await mountList(client);
expect(refetched).toBe(true);
unsubscribe();
});
it("still refetches on mount when the cache holds an older read", async () => {
const { client } = setup();
const queryFn = vi.fn(async () => []);
+8 -1
View File
@@ -93,11 +93,18 @@ export function prefetchMedicationsList(
* the identical request a second time, a few hundred milliseconds after the
* first answer landed, on every visit to the page. A return to the page with
* an older cache entry, or a direct load, still refetches on mount.
*
* An invalidated entry always refetches: a dose logged elsewhere in the app
* between the prefetch and the mount invalidates these reads, and the
* prefetched answer is then the stale one.
*/
export function refetchMedicationsOnMount(
read: MedicationsRead,
): (query: { state: { dataUpdatedAt: number } }) => boolean | "always" {
): (query: {
state: { dataUpdatedAt: number; isInvalidated: boolean };
}) => boolean | "always" {
return (query) => {
if (query.state.isInvalidated) return "always";
const startedAt = prefetchStartedAt.get(read);
const answeredThisVisit =
startedAt !== undefined &&
+1 -1
View File
@@ -597,7 +597,7 @@ async function runRollupAggregate(
* every reading, so the band and the windowed slope are untouched.
*
* Hours and days are UTC, matching the UTC day this tier buckets by; migration
* 0373 applies the same statistic to rows stored before it.
* 0372 applies the same statistic to rows stored before it.
*/
async function applyHourlyMeanDay(
db: Pick<typeof prisma, "$queryRawUnsafe">,
+2 -6
View File
@@ -18,12 +18,8 @@
* of bubbling out of `prisma.createMany` with a useless message.
*/
import { describe, expect, it } from "vitest";
import {
backupPayloadSchema,
isCompatibleSchemaVersion,
parseBackupPayload,
summarizeBackup,
} from "../backup";
import { backupPayloadSchema, parseBackupPayload } from "../backup";
import { isCompatibleSchemaVersion, summarizeBackup } from "../backup-summary";
const baseEntry = {
date: "2026-05-08",
+216
View File
@@ -0,0 +1,216 @@
/**
* What a backup is, without the schema that checks one.
*
* `backupPayloadSchema` (`backup.ts`) is the largest Zod tree in the server:
* built at import, it holds about 12 MB of heap. Everything here is what the
* backup paths need besides that check: the schema version, whether a file's
* version is one this server reads, and the per-section counts of a payload
* that has already been parsed. Importing this module never builds the
* schema, so a module on the boot path (the restore job, the stored-backup
* readers, the export writer) can import it freely and load `backup.ts` with
* a dynamic `import()` at the moment a file is actually validated.
*/
import type { BackupPayload } from "@/lib/validations/backup";
export const BACKUP_SCHEMA_VERSION = "2" as const;
export const LEGACY_BACKUP_SCHEMA_VERSION = "1" as const;
/**
* Numeric counts of each backed-up record kind. Returned in the
* upload + restore API responses so the admin sees what they
* uploaded/restored without having to download the file again.
*/
export interface BackupSummary {
schemaVersion: string;
userId: string;
exportedAt: string;
measurements: number;
medications: number;
intakeEvents: number;
/** Side effects recorded against a drug, across every medication. */
medicationSideEffects: number;
/** Pinned efficacy targets, across every medication. */
medicationEfficacyTargets: number;
/** Archived schedule eras, across every medication. */
medicationScheduleRevisions: number;
moodEntries: number;
/** v1.15.0 — observed cycle spans in the backup. */
cycles: number;
/** v1.15.0 — cycle day-logs in the backup. */
cycleDayLogs: number;
/** Lab results in the backup. */
labResults: number;
nutrientDays: number;
/** User-scoped biomarker catalog entries in the backup. */
biomarkers: number;
/** Illness episodes, including flares/exacerbations. */
illnessEpisodes: number;
/** Illness day-logs across every episode. */
illnessDayLogs: number;
/** Allergy/intolerance records in the backup. */
allergies: number;
/** Family-history entries in the backup. */
familyHistory: number;
/** Workout summary records in the backup. */
workouts: number;
/** Document records (ciphertext included in canonical DR payloads). */
documents: number;
/** Which documents were filed against which condition. */
documentConditionLinks: number;
/** Facts staged out of a document, with their review decision. */
extractedFacts: number;
/** 1 when the account's durable self-context rides the file, 0 otherwise. */
healthProfile: number;
/** Effective-dated structured health-profile revisions. */
healthProfileFactRevisions: number;
/** Metrics the account defined itself. */
customMetrics: number;
/** Readings across every user-defined metric. */
customMetricEntries: number;
/** Persisted accepted correlation identities and dismissal decisions. */
correlationPatterns: number;
/** Stored day curves for the cumulative metrics, across every metric. */
intradayProfiles: number;
/** Local days whose health score was written down as it was shown. */
healthScoreRecords: number;
/** 1 when the record's setup answers ride the file, 0 otherwise. */
onboardingRecords: number;
/** v1.37.19 (A6-8) — the visit address book. */
practitioners: number;
/** v1.37.19 (A6-8) — doctor visits, planned and past. */
encounters: number;
/** v1.37.19 (A6-8) — the three encounter link tables, summed. */
encounterLinks: number;
/** v1.37.19 (A6-8) — immunization log entries. */
vaccinations: number;
/** v1.37.19 (A6-8) — vaccination↔document links. */
vaccinationLinks: number;
/** v1.37.20 (#223 / iOS #68) — Vorsorge reminder cadences. */
measurementReminders: number;
/** v1.37.20 (#223 / iOS #68) — completion-ledger rows across every reminder. */
measurementReminderEvents: number;
/** Coach conversation threads. */
coachConversations: number;
/** Coach turns across every thread. */
coachMessages: number;
/** Durable facts, agreed plans and things to bring back up. */
coachFacts: number;
coachPlans: number;
coachReminders: number;
/** Completed screener administrations. Disaster-recovery payloads only. */
mentalHealthAssessments: number;
/** Consent records. Disaster-recovery payloads only. */
consentReceipts: number;
/** Personal bests across every metric and sport slot. */
personalRecords: number;
/** Badges the account has unlocked, each with the day it was earned. */
userAchievements: number;
/** Local days with an environmental reading. */
environmentContexts: number;
/** Declared stretches spent away from home. */
environmentTravelLocations: number;
/** ECG strips, with their rhythm verdict and their trace. */
ecgRecordings: number;
}
export function summarizeBackup(payload: BackupPayload): BackupSummary {
return {
schemaVersion: payload.schemaVersion,
userId: payload.userId,
exportedAt: payload.exportedAt,
measurements: payload.measurements.length,
medications: payload.medications.length,
intakeEvents: payload.intakeEvents.length,
medicationSideEffects: payload.medications.reduce(
(sum, medication) => sum + medication.sideEffects.length,
0,
),
medicationEfficacyTargets: payload.medications.reduce(
(sum, medication) => sum + medication.efficacyTargets.length,
0,
),
medicationScheduleRevisions: payload.medications.reduce(
(sum, medication) => sum + medication.scheduleRevisions.length,
0,
),
moodEntries: payload.moodEntries.length,
cycles: payload.cycles.length,
cycleDayLogs: payload.cycleDayLogs.length,
nutrientDays: payload.nutrientDays.length,
labResults: payload.labResults.length,
biomarkers: payload.biomarkers.length,
illnessEpisodes: payload.illnessEpisodes.length,
illnessDayLogs: payload.illnessEpisodes.reduce(
(sum, e) => sum + e.dayLogs.length,
0,
),
allergies: payload.allergies.length,
familyHistory: payload.familyHistory.length,
workouts: payload.workouts.length,
documents: payload.documents.length,
// Counted from the release that carries them, so the admin's "what did I
// just restore" answer never under-counts a filed vault the way it once
// did for visits.
documentConditionLinks: payload.documentConditionLinks.length,
extractedFacts: payload.extractedFacts.length,
healthProfile: payload.healthProfile ? 1 : 0,
healthProfileFactRevisions: payload.healthProfileFacts.length,
customMetrics: payload.customMetrics.length,
customMetricEntries: payload.customMetrics.reduce(
(sum, metric) => sum + metric.entries.length,
0,
),
correlationPatterns: payload.correlationPatterns.length,
intradayProfiles: payload.intradayProfiles.length,
healthScoreRecords: payload.healthScoreRecords.length,
onboardingRecords: payload.onboardingRecord ? 1 : 0,
// v1.37.19 (A6-8) — the sections restored since 08-01 were written and
// restored but absent from this report, so the admin's "what did I just
// restore" answer silently under-counted a file that carried visits or
// an Impfpass.
practitioners: payload.practitioners.length,
encounters: payload.encounters.length,
encounterLinks:
payload.encounterDocumentLinks.length +
payload.encounterLabLinks.length +
payload.encounterConditionLinks.length,
vaccinations: payload.vaccinations.length,
vaccinationLinks: payload.vaccinationDocumentLinks.length,
// v1.37.20 (#223 / iOS #68) — counted from the release that carries them,
// so the admin's "what did I just restore" answer never under-counts a
// file with reminders the way it once did for visits.
measurementReminders: payload.measurementReminders.length,
measurementReminderEvents: payload.measurementReminderEvents.length,
coachConversations: payload.coachConversations.length,
coachMessages: payload.coachConversations.reduce(
(total, conversation) => total + conversation.messages.length,
0,
),
coachFacts: payload.coachFacts.length,
coachPlans: payload.coachPlans.length,
coachReminders: payload.coachReminders.length,
mentalHealthAssessments: payload.mentalHealthAssessments.length,
consentReceipts: payload.consentReceipts.length,
// Counted from the release that carries them, so the admin's "what did I
// just restore" answer never under-counts a file with bests, badges or an
// environmental history the way it once did for visits.
personalRecords: payload.personalRecords.length,
userAchievements: payload.userAchievements.length,
environmentContexts: payload.environmentContexts.length,
environmentTravelLocations: payload.environmentTravelLocations.length,
ecgRecordings: payload.ecgRecordings.length,
};
}
/**
* The schemaVersion the system understands today. Used by the upload
* route to reject inbound files written by a *future* HealthLog instance
* — restoring them under the current code might silently drop data the
* new shape carried.
*/
export function isCompatibleSchemaVersion(version: string): boolean {
return (
version === LEGACY_BACKUP_SCHEMA_VERSION ||
version === BACKUP_SCHEMA_VERSION
);
}
+4 -203
View File
@@ -88,14 +88,15 @@ import {
DOCUMENT_SOURCE_SYSTEMS,
} from "@/lib/validations/inbound-documents";
import { REMINDER_EVENT_SOURCES } from "@/lib/measurement-reminders/satisfy";
import {
BACKUP_SCHEMA_VERSION,
LEGACY_BACKUP_SCHEMA_VERSION,
} from "@/lib/validations/backup-summary";
import {
UNITS_PER_DOSE_MESSAGE,
isSupportedUnitsPerDose,
} from "@/lib/medications/units-per-dose";
export const BACKUP_SCHEMA_VERSION = "2" as const;
const LEGACY_BACKUP_SCHEMA_VERSION = "1" as const;
const isoDateTime = z
.string()
.min(1)
@@ -2078,193 +2079,6 @@ export type BackupPayload = z.infer<typeof backupPayloadSchema>;
export const backupMeasurementSchema = measurementSchema;
export type BackupMeasurement = z.infer<typeof measurementSchema>;
/**
* Numeric counts of each backed-up record kind. Returned in the
* upload + restore API responses so the admin sees what they
* uploaded/restored without having to download the file again.
*/
export interface BackupSummary {
schemaVersion: string;
userId: string;
exportedAt: string;
measurements: number;
medications: number;
intakeEvents: number;
/** Side effects recorded against a drug, across every medication. */
medicationSideEffects: number;
/** Pinned efficacy targets, across every medication. */
medicationEfficacyTargets: number;
/** Archived schedule eras, across every medication. */
medicationScheduleRevisions: number;
moodEntries: number;
/** v1.15.0 — observed cycle spans in the backup. */
cycles: number;
/** v1.15.0 — cycle day-logs in the backup. */
cycleDayLogs: number;
/** Lab results in the backup. */
labResults: number;
nutrientDays: number;
/** User-scoped biomarker catalog entries in the backup. */
biomarkers: number;
/** Illness episodes, including flares/exacerbations. */
illnessEpisodes: number;
/** Illness day-logs across every episode. */
illnessDayLogs: number;
/** Allergy/intolerance records in the backup. */
allergies: number;
/** Family-history entries in the backup. */
familyHistory: number;
/** Workout summary records in the backup. */
workouts: number;
/** Document records (ciphertext included in canonical DR payloads). */
documents: number;
/** Which documents were filed against which condition. */
documentConditionLinks: number;
/** Facts staged out of a document, with their review decision. */
extractedFacts: number;
/** 1 when the account's durable self-context rides the file, 0 otherwise. */
healthProfile: number;
/** Effective-dated structured health-profile revisions. */
healthProfileFactRevisions: number;
/** Metrics the account defined itself. */
customMetrics: number;
/** Readings across every user-defined metric. */
customMetricEntries: number;
/** Persisted accepted correlation identities and dismissal decisions. */
correlationPatterns: number;
/** Stored day curves for the cumulative metrics, across every metric. */
intradayProfiles: number;
/** Local days whose health score was written down as it was shown. */
healthScoreRecords: number;
/** 1 when the record's setup answers ride the file, 0 otherwise. */
onboardingRecords: number;
/** v1.37.19 (A6-8) — the visit address book. */
practitioners: number;
/** v1.37.19 (A6-8) — doctor visits, planned and past. */
encounters: number;
/** v1.37.19 (A6-8) — the three encounter link tables, summed. */
encounterLinks: number;
/** v1.37.19 (A6-8) — immunization log entries. */
vaccinations: number;
/** v1.37.19 (A6-8) — vaccination↔document links. */
vaccinationLinks: number;
/** v1.37.20 (#223 / iOS #68) — Vorsorge reminder cadences. */
measurementReminders: number;
/** v1.37.20 (#223 / iOS #68) — completion-ledger rows across every reminder. */
measurementReminderEvents: number;
/** Coach conversation threads. */
coachConversations: number;
/** Coach turns across every thread. */
coachMessages: number;
/** Durable facts, agreed plans and things to bring back up. */
coachFacts: number;
coachPlans: number;
coachReminders: number;
/** Completed screener administrations. Disaster-recovery payloads only. */
mentalHealthAssessments: number;
/** Consent records. Disaster-recovery payloads only. */
consentReceipts: number;
/** Personal bests across every metric and sport slot. */
personalRecords: number;
/** Badges the account has unlocked, each with the day it was earned. */
userAchievements: number;
/** Local days with an environmental reading. */
environmentContexts: number;
/** Declared stretches spent away from home. */
environmentTravelLocations: number;
/** ECG strips, with their rhythm verdict and their trace. */
ecgRecordings: number;
}
export function summarizeBackup(payload: BackupPayload): BackupSummary {
return {
schemaVersion: payload.schemaVersion,
userId: payload.userId,
exportedAt: payload.exportedAt,
measurements: payload.measurements.length,
medications: payload.medications.length,
intakeEvents: payload.intakeEvents.length,
medicationSideEffects: payload.medications.reduce(
(sum, medication) => sum + medication.sideEffects.length,
0,
),
medicationEfficacyTargets: payload.medications.reduce(
(sum, medication) => sum + medication.efficacyTargets.length,
0,
),
medicationScheduleRevisions: payload.medications.reduce(
(sum, medication) => sum + medication.scheduleRevisions.length,
0,
),
moodEntries: payload.moodEntries.length,
cycles: payload.cycles.length,
cycleDayLogs: payload.cycleDayLogs.length,
nutrientDays: payload.nutrientDays.length,
labResults: payload.labResults.length,
biomarkers: payload.biomarkers.length,
illnessEpisodes: payload.illnessEpisodes.length,
illnessDayLogs: payload.illnessEpisodes.reduce(
(sum, e) => sum + e.dayLogs.length,
0,
),
allergies: payload.allergies.length,
familyHistory: payload.familyHistory.length,
workouts: payload.workouts.length,
documents: payload.documents.length,
// Counted from the release that carries them, so the admin's "what did I
// just restore" answer never under-counts a filed vault the way it once
// did for visits.
documentConditionLinks: payload.documentConditionLinks.length,
extractedFacts: payload.extractedFacts.length,
healthProfile: payload.healthProfile ? 1 : 0,
healthProfileFactRevisions: payload.healthProfileFacts.length,
customMetrics: payload.customMetrics.length,
customMetricEntries: payload.customMetrics.reduce(
(sum, metric) => sum + metric.entries.length,
0,
),
correlationPatterns: payload.correlationPatterns.length,
intradayProfiles: payload.intradayProfiles.length,
healthScoreRecords: payload.healthScoreRecords.length,
onboardingRecords: payload.onboardingRecord ? 1 : 0,
// v1.37.19 (A6-8) — the sections restored since 08-01 were written and
// restored but absent from this report, so the admin's "what did I just
// restore" answer silently under-counted a file that carried visits or
// an Impfpass.
practitioners: payload.practitioners.length,
encounters: payload.encounters.length,
encounterLinks:
payload.encounterDocumentLinks.length +
payload.encounterLabLinks.length +
payload.encounterConditionLinks.length,
vaccinations: payload.vaccinations.length,
vaccinationLinks: payload.vaccinationDocumentLinks.length,
// v1.37.20 (#223 / iOS #68) — counted from the release that carries them,
// so the admin's "what did I just restore" answer never under-counts a
// file with reminders the way it once did for visits.
measurementReminders: payload.measurementReminders.length,
measurementReminderEvents: payload.measurementReminderEvents.length,
coachConversations: payload.coachConversations.length,
coachMessages: payload.coachConversations.reduce(
(total, conversation) => total + conversation.messages.length,
0,
),
coachFacts: payload.coachFacts.length,
coachPlans: payload.coachPlans.length,
coachReminders: payload.coachReminders.length,
mentalHealthAssessments: payload.mentalHealthAssessments.length,
consentReceipts: payload.consentReceipts.length,
// Counted from the release that carries them, so the admin's "what did I
// just restore" answer never under-counts a file with bests, badges or an
// environmental history the way it once did for visits.
personalRecords: payload.personalRecords.length,
userAchievements: payload.userAchievements.length,
environmentContexts: payload.environmentContexts.length,
environmentTravelLocations: payload.environmentTravelLocations.length,
ecgRecordings: payload.ecgRecordings.length,
};
}
/**
* Parse a JSON blob (string or already-parsed object) against
* `backupPayloadSchema`, returning a typed payload. Throws ZodError on
@@ -2280,16 +2094,3 @@ export function parseBackupPayload(input: string | unknown): BackupPayload {
const raw = typeof input === "string" ? JSON.parse(input) : input;
return backupPayloadSchema.parse(raw);
}
/**
* The schemaVersion the system understands today. Used by the upload
* route to reject inbound files written by a *future* HealthLog instance
* — restoring them under the current code might silently drop data the
* new shape carried.
*/
export function isCompatibleSchemaVersion(version: string): boolean {
return (
version === LEGACY_BACKUP_SCHEMA_VERSION ||
version === BACKUP_SCHEMA_VERSION
);
}
@@ -9,9 +9,10 @@
verbatim; that has to be checked against a real export (see #1109).
Records, in order:
1. weight 80.1 carries the HealthLog origin marker -> left out
1. weight 80.1 origin marker and a HealthLog row id -> left out
when the importing account holds that row
2. weight 80.2 user-entered, no marker -> imported
3. weight 80.3 marker among other metadata entries -> left out
3. weight 80.3 marker among other metadata entries, no id -> imported
4. weight 80.4 no metadata at all -> imported
5. weight 80.5 origin key with another value -> imported
-->
@@ -9,7 +9,12 @@
* twice and the rollups count it twice.
*
* Asserted against a real Postgres:
* - a record carrying the marker is left out, and counted;
* - a record carrying the marker and a row id of this account is left out,
* and counted; the marker alone proves nothing about this account (a move
* to a new instance without a backup), so a marked record whose id is
* missing, unknown or another account's is imported, a deleted row of this
* account still counts, and a marked cycle day counts only when the
* account holds a day-log on it;
* - a record without it lands as before;
* - the marker is read from a child `<MetadataEntry>` that follows the
* record's open tag, so the record is committed at its close tag;
@@ -56,13 +61,46 @@ ${records}
const weight = (value: number, end: string, children = "") =>
` <Record type="HKQuantityTypeIdentifierBodyMass" sourceName="Health" unit="kg" startDate="${end}" endDate="${end}" value="${value}">${children}</Record>`;
const OWN = `<MetadataEntry key="dev.healthlog.app.origin" value="healthlog"/>`;
const MARKER = `<MetadataEntry key="dev.healthlog.app.origin" value="healthlog"/>`;
const ext = (id: string) =>
`<MetadataEntry key="HKExternalUUID" value="${id}"/>`;
/** The row the app mirrored: a weight of this account, its id on the sample. */
const OWN_WEIGHT_ID = "own-weight-row";
const OWN_SYS_ID = "own-sys-row";
const OWN = `${MARKER}${ext(OWN_WEIGHT_ID)}`;
const OWN_BP = `${MARKER}${ext(OWN_SYS_ID)}`;
const OTHER = `<MetadataEntry key="HKWasUserEntered" value="1"/>`;
/**
* A row of this account the samples point at. Withings and ten minutes off the
* samples, so neither the series under test nor the manual-value match sees it.
*/
async function seedOwnRow(
userId: string,
id: string,
type: "WEIGHT" | "BLOOD_PRESSURE_SYS" | "ACTIVITY_STEPS",
deletedAt: Date | null = null,
) {
await prisma.measurement.create({
data: {
id,
userId,
type,
value: type === "ACTIVITY_STEPS" ? 1000 : 70,
unit:
type === "WEIGHT" ? "kg" : type === "ACTIVITY_STEPS" ? "steps" : "mmHg",
source: "WITHINGS",
measuredAt: new Date("2026-05-14T06:10:00.000Z"),
deletedAt,
},
});
}
async function run(records: string) {
const user = await prisma.user.create({
data: { username: "own-samples", email: "own@example.test", role: "USER" },
});
await seedOwnRow(user.id, OWN_WEIGHT_ID, "WEIGHT");
const result = await streamParseExportXml({
xmlPath: writeXml(records),
userId: user.id,
@@ -139,6 +177,7 @@ describe("blood pressure correlations and self-closing records", () => {
const user = await prisma.user.create({
data: { username: "own-bp", email: "bp@example.test", role: "USER" },
});
await seedOwnRow(user.id, OWN_SYS_ID, "BLOOD_PRESSURE_SYS");
const result = await streamParseExportXml({
xmlPath: writeXml(records),
userId: user.id,
@@ -148,6 +187,7 @@ describe("blood pressure correlations and self-closing records", () => {
const rows = await prisma.measurement.findMany({
where: {
userId: user.id,
source: "APPLE_HEALTH",
type: { in: ["BLOOD_PRESSURE_SYS", "BLOOD_PRESSURE_DIA"] },
},
orderBy: [{ type: "asc" }],
@@ -173,7 +213,7 @@ describe("blood pressure correlations and self-closing records", () => {
[bpRecord("Systolic", 121, AT), bpRecord("Diastolic", 79, AT)].join(
"\n",
),
`\n ${OWN}`,
`\n ${OWN_BP}`,
),
);
expect(values).toEqual([]);
@@ -186,7 +226,7 @@ describe("blood pressure correlations and self-closing records", () => {
bpRecord("Diastolic", 79, AT),
].join("\n");
const { values } = await runBp(
` <Correlation type="HKCorrelationTypeIdentifierBloodPressure" sourceName="Health" startDate="${AT}" endDate="${AT}">\n${records}\n ${OWN}\n </Correlation>`,
` <Correlation type="HKCorrelationTypeIdentifierBloodPressure" sourceName="Health" startDate="${AT}" endDate="${AT}">\n${records}\n ${OWN_BP}\n </Correlation>`,
);
expect(values).toEqual([]);
});
@@ -195,7 +235,7 @@ describe("blood pressure correlations and self-closing records", () => {
const { values } = await runBp(
correlation(
[
bpRecord("Systolic", 121, AT, `\n ${OWN}\n `),
bpRecord("Systolic", 121, AT, `\n ${OWN_BP}\n `),
bpRecord("Diastolic", 79, AT),
].join("\n"),
),
@@ -210,7 +250,7 @@ describe("blood pressure correlations and self-closing records", () => {
[bpRecord("Systolic", 121, AT), bpRecord("Diastolic", 79, AT)].join(
"\n",
),
`\n ${OWN}`,
`\n ${OWN_BP}`,
),
bpRecord("Systolic", 130, "2026-05-14 09:00:00 +0200"),
].join("\n"),
@@ -230,10 +270,11 @@ describe("the synthetic export fixture", () => {
// `export-own-origin.synthetic.xml` is invented, in the shape Apple's DTD
// documents. It shows the parser reads a file laid out like an export; it
// cannot show that Apple writes a custom metadata key verbatim.
it("leaves out the two marked records and imports the other three", async () => {
it("leaves out the marked record this account holds and imports the other four", async () => {
const user = await prisma.user.create({
data: { username: "own-fixture", email: "fx@example.test", role: "USER" },
});
await seedOwnRow(user.id, "00000000-0000-0000-0000-000000000001", "WEIGHT");
const result = await streamParseExportXml({
xmlPath: join(
process.cwd(),
@@ -247,9 +288,9 @@ describe("the synthetic export fixture", () => {
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
orderBy: { measuredAt: "asc" },
});
expect(rows.map((r) => r.value)).toEqual([80.2, 80.4, 80.5]);
expect(rows.map((r) => r.value)).toEqual([80.2, 80.3, 80.4, 80.5]);
expect(result.writtenByHealthLog).toEqual({
byMarker: 2,
byMarker: 1,
byExternalId: 0,
matchedManual: 0,
});
@@ -421,8 +462,6 @@ describe("the manual-mirror lookup stays off the hot path", () => {
describe("a sample whose HKExternalUUID is a HealthLog row id", () => {
const AT = "2026-05-14 08:00:00 +0200";
const ext = (id: string) =>
`<MetadataEntry key="HKExternalUUID" value="${id}"/>`;
async function runWith(
seed: (userId: string) => Promise<Record<string, string>>,
@@ -524,11 +563,12 @@ describe("a sample whose HKExternalUUID is a HealthLog row id", () => {
describe("each sample left out is counted once", () => {
it("does not count a marked record twice when it appears top level and inside its correlation", async () => {
const AT = "2026-05-14 08:00:00 +0200";
const sys = bpRecord("Systolic", 121, AT, `\n ${OWN}\n `);
const dia = bpRecord("Diastolic", 79, AT, `\n ${OWN}\n `);
const sys = bpRecord("Systolic", 121, AT, `\n ${OWN_BP}\n `);
const dia = bpRecord("Diastolic", 79, AT, `\n ${OWN_BP}\n `);
const user = await prisma.user.create({
data: { username: "own-once", email: "once@example.test", role: "USER" },
});
await seedOwnRow(user.id, OWN_SYS_ID, "BLOOD_PRESSURE_SYS");
const result = await streamParseExportXml({
xmlPath: writeXml(
[sys, dia, correlation([sys, dia].join("\n"))].join("\n"),
@@ -542,7 +582,7 @@ describe("each sample left out is counted once", () => {
});
describe("a cycle sample HealthLog wrote itself", () => {
it("is left out of the day-log import", async () => {
it("is left out on a day this account holds, imported on a day it does not", async () => {
const user = await prisma.user.create({
data: {
username: "own-cycle",
@@ -552,11 +592,20 @@ describe("a cycle sample HealthLog wrote itself", () => {
timezone: "Europe/Berlin",
},
});
// The day the app mirrored from: a day-log of this account.
await prisma.cycleDayLog.create({
data: { userId: user.id, date: "2026-03-02", flow: "MEDIUM" },
});
const flow = (day: string, children = "") =>
` <Record type="HKCategoryTypeIdentifierMenstrualFlow" sourceName="Health" value="HKCategoryValueMenstrualFlowMedium" startDate="${day} 08:00:00 +0000" endDate="${day} 08:00:00 +0000">${children}</Record>`;
const result = await streamParseExportXml({
xmlPath: writeXml(
[flow("2026-03-02", OWN), flow("2026-03-03")].join("\n"),
[
flow("2026-03-02", MARKER),
flow("2026-03-03"),
// Marked, but no day-log here: a move to a new instance.
flow("2026-03-04", MARKER),
].join("\n"),
),
userId: user.id,
userTimezone: "Europe/Berlin",
@@ -566,7 +615,119 @@ describe("a cycle sample HealthLog wrote itself", () => {
where: { userId: user.id },
orderBy: { date: "asc" },
});
expect(days.map((d) => d.date)).toEqual(["2026-03-03"]);
expect(days.map((d) => d.date)).toEqual([
"2026-03-02",
"2026-03-03",
"2026-03-04",
]);
// The held day stays the account's own: the marked sample did not
// re-import it as an Apple Health day.
expect(days[0].source).toBe("MANUAL");
expect(result.writtenByHealthLog.byMarker).toBe(1);
});
});
describe("a marked record this account does not hold", () => {
// A move to a new instance without a backup: the export still carries the
// marker on every value the app mirrored, but none of the rows exist here.
it("is imported when the id is missing, unknown or another account's", async () => {
const other = await prisma.user.create({
data: { username: "own-move-2", email: "mv2@example.test", role: "USER" },
});
await seedOwnRow(other.id, "foreign-weight-row", "WEIGHT");
const user = await prisma.user.create({
data: { username: "own-move", email: "mv@example.test", role: "USER" },
});
const result = await streamParseExportXml({
xmlPath: writeXml(
[
weight(80.1, "2026-05-14 08:00:00 +0200", MARKER),
weight(
80.2,
"2026-05-14 09:00:00 +0200",
`${MARKER}${ext("00000000-0000-4000-8000-000000000009")}`,
),
weight(
80.3,
"2026-05-14 10:00:00 +0200",
`${MARKER}${ext("foreign-weight-row")}`,
),
].join("\n"),
),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
const rows = await prisma.measurement.findMany({
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
orderBy: { measuredAt: "asc" },
});
expect(rows.map((r) => r.value)).toEqual([80.1, 80.2, 80.3]);
expect(result.writtenByHealthLog).toEqual({
byMarker: 0,
byExternalId: 0,
matchedManual: 0,
});
});
it("is still left out when its id names a deleted row of this account", async () => {
const user = await prisma.user.create({
data: {
username: "own-deleted",
email: "del@example.test",
role: "USER",
},
});
await seedOwnRow(user.id, OWN_WEIGHT_ID, "WEIGHT", new Date());
const result = await streamParseExportXml({
xmlPath: writeXml(weight(80.1, "2026-05-14 08:00:00 +0200", OWN)),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
const rows = await prisma.measurement.findMany({
where: { userId: user.id, type: "WEIGHT", source: "APPLE_HEALTH" },
});
expect(rows).toHaveLength(0);
expect(result.writtenByHealthLog.byMarker).toBe(1);
});
it("adds a marked cumulative sample to its day unless its id is this account's", async () => {
const user = await prisma.user.create({
data: { username: "own-steps", email: "st@example.test", role: "USER" },
});
await seedOwnRow(user.id, "own-steps-row", "ACTIVITY_STEPS");
const steps = (value: number, at: string, children: string) =>
` <Record type="HKQuantityTypeIdentifierStepCount" sourceName="Health" unit="count" startDate="${at}" endDate="${at}" value="${value}">${children}</Record>`;
const result = await streamParseExportXml({
xmlPath: writeXml(
[
steps(
700,
"2026-05-14 08:00:00 +0200",
`${MARKER}${ext("own-steps-row")}`,
),
steps(
300,
"2026-05-14 09:00:00 +0200",
`${MARKER}${ext("not-a-row")}`,
),
steps(200, "2026-05-14 10:00:00 +0200", ""),
].join("\n"),
),
userId: user.id,
userTimezone: "Europe/Berlin",
prisma,
});
const rows = await prisma.measurement.findMany({
where: {
userId: user.id,
type: "ACTIVITY_STEPS",
source: "APPLE_HEALTH",
},
});
// 300 + 200: the 700 the app mirrored from this account's row stays out.
expect(rows.map((r) => r.value)).toEqual([500]);
expect(result.writtenByHealthLog.byMarker).toBe(1);
});
});
@@ -118,6 +118,11 @@ describe("weekly copy preview under a memory budget", () => {
});
it("holds nothing of the record once the producer is done", async () => {
// The backup schema the preview checks each section against is a module
// loaded once per process (about 12 MB, `backup-summary.ts`), not memory
// that grows with the record: load it before the baseline, as a running
// server has long since done.
await import("@/lib/validations/backup");
const baseline = liveHeapBytes();
let peakHeld = 0;
let heldAtEnd = 0;
@@ -127,7 +132,7 @@ describe("weekly copy preview under a memory budget", () => {
rawPeak = Math.max(rawPeak, process.memoryUsage().heapUsed);
}, 5);
// Wired exactly as the weekly job wires it (`backup-handlers.ts`).
const preview = createBackupPreviewCollector();
const preview = await createBackupPreviewCollector();
let counts: FullBackupCounts | undefined;
try {
await storeBackupBlob(
+2 -2
View File
@@ -126,8 +126,8 @@ async function preview(id: string) {
/** The counts a full read of the stored copy gives: the old preview. */
async function fullReadSummary(id: string) {
const { parseBackupPayload, summarizeBackup } =
await import("@/lib/validations/backup");
const { parseBackupPayload } = await import("@/lib/validations/backup");
const { summarizeBackup } = await import("@/lib/validations/backup-summary");
const opensBefore = opened.count;
const payload = parseBackupPayload(
JSON.parse(await readStoredBackup(getPrismaClient(), id)),
+4 -3
View File
@@ -79,7 +79,8 @@ import { legacyStreamedBlobFrom } from "@/__tests__/helpers/legacy-backup-blob";
import { streamFullBackupJson } from "@/lib/export/full-backup-stream";
import { BackupKeyIdCollector } from "@/lib/export/backup-key-ids";
import { createBackupPreviewCollector } from "@/lib/export/backup-preview";
import { parseBackupPayload, summarizeBackup } from "@/lib/validations/backup";
import { parseBackupPayload } from "@/lib/validations/backup";
import { summarizeBackup } from "@/lib/validations/backup-summary";
import {
TWO_ENDED_MODELS,
USER_COLUMN_BACKUP_CLASS,
@@ -1494,7 +1495,7 @@ describe("every model the plan claims two-ended survives a real restore", () =>
// below then covers BOTH, since the row that gets restored comes from the
// streamed bytes.
let streamedJson = "";
const previewCollector = createBackupPreviewCollector();
const previewCollector = await createBackupPreviewCollector();
const streamedCounts = await streamFullBackupJson(
prisma,
OWNER_ID,
@@ -3778,7 +3779,7 @@ describe("every column of every two-ended model survives a real restore", () =>
const exportedAt = new Date("2026-08-01T00:00:00.000Z");
let json = "";
const previewCollector = createBackupPreviewCollector();
const previewCollector = await createBackupPreviewCollector();
const counts = await streamFullBackupJson(
prisma,
OWNER_ID,
@@ -0,0 +1,232 @@
/**
* The shared pulse day weight (`dayWeightedRows`) must not depend on planner
* statistics. A new account importing a year of pulse lands below the
* autoanalyze threshold, so the first reads run against a table the planner
* believes is empty. The earlier shape joined the CTE to an hour grouping of
* itself; on such a table the planner chose a nested loop and the all-time
* pulse aggregate ran into the 60 s production statement timeout.
*
* This file seeds a year of dense pulse (well over 100k rows) with autovacuum
* off for the table and no ANALYZE, then:
*
* - times the real all-time reader (`readAllTimeExtremes`) and a bare
* day-weighted mean, each well under the timeout;
* - after ANALYZE, compares the weights per day, per frame and per source
* against the earlier self-join SQL kept below as the oracle.
*/
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import type { Prisma } from "@/generated/prisma/client";
import { readAllTimeExtremes } from "@/lib/insights/feature-blocks";
import {
SESSION_DAY_FRAME,
dayWeightedRows,
zoneDayFrame,
type DayFrame,
} from "@/lib/measurements/day-mean";
const prisma = getPrismaClient();
const USER = "day-weight-scale";
const DAY = 86_400_000;
const MINUTE = 60_000;
const DAYS = 365;
/** Generous against a 60 s production timeout, far above what one scan takes. */
const BOUND_MS = 15_000;
const start =
Date.UTC(
new Date().getUTCFullYear(),
new Date().getUTCMonth(),
new Date().getUTCDate(),
) -
DAYS * DAY;
/**
* About 330 readings a day, unevenly spread so hours carry different counts,
* plus a second source on every fifth day so the per-source weights differ.
*/
function seedRows(): Prisma.MeasurementCreateManyInput[] {
const rows: Prisma.MeasurementCreateManyInput[] = [];
let i = 0;
for (let d = 0; d < DAYS; d += 1) {
for (let m = 0; m < 24 * 60; m += 4) {
if ((m + d) % 13 === 0) continue;
const hour = Math.floor(m / 60);
// Night hours sparse, day hours dense.
if (hour < 6 && m % 12 !== 0) continue;
rows.push({
id: `${USER}-a-${i}`,
userId: USER,
type: "PULSE",
unit: "bpm",
source: "APPLE_HEALTH",
value: 55 + ((i * 7 + d) % 50),
measuredAt: new Date(start + d * DAY + m * MINUTE + (i % 3) * 1000),
});
i += 1;
}
if (d % 5 === 0) {
for (const h of [7, 7, 13, 22]) {
rows.push({
id: `${USER}-w-${i}`,
userId: USER,
type: "PULSE",
unit: "bpm",
source: "WITHINGS",
value: 60 + (i % 20),
measuredAt: new Date(
start + d * DAY + h * 3_600_000 + (i % 50) * MINUTE,
),
});
i += 1;
}
}
}
return rows;
}
/** The earlier self-join, kept verbatim as the oracle for the weights. */
function oracleWeightedRows(
source: string,
frame: DayFrame,
bySource: boolean,
): string {
const ts = (column: string) =>
frame.kind === "session"
? column
: `((${column} AT TIME ZONE 'UTC') AT TIME ZONE ${frame.tzSql})`;
const hourOf = (alias: string) =>
`date_trunc('hour', ${ts(`${alias}."measured_at"`)})`;
const dayOf = (alias: string) =>
`date_trunc('day', ${ts(`${alias}."measured_at"`)})`;
const src = bySource ? `, t."source"` : "";
const part = bySource ? `, x."source"` : "";
const join = bySource ? ` AND hw."source" = s."source"` : "";
return `(
SELECT s.*,
(CASE WHEN (s."type")::text IN ('PULSE')
THEN 1.0 / (hw.n * hw.hours)
ELSE 1.0 END)::double precision AS day_weight
FROM ${source} s
LEFT JOIN (
SELECT x."type"${part}, x.local_hour, x.n,
COUNT(*) OVER (PARTITION BY x."type"${part}, x.local_day) AS hours
FROM (
SELECT t."type"${src},
${hourOf("t")} AS local_hour,
${dayOf("t")} AS local_day,
COUNT(*) AS n
FROM ${source} t
WHERE (t."type")::text IN ('PULSE')
GROUP BY ${bySource ? "1, 2, 3, 4" : "1, 2, 3"}
) x
) hw
ON hw."type" = s."type"${join}
AND hw.local_hour = ${hourOf("s")}
)`;
}
const SRC_CTE = `WITH src AS (
SELECT m."id", m."type", m."source", m."measured_at", m."value"
FROM measurements m
WHERE m."user_id" = $1 AND m."deleted_at" IS NULL AND m."type" = 'PULSE'
)`;
async function timed<T>(fn: () => Promise<T>): Promise<[T, number]> {
const t0 = performance.now();
const out = await fn();
return [out, performance.now() - t0];
}
let rowCount = 0;
beforeAll(async () => {
await truncateAllTables(prisma);
await prisma.$executeRawUnsafe(
"ALTER TABLE measurements SET (autovacuum_enabled = false)",
);
await prisma.user.create({ data: { id: USER, username: USER } });
const rows = seedRows();
rowCount = rows.length;
for (let k = 0; k < rows.length; k += 20_000) {
await prisma.measurement.createMany({ data: rows.slice(k, k + 20_000) });
}
}, 180_000);
afterAll(async () => {
await prisma.$executeRawUnsafe(
"ALTER TABLE measurements RESET (autovacuum_enabled)",
);
});
describe("day weight on an unanalysed table", () => {
it("seeds a year of dense pulse", () => {
expect(rowCount).toBeGreaterThan(100_000);
});
it("answers the all-time pulse aggregate well under the timeout", async () => {
const [out, ms] = await timed(() => readAllTimeExtremes(USER, ["PULSE"]));
console.info(`[day-weight] readAllTimeExtremes ${ms.toFixed(0)} ms`);
expect(out.get("PULSE")?.mean).toBeGreaterThan(55);
expect(ms).toBeLessThan(BOUND_MS);
});
it("answers a bare day-weighted mean well under the timeout", async () => {
const [rows, ms] = await timed(() =>
prisma.$queryRawUnsafe<Array<{ mean: number }>>(
`${SRC_CTE}
SELECT (SUM(w."value" * w.day_weight) / SUM(w.day_weight))::double precision AS mean
FROM ${dayWeightedRows("src", zoneDayFrame("$2"), { bySource: true })} w`,
USER,
"Europe/Berlin",
),
);
console.info(`[day-weight] zone + bySource mean ${ms.toFixed(0)} ms`);
expect(rows[0].mean).toBeGreaterThan(55);
expect(ms).toBeLessThan(BOUND_MS);
});
});
describe("day weight against the earlier self-join", () => {
beforeAll(async () => {
// Statistics now, so the oracle plans the way it did in production.
await prisma.$executeRawUnsafe("ANALYZE measurements");
}, 120_000);
const cases: Array<[string, DayFrame, boolean, unknown[]]> = [
["session frame", SESSION_DAY_FRAME, false, []],
["session frame by source", SESSION_DAY_FRAME, true, []],
["Berlin frame", zoneDayFrame("$2"), false, ["Europe/Berlin"]],
["Kolkata frame by source", zoneDayFrame("$2"), true, ["Asia/Kolkata"]],
];
it.each(cases)(
"%s gives every row the same weight",
async (_label, frame, bySource, extra) => {
const query = (sub: string) =>
prisma.$queryRawUnsafe<Array<{ id: string; w: number }>>(
`${SRC_CTE}
SELECT w."id" AS id, w.day_weight AS w
FROM ${sub} w
ORDER BY w."id"`,
USER,
...extra,
);
const fresh = await query(dayWeightedRows("src", frame, { bySource }));
const oracle = await query(oracleWeightedRows("src", frame, bySource));
expect(fresh.length).toBe(rowCount);
expect(fresh.length).toBe(oracle.length);
let mismatches = 0;
for (let k = 0; k < fresh.length; k += 1) {
if (
fresh[k].id !== oracle[k].id ||
Math.abs(fresh[k].w - oracle[k].w) > 1e-12
) {
mismatches += 1;
}
}
expect(mismatches).toBe(0);
},
);
});
@@ -0,0 +1,296 @@
/**
* The MCP single-metric reads build one source's section from that source's
* rows (`source-snapshot.ts`) instead of a whole Coach snapshot. The promise
* is that nothing a client receives changes, so this pins it against the full
* builder on real rows: for every seeded source, across windows, the tool's
* result serialises to exactly what `buildCoachSnapshot` scoped to that source
* yields for the same section and grounding — the result `get_metric_series`
* returned before. A dense series pushes the feature extraction's all-type row
* cap into play, a year of blood pressure is large enough to go back to the
* full builder, and an absent source stays an explicit absence.
*/
import { beforeAll, describe, expect, it, vi } from "vitest";
import { getPrismaClient, truncateAllTables } from "./setup";
import { MCP_TOOLS } from "@/lib/mcp/tools";
import { MCP_RESOURCE_TEMPLATES } from "@/lib/mcp/resources";
import type { McpAuthContext } from "@/lib/mcp/auth";
import {
__resetCoachSnapshotCacheForTests,
buildCoachSnapshot,
} from "@/lib/ai/coach/snapshot";
import { SOURCE_SNAPSHOT_FALLBACK_CHARS } from "@/lib/ai/coach/source-snapshot";
import { COACH_SOURCE_SNAPSHOT_KEY } from "@/lib/ai/coach/tools/source-keys";
import { recomputeUserRollups } from "@/lib/rollups/measurement-rollups";
import { ensureUserMoodRollupsFresh } from "@/lib/rollups/mood-rollups";
import type { CoachScopeSource, CoachScopeWindow } from "@/lib/ai/coach/types";
import type { MeasurementType, SleepStage } from "@/generated/prisma/client";
vi.mock("@/lib/db-compat", () => ({
ensureDbCompatibility: vi.fn().mockResolvedValue(undefined),
}));
vi.mock("@/lib/jobs/boss-instance", () => ({
getGlobalBoss: vi.fn(() => null),
}));
const DAY_MS = 86_400_000;
const HOUR_MS = 3_600_000;
let userId = "";
function ctx(): McpAuthContext {
return {
userId,
tokenId: "token-int",
scopes: ["health:read"],
binding: `${userId}:token-int`,
canRead: true,
canWrite: false,
};
}
function tool(name: string) {
const def = MCP_TOOLS.find((t) => t.name === name);
if (!def) throw new Error(`tool ${name} not registered`);
return def;
}
/** What `get_metric_series` answered before: the full builder, sliced. */
async function oldPathResult(
metric: CoachScopeSource,
window: CoachScopeWindow | undefined,
) {
__resetCoachSnapshotCacheForTests();
const snapshot = await buildCoachSnapshot(userId, {
sources: [metric],
window,
});
const section = snapshot.sections[COACH_SOURCE_SNAPSHOT_KEY[metric]!];
if (section === undefined) return { section, result: undefined };
return {
section,
result: {
present: true,
data: { metric, section },
grounding: snapshot.referenceGrounding ?? undefined,
},
};
}
async function newPathResult(
metric: CoachScopeSource,
window: CoachScopeWindow | undefined,
) {
__resetCoachSnapshotCacheForTests();
return tool("get_metric_series").run(
ctx(),
window ? { metric, window } : { metric },
);
}
const PRESENT: CoachScopeSource[] = [
"pulse",
"resting_hr",
"bp",
"weight",
"sleep",
"steps",
"hrv",
"body_fat",
"mood",
];
const WINDOWS: Array<CoachScopeWindow | undefined> = [
undefined,
"last7days",
"last90days",
"allTime",
];
beforeAll(async () => {
const prisma = getPrismaClient();
await truncateAllTables(prisma);
const user = await prisma.user.create({
data: {
username: "mcp-source-snapshot",
email: "mcp-source-snapshot@example.test",
timezone: "Europe/Berlin",
dateOfBirth: new Date("1980-05-01T00:00:00Z"),
heightCm: 180,
},
});
userId = user.id;
const now = Date.now();
const rows: Array<{
userId: string;
type: MeasurementType;
value: number;
unit: string;
source: "APPLE_HEALTH" | "WITHINGS";
measuredAt: Date;
sleepStage?: SleepStage;
externalId?: string;
}> = [];
const add = (
type: MeasurementType,
unit: string,
count: number,
stepMs: number,
value: (i: number) => number,
source: "APPLE_HEALTH" | "WITHINGS" = "APPLE_HEALTH",
) => {
for (let i = 0; i < count; i++) {
rows.push({
userId,
type,
value: value(i),
unit,
source,
measuredAt: new Date(now - i * stepMs - 7 * 60_000),
});
}
};
// Hourly pulse for 400 days: more than the feature extraction's 6000-row
// all-type cap on its own, so the cap decides which weight rows the core
// aggregates see, in both builds.
add("PULSE", "bpm", 400 * 24, HOUR_MS, (i) => 60 + ((i * 7) % 25));
add("RESTING_HEART_RATE", "bpm", 400, DAY_MS, (i) => 52 + (i % 9));
add("HEART_RATE_VARIABILITY", "ms", 400, DAY_MS, (i) => 40 + (i % 13));
add(
"BLOOD_PRESSURE_SYS",
"mmHg",
800,
12 * HOUR_MS,
(i) => 115 + ((i * 7) % 22),
"WITHINGS",
);
add(
"BLOOD_PRESSURE_DIA",
"mmHg",
800,
12 * HOUR_MS,
(i) => 74 + ((i * 5) % 13),
"WITHINGS",
);
add("WEIGHT", "kg", 400, DAY_MS, (i) => 81 + (i % 6) * 0.3, "WITHINGS");
add("BODY_FAT", "%", 120, 3 * DAY_MS, (i) => 20 + (i % 5) * 0.2, "WITHINGS");
for (let d = 1; d <= 120; d++) {
rows.push({
userId,
type: "ACTIVITY_STEPS",
value: 6000 + ((d * 977) % 5000),
unit: "steps",
source: "APPLE_HEALTH",
measuredAt: new Date(now - d * DAY_MS),
externalId: `stats:HKQuantityTypeIdentifierStepCount:${d}`,
});
const stages: SleepStage[] = ["CORE", "DEEP", "CORE", "REM", "AWAKE"];
stages.forEach((sleepStage, s) => {
rows.push({
userId,
type: "SLEEP_DURATION",
value: 45,
unit: "minutes",
source: "APPLE_HEALTH",
measuredAt: new Date(now - d * DAY_MS - 6 * HOUR_MS + s * 50 * 60_000),
sleepStage,
});
});
}
await prisma.measurement.createMany({ data: rows });
const moods = Array.from({ length: 90 }, (_, d) => {
const at = new Date(now - d * DAY_MS - 2 * HOUR_MS);
return {
userId,
date: at.toISOString().slice(0, 10),
mood: "GUT",
score: 2 + (d % 4),
moodLoggedAt: at,
};
});
await prisma.moodEntry.createMany({ data: moods });
await recomputeUserRollups(userId, {
from: new Date(now - 420 * DAY_MS),
to: new Date(now),
});
// The mood feature warms its rollup tier fire-and-forget on first read;
// settle it now so both builds read the same tier.
await ensureUserMoodRollupsFresh(userId);
// No ANALYZE here, on purpose: a fresh import below the autoanalyze
// threshold is what production runs too, and the all-time pulse aggregate
// must not depend on planner statistics
// (`day-weight-unanalysed-scale.test.ts`).
}, 120_000);
describe("get_metric_series built from one source's rows", () => {
it.each(PRESENT.flatMap((m) => WINDOWS.map((w) => [m, w] as const)))(
"%s over %s answers exactly what the full snapshot answered",
async (metric, window) => {
const before = await oldPathResult(metric, window);
expect(before.result).toBeDefined();
const after = await newPathResult(metric, window);
expect(JSON.stringify(after)).toBe(JSON.stringify(before.result));
},
);
it("sends a section large enough for the budget pass to the full builder", async () => {
// Without a case past the threshold the fallback branch is never run and
// the equality above would say nothing about it.
const sizes = await Promise.all(
(["bp", "pulse", "weight"] as const).map(async (m) => {
const { section } = await oldPathResult(m, "allTime");
return JSON.stringify({ section }, null, 2).length;
}),
);
expect(Math.max(...sizes)).toBeGreaterThan(SOURCE_SNAPSHOT_FALLBACK_CHARS);
});
it("keeps an absent source an explicit absence", async () => {
for (const metric of ["vo2_max", "spo2", "walking_speed"] as const) {
expect((await oldPathResult(metric, undefined)).result).toBeUndefined();
const after = (await newPathResult(metric, undefined)) as {
present: boolean;
reason?: string;
};
expect(after.present).toBe(false);
expect(after.reason).toBe("no_data");
}
});
it("answers get_metrics and the metric resource with the same per-metric results", async () => {
const metrics = ["resting_hr", "pulse", "bp", "weight", "vo2_max"] as const;
__resetCoachSnapshotCacheForTests();
const batch = (await tool("get_metrics").run(ctx(), {
metrics: [...metrics],
})) as {
present: boolean;
results: Array<{ metric: string; present: boolean; data?: unknown }>;
};
expect(batch.present).toBe(true);
for (const entry of batch.results) {
const metric = entry.metric as CoachScopeSource;
const { result } = await oldPathResult(metric, undefined);
if (!result) {
expect(entry.present).toBe(false);
continue;
}
expect(JSON.stringify(entry)).toBe(
JSON.stringify({
metric,
present: true,
data: result.data,
...(result.grounding ? { grounding: result.grounding } : {}),
}),
);
}
const resource = MCP_RESOURCE_TEMPLATES.find((r) => r.name === "metric")!;
__resetCoachSnapshotCacheForTests();
const viaResource = await resource.read(ctx(), { type: "resting_hr" });
expect(JSON.stringify(viaResource)).toBe(
JSON.stringify((await oldPathResult("resting_hr", undefined)).result),
);
});
});