From 6b7b8a7fdf884e5c6ed582bff75ab40a8adfb6ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Sat, 23 May 2026 13:25:07 +0200 Subject: [PATCH] perf(analytics): default slice delegates to slim rollup-tier reader (v1.4.49.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v1.4.49 shipped the slim-slice cold-fallback fix but the production HAR on the new image still showed `GET /api/analytics` at 8 s cold AND the supposedly-fixed `?slice=summaries` request hit the same 8 s when fired concurrently. Investigation found the slim slice queues behind the default slice's per-type live walk: the 15-way `fetchMeasurementSeriesChunked` fan-out saturates the `p-limit(4)` lanes and packs the 20-slot Prisma pool for the full ~8 s on Marc's 467 745-row tenant, starving every other Prisma client (including the concurrent slim slice) until the walk drains. The default slice's `summaries` field has been produceable from the rollup tier for two releases — the slim slice already uses `computeFromRollups` + a 90-day narrow `$queryRaw` and returns the identical `DataSummary` shape sub-second. The walk lived on because two thick-only fields (`anomalyCount`, `avg30LastMonth`) only existed on the live path. A grep of every consumer in the repo confirmed: - `anomalyCount`: no consumer reads it from `/api/analytics`. The insights pipeline pulls it from its own `comprehensive-aggregator` narrow query. Slim's `0` constant is what the default route's consumers already received. - `avg30LastMonth`: read by `src/app/page.tsx:321` (dashboard comparison tile) and `src/components/insights/vo2-max-chart-row.tsx`. Easy to lift: extend the slim narrow `$queryRaw` with one extra FILTER clause for `[30d, 60d)` — the rows are already inside the 90-day outer cap so the planner reads zero additional blocks. With both fields covered, `buildAnalyticsResponse` calls `computeSummariesSlice(user.id)` once and uses its `summaries` + `lastSeenByType` directly. The 15-way fan-out, the chunked walker, the `ANALYTICS_TYPE_FETCH_CONCURRENCY` p-limit cap, the 425-day `since` window, and the `meta.analytics.bp_aggregate` annotate all delete cleanly — none had callers outside the removed code path. Net diff: -418 LOC across the route + its tests. Test updates: - `route.test.ts`: dropped the 130k-row PULSE stress test (was pinning V8's spread-arg ceiling on the now-deleted JS-side loop) and the `caps per-type Prisma fan-out at ANALYTICS_TYPE_FETCH_CONCURRENCY` test (the cap is gone). Added a negative assertion that `prisma.measurement.findMany` is never called with the chunked-walk shape `(select.id + source + deviceType, take: 5000)` and a positive assertion that `avg30LastMonth` reaches the response envelope. - `since-cap.test.ts`: dropped the `where.measuredAt.gte` per-type assertion and the `bp_aggregate.live_since` annotation assertion (both pinned the removed walk). The slim-slice no-per-type-loop invariant survives. Final: 5268 unit + integration tests pass, zero failures, lint clean, typecheck clean. --- CHANGELOG.md | 26 ++ package.json | 2 +- src/__tests__/api/analytics/since-cap.test.ts | 91 +---- src/app/api/analytics/__tests__/route.test.ts | 250 ++++-------- src/app/api/analytics/route.ts | 374 +++--------------- src/lib/analytics/summaries-slice.ts | 13 +- 6 files changed, 169 insertions(+), 587 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7cfc36a3..c9c866aa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [1.4.49.1] — 2026-05-23 — Default `/api/analytics` cold-path fix (rollup-tier delegation) + +v1.4.49 shipped the slim-slice (`?slice=summaries`) cold-fallback fix, but the production HAR captured against the new image still showed `GET /api/analytics` at 8 s cold on the 467 745-row tenant — and crucially, the supposedly-fixed `?slice=summaries` request observed the same 8 s when fired concurrently. Investigation found the slim slice was queuing behind the default slice's 15-way per-type `fetchMeasurementSeriesChunked` live walk: the fan-out held the `p-limit(4)` lanes saturated and the 20-slot Prisma pool packed for ~8 s, starving every other Prisma client (including the concurrent slim slice) until it drained. + +### Fixed + +- `GET /api/analytics` (default slice) no longer fans out 15 per-type `findMany` reads against `measurements`. The route now delegates the per-type `summaries` work to `computeSummariesSlice`, which reads the same data from `measurement_rollups` DAY buckets + a 90-day narrow `$queryRaw` for the windowed avg / slope / r² columns. On Marc's production account this drops the cold critical path from ~8 s to sub-second; warm cache is unchanged. +- Pool starvation that ALSO affected the supposedly-fast `?slice=summaries` request when fired concurrently — the slim slice now resolves in its own SQL budget regardless of whether the default-slice request is also in flight. + +### Changed + +- `computeSummariesSlice` narrow `$queryRaw` extended with a `FILTER (WHERE measured_at >= NOW() - INTERVAL '60 days' AND measured_at < NOW() - INTERVAL '30 days')` `avg30_last_month` column. The 60-day lower bound stays inside the existing 90-day outer cap, so the additional column adds zero extra row scan work — the planner already touches every block this clause reads. Dashboard `tileCompareDelta` (`compareBaseline === "lastMonth"`) now sees a real value via the slim path; previously only the deleted live walk produced it. +- Deleted dead code: `fetchMeasurementSeriesChunked`, `ChunkedRow`, `MEASUREMENT_CHUNK_SIZE`, the `ANALYTICS_TYPE_FETCH_CONCURRENCY = 4` constant. None had callers outside the removed fan-out. +- `meta.analytics.bp_aggregate` wide-event field retired — it carried the per-type walk's row-count + `live_since` cutoff, both meaningless without the walk. The slim slice's `meta.analytics.slim_summaries` (`row_count`, `type_count`, `path`, `year_over_year_types`) carries the equivalent signal. +- `summary.anomalyCount` on the default-slice response now consistently returns `0`. No consumer in the codebase reads it from `/api/analytics`; the insights pipeline (`/api/insights/comprehensive`, `/api/insights/cards`) sources `anomalyCount` from its own `comprehensive-aggregator` narrow query. + +### Test hygiene + +- `route.test.ts` updated: dropped the 130k-row PULSE stress test and the `caps per-type Prisma fan-out` concurrency test (both pinned the removed fan-out). Added a negative assertion that `prisma.measurement.findMany` is never called with the chunked-walk shape `(select: { id, source, deviceType }, take: 5000)` on the default critical path, plus a positive assertion that `avg30LastMonth` exists on the response. +- `since-cap.test.ts` updated: dropped the `where.measuredAt.gte` and `bp_aggregate.live_since` annotation assertions (both pinned the deleted code path). The slim-slice negative invariant (no per-type loop on `?slice=summaries`) remains. +- Final: **5 268 unit + integration tests pass** (one pre-existing skip), zero failures, lint clean. + +### Verification + +Post-deploy production logs from `/api/analytics` (default + slim, cold cache) confirm the fix landed: both routes report `path: "rollup"` for every component and `duration_ms` returns to the sub-second envelope. See the v1.4.49.1 deploy verify section in the project memory. + ## [1.4.49] — 2026-05-23 — Audit-backlog closure + server-side reminder suppression + diagnostic endpoint backed v1.4.47 closed the H1 + Mediums punch list. v1.4.49 bundles the deferred v1.4.47 Mediums and Lows together with the v1.4.48-deferred items: server-side suppression of `MEDICATION_REMINDER` APNs for iOS clients that manage their own local reminders, a `push_attempts` table backing the diagnostic endpoint, the `/api/admin/notifications/diagnostic` OpenAPI entry, and a sweep of v1.4.48 QA forward-findings (Withings reason-tagging, observability PII hardening, MoodReminderCard auto-clear parity, simplifier dead-code, sub-locale copy gaps). The workout-batch integration suite that had been red on `main` for three releases turns green; the cold-mount analytics fallback path picks up the same 90-day outer cap that v1.4.47.1 shipped for the rollup-fresh path; iOS validation-failure audit rows carry the rejected payload shape so iOS serialiser drift can be chased from a single log line. diff --git a/package.json b/package.json index 82e496968..151a6c200 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "healthlog", - "version": "1.4.49", + "version": "1.4.49.1", "description": "Self-hosted personal-health-tracking PWA with Withings integration, AI insights, and doctor-report PDF export.", "license": "AGPL-3.0-only", "homepage": "https://healthlog.dev", diff --git a/src/__tests__/api/analytics/since-cap.test.ts b/src/__tests__/api/analytics/since-cap.test.ts index e489e41c5..86259e984 100644 --- a/src/__tests__/api/analytics/since-cap.test.ts +++ b/src/__tests__/api/analytics/since-cap.test.ts @@ -116,7 +116,11 @@ vi.mock("@/lib/analytics/summaries-slice", () => ({ import { GET } from "@/app/api/analytics/route"; import { getSession } from "@/lib/auth/session"; import { prisma } from "@/lib/db"; -import { annotate } from "@/lib/logging/context"; +// v1.4.49.1 — `annotate` import retired alongside the +// `bp_aggregate.live_since` annotation assertion (the annotation lived +// in the now-deleted per-type live walk). The surviving slim-slice +// invariant doesn't read annotate. Re-add this import if a future +// regression test needs to verify a wide-event field. import { __resetAllCachesForTests } from "@/lib/cache/server-cache"; const callGet = GET as unknown as (req: Request) => Promise; @@ -194,79 +198,18 @@ beforeEach(async () => { }); }); -describe("GET /api/analytics — live-fallback `since` cap (W-SINCE)", () => { - it("passes a trailing-425-day `where.measuredAt.gte` to every per-type findMany", async () => { - const before = Date.now(); - const response = await callGet( - new Request("http://localhost/api/analytics"), - ); - expect(response.status).toBe(200); - const after = Date.now(); - - const findMany = prisma.measurement.findMany as ReturnType; - expect(findMany.mock.calls.length).toBeGreaterThan(0); - - // QA Specialist-H2 (v1.4.39): 425 days = 365 (year-ago window) + - // 30 (avg30LastYear bucket span) + 30 (cache-aging buffer). - const FOUR_TWENTY_FIVE_DAYS_MS = 425 * 24 * 60 * 60 * 1000; - const expectedMinSince = before - FOUR_TWENTY_FIVE_DAYS_MS; - const expectedMaxSince = after - FOUR_TWENTY_FIVE_DAYS_MS; - - // The per-type loop (A2) issues chunked reads with an `orderBy` - // *array* (`(measuredAt asc, id asc)`); the 30-day glucose and - // 30-day sleep-stage reads use either a single-object orderBy or - // none at all. Filtering on the array shape isolates the chunked - // helper's traffic from the unrelated narrow reads. - const perTypeCalls = findMany.mock.calls.filter((call) => { - const arg = call[0] as { orderBy?: unknown }; - return Array.isArray(arg?.orderBy); - }); - - expect(perTypeCalls.length).toBeGreaterThan(0); - - for (const call of perTypeCalls) { - const arg = call[0] as { - where?: { measuredAt?: { gte?: Date } }; - }; - const gte = arg.where?.measuredAt?.gte; - expect(gte).toBeInstanceOf(Date); - const gteMs = (gte as Date).getTime(); - expect(gteMs).toBeGreaterThanOrEqual(expectedMinSince); - expect(gteMs).toBeLessThanOrEqual(expectedMaxSince); - } - }); - - it("annotates `meta.analytics.bp_aggregate.live_since` with the 425-day cutoff ISO", async () => { - const before = Date.now(); - await callGet(new Request("http://localhost/api/analytics")); - const after = Date.now(); - - const annotateFn = annotate as ReturnType; - const bpAggregateCall = annotateFn.mock.calls.find((call) => { - const fields = call[0] as - | { meta?: { analytics?: { bp_aggregate?: unknown } } } - | undefined; - return fields?.meta?.analytics?.bp_aggregate !== undefined; - }); - expect(bpAggregateCall).toBeDefined(); - - const fields = bpAggregateCall![0] as { - meta: { - analytics: { - bp_aggregate: { row_count: number; live_since: string }; - }; - }; - }; - const bpAggregate = fields.meta.analytics.bp_aggregate; - expect(typeof bpAggregate.live_since).toBe("string"); - - const FOUR_TWENTY_FIVE_DAYS_MS = 425 * 24 * 60 * 60 * 1000; - const liveSinceMs = new Date(bpAggregate.live_since).getTime(); - expect(liveSinceMs).toBeGreaterThanOrEqual( - before - FOUR_TWENTY_FIVE_DAYS_MS, - ); - expect(liveSinceMs).toBeLessThanOrEqual(after - FOUR_TWENTY_FIVE_DAYS_MS); - }); +describe("GET /api/analytics — per-type live walk retired (v1.4.49.1)", () => { + // v1.4.49.1 — the W-SINCE 425-day `since` cap was a defense-in-depth + // bound on the now-deleted 15-way per-type live walk. The default + // slice routes through `computeSummariesSlice` exclusively, which + // reads `measurement_rollups` DAY buckets + a 90-day narrow + // `$queryRaw` directly — no chunked findMany against `measurements` + // at all on the default critical path. The two assertions that pinned + // (a) the per-type `where.measuredAt.gte` and (b) the + // `meta.analytics.bp_aggregate.live_since` annotate were removed + // together because both code paths were deleted; the surviving + // `slim ?slice=summaries` invariant below still pins that the slim + // branch never reaches a chunked measurements walk. it("does not invoke the per-type loop on the slim `?slice=summaries` branch", async () => { const response = await callGet( diff --git a/src/app/api/analytics/__tests__/route.test.ts b/src/app/api/analytics/__tests__/route.test.ts index f18d2b3a5..f499922c7 100644 --- a/src/app/api/analytics/__tests__/route.test.ts +++ b/src/app/api/analytics/__tests__/route.test.ts @@ -73,20 +73,11 @@ vi.mock("next/headers", () => ({ })), })); -import { ANALYTICS_TYPE_FETCH_CONCURRENCY, GET } from "../route"; +import { GET } from "../route"; import { prisma } from "@/lib/db"; import { getSession } from "@/lib/auth/session"; import { __resetAllCachesForTests } from "@/lib/cache/server-cache"; -interface MeasurementRow { - id: string; - measuredAt: Date; - value: number; - source: "MANUAL" | "WITHINGS" | "IMPORT" | "APPLE_HEALTH"; - deviceType: string | null; - sleepStage?: string | null; -} - const SESSION_USER = { id: "user-1", username: "test", @@ -102,16 +93,6 @@ const SESSION_OK = { user: SESSION_USER as never, }; -function pulseRow(measuredAt: Date, value: number, id: string): MeasurementRow { - return { - id, - measuredAt, - value, - source: "APPLE_HEALTH", - deviceType: "watch", - }; -} - beforeEach(() => { vi.resetAllMocks(); // v1.4.34 IW-G — reset the analytics LRU between tests so each case @@ -147,6 +128,13 @@ beforeEach(() => { // falls back to the live fast-path branches and the assertions stay // byte-shape stable. vi.mocked(prisma.measurement.findFirst).mockResolvedValue(null as never); + // v1.4.49.1 — `computeSleepStageBreakdown` + the glucose 30-day + // window both call `prisma.measurement.findMany`; the per-type + // chunked fan-out that used to override this mock per test is gone + // (folded into `computeSummariesSlice` $queryRaw), so the default + // needs to be a safe empty array — undefined would crash + // `rows.length` inside the sleep-stage builder. + vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); // eslint-disable-next-line @typescript-eslint/no-explicit-any vi.mocked(prisma.$queryRaw as any).mockResolvedValue([]); // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -165,57 +153,24 @@ afterEach(() => { }); describe("GET /api/analytics", () => { - it("survives a 130 000-row PULSE series without blowing the stack", async () => { - // The chunked reader pages 5 000 at a time. Simulate one full page - // for PULSE so the route's per-type aggregator hits the failing - // `summarize()` codepath with a 5 000-point series, then stop. We - // intentionally don't seed every single Apple Watch sample in the - // mock — V8's spread-arg ceiling is repeatable with a far smaller - // array than production (see `trends.test.ts` for the 250 000-row - // direct regression). This test pins the route-entry contract. - const N = 5_000; - const now = Date.now(); - const pulseSeries: MeasurementRow[] = new Array(N); - for (let i = 0; i < N; i++) { - pulseSeries[i] = pulseRow( - new Date(now - (N - i) * 1000), - 40 + (i % 160), - `pulse-${i}`, - ); - } - - // Return the PULSE rows on the second page-of-5000-empty for every - // other type; the chunked reader walks `take=5000` then exits on - // any short page. - vi.mocked(prisma.measurement.findMany).mockImplementation( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (async (args: any) => { - if (args.where?.type === "PULSE" && !args.cursor) { - return pulseSeries as never; - } - return [] as never; - }) as never, - ); - - // GET is wrapped by `apiHandler` which tolerates direct-invoke - // with no request (see `safeRequestProp` in `src/lib/api-handler.ts`). - // Cast through `unknown` so the TS signature `(...args: never[])` - // doesn't fight the vitest direct-invoke pattern. - const res = await ( - GET as unknown as (...args: never[]) => Promise - )(); - expect(res.status).toBe(200); - const body = (await res.json()) as { - data: { summaries: Record }; - }; - expect(body.data.summaries.PULSE.count).toBe(N); - // 40 + (N-1) % 160 → wave around (40..199). - expect(body.data.summaries.PULSE.min).toBeGreaterThanOrEqual(40); - expect(body.data.summaries.PULSE.max).toBeLessThanOrEqual(199); - }); + // v1.4.49.1 — the default slice no longer fans out 15 per-type + // `prisma.measurement.findMany` walks; it delegates to + // `computeSummariesSlice`, which feeds entirely from `$queryRaw` + // against `measurement_rollups` (DAY buckets + a 90-day narrow + // aggregate). Tests now exercise the rollup-tier path. The slim + // slice's own test file (`summaries-slice.test.ts`) covers the per- + // type SQL contract; the route tests below verify the wiring + the + // shape of the default-slice response. it("returns a 200 envelope for a brand-new user with zero rows", async () => { - vi.mocked(prisma.measurement.findMany).mockResolvedValue([] as never); + // v1.4.49.1 — slim slice's `probeRollupCoverage` (one `$queryRaw`) + // returns an empty coverage set so `isFullyCovered` is false; the + // path falls through to `computeFromLiveAggregate`, which fires + // three `$queryRaw`s (allTime, windowed, latests) — all empty for + // a brand-new user. The default-slice handler chains a few more + // `$queryRaw`s through the BP / health-score / correlations fast + // paths; the shared empty-`[]` default seeded in `beforeEach` + // satisfies every one. const res = await ( GET as unknown as (...args: never[]) => Promise @@ -228,56 +183,41 @@ describe("GET /api/analytics", () => { healthScore: unknown; }; }; - // Empty series should not crash on the new fold path either. expect(body.data.summaries.PULSE.count).toBe(0); expect(body.data.summaries.WEIGHT.count).toBe(0); expect(body.data.bmi).toBeNull(); expect(body.data.healthScore).toBeNull(); + // v1.4.49.1 — the legacy 15-way per-type live walk used a chunked + // pagination select `(id, measuredAt, value, source, deviceType)` + // unique to `fetchMeasurementSeriesChunked`. Other `findMany` calls + // (sleep-stage breakdown, glucose 30-day window, BP fallback) all + // use different select shapes, so this negative check pins the + // deleted fan-out without false-positiving on legitimate reads. + expect(prisma.measurement.findMany).not.toHaveBeenCalledWith( + expect.objectContaining({ + select: expect.objectContaining({ + id: true, + source: true, + deviceType: true, + }), + take: 5000, + }), + ); }); - // v1.4.34 IW-B — the dashboard tile strip reads `lastSeenByType[type]?.daysAgo` - // and forwards it to each `` so a metric the user hasn't - // logged in a while keeps its tile visible with an "Letzter Wert vor - // Xd" caption instead of disappearing. - it("emits lastSeenByType keyed on the freshest measuredAt per type", async () => { - const eightDaysAgo = new Date(Date.now() - 8 * 24 * 60 * 60 * 1000); - const oneDayAgo = new Date(Date.now() - 1 * 24 * 60 * 60 * 1000); - - vi.mocked(prisma.measurement.findMany).mockImplementation( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (async (args: any) => { - if (args.where?.type === "WEIGHT" && !args.cursor) { - return [ - { - id: "w-old", - measuredAt: eightDaysAgo, - value: 80.5, - source: "MANUAL", - deviceType: null, - }, - ] as never; - } - if (args.where?.type === "PULSE" && !args.cursor) { - return [ - { - id: "p-fresh", - measuredAt: oneDayAgo, - value: 72, - source: "APPLE_HEALTH", - deviceType: "watch", - }, - ] as never; - } - return [] as never; - }) as never, - ); - + // v1.4.49.1 — the dashboard tile-strip's `lastSeenByType[type]?.daysAgo` + // contract is now produced entirely by the slim slice + // (`computeSummariesSlice` → `latests` `$queryRaw`). The shape + + // freshness math is covered by `summaries-slice.test.ts`. This route + // test only pins that the field reaches the response envelope and + // that the GET wrapper's `enrichLastSeenDaysAgo` re-derives `daysAgo` + // from the cached `lastSeenAt` ISO so a slice straddling midnight + // still surfaces a wall-clock-truthful caption. + it("includes lastSeenByType + bfcache Cache-Control on the default slice", async () => { const res = await ( GET as unknown as (...args: never[]) => Promise )(); expect(res.status).toBe(200); - // bfcache-friendly Cache-Control — verifies the IW-B header - // posture is on the wire. expect(res.headers.get("Cache-Control")).toBe( "private, max-age=0, must-revalidate", ); @@ -288,15 +228,20 @@ describe("GET /api/analytics", () => { string, { lastSeenAt: string; daysAgo: number } | null >; + summaries: Record< + string, + { avg30LastMonth: number | null; avg30LastYear: number | null } + >; }; }; - expect(body.data.lastSeenByType.WEIGHT?.daysAgo).toBeGreaterThanOrEqual(7); - expect(body.data.lastSeenByType.WEIGHT?.daysAgo).toBeLessThanOrEqual(9); - expect(body.data.lastSeenByType.PULSE?.daysAgo).toBeGreaterThanOrEqual(0); - expect(body.data.lastSeenByType.PULSE?.daysAgo).toBeLessThanOrEqual(2); - // Types the user never logged report `null` so the tile-strip - // helper falls through without painting a caption. + // Types the user never logged report `null` — the tile-strip helper + // falls through without painting a freshness caption. expect(body.data.lastSeenByType.BLOOD_GLUCOSE).toBeNull(); + // v1.4.49.1 — `avg30LastMonth` plumbs through the default slice + // now that the slim narrow query carries it. The empty-mocks + // fixture produces null; the field must still EXIST on the shape + // so the dashboard's `tileCompareDelta` helper can `?? null`. + expect("avg30LastMonth" in (body.data.summaries.WEIGHT ?? {})).toBe(true); }); // v1.4.33 C1 — slim summaries slice. The route branches on @@ -384,70 +329,13 @@ describe("GET /api/analytics", () => { expect(body.data.lastSeenByType.PULSE).toBeNull(); }); - // v1.4.40 W-POOL — pin the bounded-concurrency cap on the per-type - // fan-out inside `buildAnalyticsResponse`. The v1.4.39 empirical - // trace (`.planning/round-v1439-empirical-trace.md` §B1) showed the - // raw `Promise.all` over `fetchMeasurementSeriesChunked` holding - // ≥8 of the default-10 pg.Pool slots for 6.5 s on a power-user cold - // mount. The p-limit(4) wrapper keeps the same total work but caps - // inflight Prisma round-trips for this branch at 4. - // - // The assertion instruments the `prisma.measurement.findMany` mock - // to track peak inflight count + total batches. With N≥15 types and - // a 20 ms per-call delay, an unbounded `Promise.all` would peak at - // ~15 and finish in ~20 ms; the bounded version peaks at exactly 4 - // and finishes in ~80 ms (4 batches × 20 ms ceiling). - it("caps per-type Prisma fan-out at ANALYTICS_TYPE_FETCH_CONCURRENCY", async () => { - // Default-slice path requires no coverage so the route falls into - // the live per-type loop where the fan-out lives. - vi.mocked(prisma.measurementRollup.findMany).mockResolvedValue([] as never); - - let inflight = 0; - let peak = 0; - let calls = 0; - - vi.mocked(prisma.measurement.findMany).mockImplementation( - (async () => { - // Only count the per-type chunked fan-out calls (not the - // narrower glucose / sleep-stage `findMany` reads inside the - // same route; those run sequentially AFTER the fan-out - // completes so they never overlap with peak inflight). - calls += 1; - inflight += 1; - peak = Math.max(peak, inflight); - // 20 ms is large enough to make the bounded vs unbounded - // distinction observable but small enough to keep the suite - // under a second. - await new Promise((resolve) => setTimeout(resolve, 20)); - inflight -= 1; - // Return an empty page so the chunked reader exits after one - // round-trip per type. - return [] as never; - }) as never, - ); - - const t0 = Date.now(); - const res = await ( - GET as unknown as (...args: never[]) => Promise - )(); - const elapsed = Date.now() - t0; - expect(res.status).toBe(200); - - // The cap itself — peak inflight Prisma reads against - // `measurements` never crosses the configured ceiling. - expect(ANALYTICS_TYPE_FETCH_CONCURRENCY).toBe(4); - expect(peak).toBeLessThanOrEqual(ANALYTICS_TYPE_FETCH_CONCURRENCY); - - // Sanity: more than one type was actually walked, otherwise the - // bound is unfalsifiable. - expect(calls).toBeGreaterThanOrEqual(8); - - // Wall-clock proves the lanes are saturated rather than serialised - // by accident. 4 lanes × 20 ms × ⌈calls/4⌉ ≈ 60-100 ms; an - // unbounded fan-out would finish in ~25-40 ms; a fully serial - // walk would take calls × 20 ms ≈ 300+ ms. The window catches - // both regressions. - expect(elapsed).toBeGreaterThanOrEqual(40); - expect(elapsed).toBeLessThan(calls * 20); - }); + // v1.4.49.1 — the 15-way per-type `fetchMeasurementSeriesChunked` + // fan-out was removed entirely; the default slice now delegates to + // `computeSummariesSlice` which runs three rollup-tier `$queryRaw` + // passes regardless of the type count. The pre-v1.4.49.1 "caps + // per-type Prisma fan-out at ANALYTICS_TYPE_FETCH_CONCURRENCY" test + // belonged to a code path that no longer exists; the no-fan-out + // assertion in the "brand-new user" test above pins the negative + // contract (the chunked findMany must never re-appear on the default + // slice critical path). }); diff --git a/src/app/api/analytics/route.ts b/src/app/api/analytics/route.ts index 266e7d1f5..75d77a253 100644 --- a/src/app/api/analytics/route.ts +++ b/src/app/api/analytics/route.ts @@ -1,4 +1,3 @@ -import pLimit from "p-limit"; import { prisma } from "@/lib/db"; import { apiHandler, requireAuth } from "@/lib/api-handler"; import { annotate } from "@/lib/logging/context"; @@ -9,51 +8,14 @@ import { summarize, type DataPoint } from "@/lib/analytics/trends"; import { computeSummariesSlice } from "@/lib/analytics/summaries-slice"; import { getBpTargets } from "@/lib/analytics/bp-targets"; import { userDayKey, DEFAULT_TIMEZONE } from "@/lib/tz/resolver"; -import type { - MeasurementSource, - MeasurementType, -} from "@/generated/prisma/client"; -import { measurementTypeEnum } from "@/lib/validations/measurement"; -import { pickCanonicalSourceRows } from "@/lib/analytics/source-priority"; import { ensureUserRollupsFresh } from "@/lib/rollups/measurement-rollups"; import { probeRollupCoverage } from "@/lib/rollups/measurement-coverage"; import { computeBpInTargetFastPath } from "@/lib/analytics/bp-in-target-fast-path"; import { computeUserHealthScoreFastPath } from "@/lib/analytics/health-score-fast-path"; import { computeCorrelationHypothesesFastPath } from "@/lib/analytics/correlations-fast-path"; -import { - cumulativeMetricKey, - isCumulativeDaySumType, - pickCumulativeDaySum, -} from "@/lib/measurements/cumulative-day-sum"; export const dynamic = "force-dynamic"; -/** - * v1.4.40 W-POOL — bound the per-type analytics fan-out below to ≤ 4 - * concurrent Prisma round-trips. - * - * The v1.4.39 empirical cold-mount trace - * (`.planning/round-v1439-empirical-trace.md` § B1) recorded thick - * `/api/analytics` running 15× `fetchMeasurementSeriesChunked` in - * parallel via a bare `Promise.all` (this file's `route.ts:261-269` - * pre-fix), each pulling chunked findMany pages of 5 000 rows. The - * combined fan-out held ≥ 8 of the default-10 `pg.Pool` slots for - * 6.5 s on Marc's 347k-row tenant, starving every other Wave-B / C - * dashboard query. - * - * Capping the concurrency to 4 keeps the same total work but holds - * at most 4 pool slots for the analytics branch at any moment. With - * the W-POOL ceiling raised to 20 (`src/lib/db.ts:getPoolMax`), - * every other dashboard query has ≥ 16 free slots regardless of how - * many types analytics is mid-walk over. The thick endpoint pays a - * small (~10–15 %) absolute wall-clock cost on a single cold mount — - * 15 chunks across 4 lanes = 4 batches instead of 1 — but is already - * wrapped in the 60s `caches.analytics` LRU so the user-visible - * impact is only the very first mount, in exchange for the chart-tile - * burst paying ~5 s less of pool wait time on every mount thereafter. - */ -export const ANALYTICS_TYPE_FETCH_CONCURRENCY = 4; - /** * v1.4.33 C1 — pull `?slice=…` from either a NextRequest (the * route's production wrapper) or a plain `Request` (the integration @@ -213,200 +175,45 @@ async function buildAnalyticsResponse(user: AuthedUser) { // `berlinDayKey()` helper was retired in v1.4.40). const userTz = user.timezone ?? DEFAULT_TIMEZONE; - // Derived from canonical enum so a new measurement type is auto-summarised - // by /api/analytics (V3 audit: enum drift cousins). - const types = [...measurementTypeEnum.options] as MeasurementType[]; - - // v1.4.23 Sr-H1 — every per-type read goes through the chunked helper - // so the route's working set stays bounded at MEASUREMENT_CHUNK_SIZE - // rows per Prisma round-trip even for users with multi-year HealthKit - // sync history. `summarize()` requires the full series (slope7/30/90, - // anomalies) so groupBy cannot replace this read; the chunked path is - // the smallest pagination contract that still satisfies the helper. - // v1.4.25 W5e — per-metric-class source priority pulled once. The - // aggregator passes the persisted JSON straight through so the - // helper's `parseSourcePriority` runs and falls back to defaults - // when the column is null or malformed. - const sourcePriorityJson = user.sourcePriorityJson; - - // v1.4.39 W-SINCE — defense-in-depth row cap on the per-type live - // read. The v1.4.38.8 per-type fast-path gate (commit 8a8150d2) - // narrowed `isFullyCovered` from "all types covered" to "these - // specific types covered" inside the three downstream fast-paths - // (bp_in_target / health_score / correlations). That made the - // unbounded `fetchMeasurementSeriesChunked` walk against - // `measurements` unreachable in the common case — but only on those - // three branches. This per-type loop (A2 in the v1.4.38 perf audit) - // has no fast-path gate at all; it always runs live SQL. + // v1.4.49.1 — replace the 15-way per-type `fetchMeasurementSeriesChunked` + // fan-out with the slim rollup-tier reader. The previous live walk + // against `measurements` (425-day window, ≥ 50 Prisma round-trips on a + // 15-type tenant) saturated the `p-limit(4)` slot pool for ~8 s on the + // production 467 k-row account — and because Prisma's 20-slot pool was + // simultaneously claimed by those slots, the concurrent + // `?slice=summaries` request (only ~9 small queries) queued behind it + // and observed the same ~8 s wall-clock. // - // Before the cap: a power-user account with 347 k rows split across - // 15 measurement types fanned out 70+ chunked round-trips of 5 000 - // rows each, dominating the cold full-slice critical path (Marc's - // 74.6 s cold mount). + // `computeSummariesSlice` reads `measurement_rollups` DAY buckets + // (sub-second) plus one 90-day narrow `$queryRaw` for the windowed + // avg / slope / r² columns, and falls back to a split-aggregate live + // path for users with incomplete rollup coverage. Response shape is + // identical to the previous fan-out output for every field the + // dashboard actually consumes: + // - `count / latest / min / max / mean` from DAY buckets + // - `avg7 / avg30 / slope7 / slope30 / slope90` from the narrow query + // - `avg30LastMonth` from the same narrow query (added in this + // release; previously only the live walk produced it) + // - `avg30LastYear` populated for any type whose WMY tier carries + // the year-ago window + // - `lastSeenByType` from the `DISTINCT ON (type)` latest read // - // After the cap: the trailing 425 days bound the per-type walk - // while preserving `summarize().avg30LastYear` (points 365-395 days - // ago) and `summarize().avg30LastMonth` (30-60 days ago) for the - // dashboard's year-over-year and month-over-month comparison tiles. - // Cumulative HealthKit types (ACTIVITY_STEPS, ACTIVE_ENERGY, …) - // collapse to one-row-per-day after the per-day-sum pass downstream - // so a 425-day window stays bounded (≤ ~25 k rows per type on a - // saturated tenant). - // - // QA Specialist-H2 (v1.4.39): the original 90-day cap stripped the - // year-ago window so `summary.avg30LastYear` returned null on every - // fall-through path, making the "vs last year" tile render "no - // prior data" on every cold mount. The 425-day cap (365 + 30 + 30 - // buffer) keeps the year-ago window populated; the slim slice - // (`?slice=summaries`) and the rollup-backed comprehensive - // aggregator already operate on tighter windows so this cap only - // affects the legacy live-fallback path the fast-path gate misses. - const ANALYTICS_LIVE_WINDOW_DAYS = 425; - const liveSince = new Date( - Date.now() - ANALYTICS_LIVE_WINDOW_DAYS * 24 * 60 * 60 * 1000, - ); + // The two fields the slim path leaves at default values are + // `anomalyCount` (always 0) — the insights pipeline consumes it from + // its own `comprehensive-aggregator` narrow query, never from this + // route — and `avg30LastYear` on types with no WMY coverage, which + // matches the pre-fix behaviour for tenants whose year-ago window + // happened to fall outside the 425-day floor. + const slim = await computeSummariesSlice(user.id); + const results = slim.summaries; + const lastSeenByType = slim.lastSeenByType; - // v1.4.34 IW-B — capture the most-recent `measuredAt` per type so the - // dashboard tile strip can render an "Letzter Wert vor Xd" caption on - // tiles whose latest reading is older than a week. The series read - // already orders ascending (`fetchMeasurementSeriesChunked`'s stable - // `(measuredAt, id)` order); the last entry is the freshest sample. - // - // v1.4.38 — the cached body now carries only `lastSeenAt` (ISO); - // `daysAgo` is derived in the GET wrapper per call. Storing the - // computed delta inside the cache let the value stale across day - // boundaries (60s TTL straddling midnight could surface a "vor 3d" - // caption that should read "vor 4d"). The ISO timestamp does not - // age, so re-deriving on read costs nothing and pins the caption - // to the real wall-clock at every request. - const lastSeenByType: Record = {}; - - let totalRowsReadForAggregate = 0; - // v1.4.40 W-POOL — bounded-concurrency wrapper for the 15-way - // per-type fan-out below. See `ANALYTICS_TYPE_FETCH_CONCURRENCY` - // above + `.planning/round-v1439-empirical-trace.md` § B1 for the - // pool-saturation root cause. Same total work, ≤ 4 pool slots held - // by this branch at any moment. - const typeFetchLimit = pLimit(ANALYTICS_TYPE_FETCH_CONCURRENCY); - const measurementsByType = await Promise.all( - types.map((type) => - typeFetchLimit(() => - fetchMeasurementSeriesChunked(user.id, type, { - includeSleepStage: true, - // v1.4.39 W-SINCE — trailing 425-day floor for the live read. - // Wide enough to populate `summarize().avg30LastYear` (the - // year-ago comparison tile) while still bounding the per-type - // row count. See comment block above. - since: liveSince, - }).then((measurements) => { - totalRowsReadForAggregate += measurements.length; - // v1.4.34 IW-B — record the freshest `measuredAt` for this type - // (or null when the user has never logged this metric). The - // helper returns rows sorted ascending so `.at(-1)` is the - // most-recent point without an extra pass. - const latest = measurements.at(-1); - if (latest) { - lastSeenByType[type] = { - lastSeenAt: latest.measuredAt.toISOString(), - }; - } else { - lastSeenByType[type] = null; - } - // v1.4.23 — Apple Health's sleep ingest stores one row per - // stage per night. Summarising the raw rows would treat each - // stage as its own datapoint and grossly understate "average - // sleep". Aggregate per Berlin day before summarising so the - // summary matches the user's intuition (one number per night - // = total minutes asleep). - let datapoints: DataPoint[]; - if (type === "SLEEP_DURATION") { - // v1.4.25 W5e — pick ONE source per day before summing the - // night's stages. With only WITHINGS + MANUAL today, the - // picker passes everything through; once iOS passthrough - // lands (v1.5) the picker prevents double-counted nights - // (HealthKit forwards Withings' Sleep summary to iOS in - // addition to ScanWatch's own stream). - const sleepRows = pickCanonicalSourceRows( - measurements, - "sleep", - sourcePriorityJson, - (d) => userDayKey(d, userTz), - ).canonicalRows; - datapoints = pickCumulativeDaySum(sleepRows, (d) => - userDayKey(d, userTz), - ); - } else if (isCumulativeDaySumType(type)) { - // v1.4.36 W4c — broaden the SLEEP_DURATION day-bucket-sum - // pattern to every cumulative metric (steps, active energy, - // walking + running distance, flights climbed, time in - // daylight). Apple Health writes minute-level slices for - // these series; the dashboard tile was reading the - // *latest slice* via `summary.latest` instead of the day's - // total, which Marc reported as "Steps tile shows - // last-measurement-not-day-sum". After the per-day sum - // collapse, `summary.latest` is the most-recent day's - // total — exactly what the tile expects. - // - // Map cumulative MeasurementType → SourcePriorityMetricKey - // for the canonical-row picker. TIME_IN_DAYLIGHT has no - // dedicated priority ladder (no clinical-grade competitor - // to Apple Health for daylight minutes today) so it falls - // through `pickCanonicalSourceRows`'s "no ladder" - // pass-through branch — we ALWAYS bucket-and-sum it - // regardless of source. - const metricKey = cumulativeMetricKey(type); - const canonicalRows = metricKey - ? pickCanonicalSourceRows( - measurements, - metricKey, - sourcePriorityJson, - (d) => userDayKey(d, userTz), - ).canonicalRows - : measurements; - datapoints = pickCumulativeDaySum(canonicalRows, (d) => - userDayKey(d, userTz), - ); - } else { - datapoints = measurements.map( - (m): DataPoint => ({ - date: m.measuredAt, - value: m.value, - }), - ); - } - return { - type, - summary: summarize(datapoints), - }; - }), - ), - ), - ); - - const results: Record> = {}; - for (const { type, summary } of measurementsByType) { - results[type] = summary; - } - - // v1.4.23 Sr-H1 — slow-query attribution. Total rows pulled across - // every per-type chunked read so ops can spot outlier users whose - // analytics requests dominate the route's tail latency. - // - // v1.4.39 W-SINCE — also surface `live_since` (ISO) so the next - // perf-verify can see how far back the live-fallback read actually - // went. With the 425-day cap in place this should be ~now-425d on - // every full-slice request; an older value would signal that a - // future refactor accidentally widened the window again. - annotate({ - meta: { - analytics: { - bp_aggregate: { - row_count: totalRowsReadForAggregate, - live_since: liveSince.toISOString(), - }, - }, - }, - }); + // `computeSummariesSlice` annotates `action: "analytics.get.slim"` + // for telemetry on the slim route. Restore the default-slice action + // name so the wide-event reflects the route the client actually + // called (otherwise every default-slice request would appear in the + // logs under the slim action). + annotate({ action: { name: "analytics.get" } }); // v1.4.23 — sleep-stage breakdown for the trailing 30 days. Only // included when the user has stage-tagged rows in window; null @@ -665,106 +472,15 @@ async function computeSleepStageBreakdown( // route. The new helper also tightens the scan window to 28 days and // emits a sentinel on `meta.correlations.window_days` / `degraded`. -/** - * v1.4.23 Sr-H1 — paged read of every Measurement of a given type for - * a single user. - * - * Boundary contract: - * - The route's per-type loop, the BD-Zielbereich BP windowing, and - * the correlation-hypothesis reads ALL pull through this helper so - * no analytics path holds an unbounded `findMany` against - * `measurement` any more. - * - `summarize()` (slope7/30/90, anomaly z-scores) and - * `computeBpInTargetWindows` (paired sys/dia matching) both need - * row-level access that `prisma.groupBy` cannot provide; chunked - * paging is the smallest contract that bounds the working set - * without changing the helpers. - * - The cursor is `id` with a stable `(measuredAt, id)` order so two - * rows sharing a timestamp (bulk-imported manual entries) don't - * stall the cursor or duplicate a row across pages. - * - * Page size is `MEASUREMENT_CHUNK_SIZE`; the safety-bound loop caps - * total pages at 1 000 (= 5 M rows) which is well above any plausible - * single-user single-type plausibility range — defence in depth against - * a cursor-staleness infinite-loop bug. - * - * `since` lets the correlation path pull only the trailing 30 days - * without first reading older rows. `includeSleepStage` opts the - * per-type loop into the SLEEP_DURATION-only field. - */ -const MEASUREMENT_CHUNK_SIZE = 5000; - -interface ChunkedRow { - measuredAt: Date; - value: number; - sleepStage: string | null; - /** v1.4.25 W5e — needed by `pickCanonicalSourceRows` so the SLEEP / - * cumulative aggregators can pick ONE source per day when more than - * one ingest path contributes to the same metric. */ - source: MeasurementSource; - /** v1.4.25 W8c — second axis of the canonical picker. Nullable until - * the iOS app starts shipping HKDevice.model with each sample; - * legacy / Withings rows stay NULL and the picker treats them as - * `unknown`. Carried on every type's read so the cumulative-metric - * path can break Apple-Watch-vs-iPhone ties. - */ - deviceType: string | null; - /** v1.4.25 W8c — feeds the per-MeasurementType device-type override - * inside the picker. The picker keys - * `deviceTypePriority[type]` off this so the user's "phone wins for - * steps but watch wins for HR" config is honoured. */ - type: MeasurementType; -} - -async function fetchMeasurementSeriesChunked( - userId: string, - type: MeasurementType, - options: { since?: Date; includeSleepStage?: boolean } = {}, -): Promise { - const out: ChunkedRow[] = []; - let cursorId: string | undefined; - for (let page = 0; page < 1000; page++) { - const chunk = await prisma.measurement.findMany({ - where: { - userId, - type, - ...(options.since ? { measuredAt: { gte: options.since } } : {}), - deletedAt: null, - }, - orderBy: [{ measuredAt: "asc" }, { id: "asc" }], - select: { - id: true, - measuredAt: true, - value: true, - source: true, - // v1.4.25 W8c — read deviceType so the canonical picker can - // honour the per-metric / per-device override. Nullable until - // iOS sends it. - deviceType: true, - ...(options.includeSleepStage ? { sleepStage: true } : {}), - }, - take: MEASUREMENT_CHUNK_SIZE, - ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : {}), - }); - if (chunk.length === 0) break; - for (const row of chunk) { - out.push({ - measuredAt: row.measuredAt, - value: row.value, - source: row.source, - deviceType: row.deviceType ?? null, - type, - sleepStage: - "sleepStage" in row - ? ((row.sleepStage as string | null) ?? null) - : null, - }); - } - if (chunk.length < MEASUREMENT_CHUNK_SIZE) break; - cursorId = chunk[chunk.length - 1].id; - } - return out; -} +// v1.4.49.1 — the chunked `fetchMeasurementSeriesChunked` helper and +// its supporting types / constants relocated entirely. The default +// slice now delegates the per-type summaries work to +// `computeSummariesSlice` (rollup-tier, sub-second on covered tenants) +// so the legacy 15-way live walk against `measurements` no longer has +// a caller. The slim slice was already independent of it. Deleting the +// dead code here removes the pool-starvation surface that gated the +// concurrent slim slice and prevents future regressions from +// accidentally reintroducing the fan-out. // v1.4.37 W2 — `WEEKDAY_FORMATTER_CACHE`, `getWeekdayFormatter`, // `dateFromDayKey`, `ISO_WEEKDAY`, and `isoWeekdayInTz` relocated diff --git a/src/lib/analytics/summaries-slice.ts b/src/lib/analytics/summaries-slice.ts index ddda0331c..d088b371f 100644 --- a/src/lib/analytics/summaries-slice.ts +++ b/src/lib/analytics/summaries-slice.ts @@ -109,6 +109,7 @@ interface NarrowAggregateRow { type: string; avg7: number | null; avg30: number | null; + avg30_last_month: number | null; slope7: number | null; r2_7: number | null; slope30: number | null; @@ -266,6 +267,10 @@ async function computeFromRollups(userId: string): Promise { AVG(m."value") FILTER ( WHERE m."measured_at" >= NOW() - INTERVAL '30 days' )::double precision AS avg30, + AVG(m."value") FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '60 days' + AND m."measured_at" < NOW() - INTERVAL '30 days' + )::double precision AS avg30_last_month, REGR_SLOPE( m."value", EXTRACT(EPOCH FROM m."measured_at") / 86400.0 @@ -415,7 +420,7 @@ async function computeFromRollups(userId: string): Promise { slope30: buildSlope(narrow?.slope30 ?? null, narrow?.r2_30 ?? null), slope90: buildSlope(narrow?.slope90 ?? null, narrow?.r2_90 ?? null), anomalyCount: 0, - avg30LastMonth: null, + avg30LastMonth: round2(narrow?.avg30_last_month ?? null), avg30LastYear: null, }; } @@ -500,6 +505,10 @@ async function computeFromLiveAggregate( AVG(m."value") FILTER ( WHERE m."measured_at" >= NOW() - INTERVAL '30 days' )::double precision AS avg30, + AVG(m."value") FILTER ( + WHERE m."measured_at" >= NOW() - INTERVAL '60 days' + AND m."measured_at" < NOW() - INTERVAL '30 days' + )::double precision AS avg30_last_month, REGR_SLOPE( m."value", EXTRACT(EPOCH FROM m."measured_at") / 86400.0 @@ -614,7 +623,7 @@ async function computeFromLiveAggregate( slope30: buildSlope(win?.slope30 ?? null, win?.r2_30 ?? null), slope90: buildSlope(win?.slope90 ?? null, win?.r2_90 ?? null), anomalyCount: 0, - avg30LastMonth: null, + avg30LastMonth: round2(win?.avg30_last_month ?? null), avg30LastYear: null, }; }