mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-08 12:32:43 +00:00
perf(analytics): slim summaries SQL aggregation, 306k rows to 8
The v1.4.37.1 fire-and-forget hotfix unblocked the event loop but live HAR still showed /api/analytics?slice=summaries at ~3.1 s per cache miss on a power-user account. Root cause: the v1.4.35 implementation read EVERY DAY rollup bucket for the user without a bucketStart window and composed count/min/max/ mean in JavaScript. On the live account that materialised as a 306k-row transfer plus a JS loop. Replace the unwindowed prisma.measurementRollup.findMany with a per-type $queryRaw GROUP BY. Returns 8 rows (one per type with count/min/max/mean already shaped server- side) instead of 306k bucket rows. The downstream JS aggregation is bypassed because the per-type aggregate arrives pre-composed. Cache-miss budget on Marc-sized accounts: ~3.1 s expected to drop into the < 100 ms range. The narrow aggregate query continues to carry the windowed avg7/avg30 and regression slope/r2 columns that the GROUP BY cannot reconstruct. Response shape is unchanged.
This commit is contained in:
1 parent
98e078d359
commit
57795418fc
3 files changed
+114
-81
No files matched your search
@@ -1,5 +1,34 @@
|
||||
# Changelog
|
||||
|
||||
## [1.4.37.2] — 2026-05-17 — Slim summaries SQL aggregation
|
||||
|
||||
Second hotfix on top of v1.4.37. The v1.4.37.1 fire-and-forget
|
||||
release removed the 60 s event-loop block but live HAR still
|
||||
showed `/api/analytics?slice=summaries` taking ~3.1 s per cache
|
||||
miss. Root cause: the v1.4.35 implementation read EVERY DAY
|
||||
rollup bucket for the user (`findMany` without a `bucketStart`
|
||||
window) and composed `count / min / max / mean` in JavaScript.
|
||||
On a power-user account that materialised as a ~306k-row
|
||||
transfer plus a JS loop. The slim slice's contract is the
|
||||
all-time per-type aggregate — exactly what a SQL `GROUP BY type`
|
||||
returns in a single round-trip.
|
||||
|
||||
### Fixed
|
||||
|
||||
- `computeFromRollups` in `src/lib/analytics/summaries-slice.ts`
|
||||
swaps the unwindowed `prisma.measurementRollup.findMany` for a
|
||||
per-type `$queryRaw` GROUP BY that hands back 8 rows of
|
||||
`count / min / max / mean` instead of ~306k bucket rows. The
|
||||
downstream JS aggregation is bypassed because the per-type
|
||||
aggregate is already shaped server-side. Cache-miss budget on
|
||||
Marc-sized accounts: ~3.1 s → < 100 ms expected.
|
||||
|
||||
### Operator notes
|
||||
|
||||
- No schema change. No env-var change. No public API change
|
||||
(the response body shape is identical to v1.4.37.1).
|
||||
- Coolify auto-deploys main on tag push.
|
||||
|
||||
## [1.4.37.1] — 2026-05-17 — Event-loop unblock on the read path
|
||||
|
||||
Hotfix on top of v1.4.37. Post-deploy verification surfaced live
|
||||
|
||||
@@ -228,10 +228,15 @@ describe("computeSummariesSlice", () => {
|
||||
});
|
||||
|
||||
describe("rollup-fresh happy path", () => {
|
||||
it("composes count/min/max/mean from DAY buckets without running the heavy aggregate", async () => {
|
||||
it("composes count/min/max/mean from the per-type rollup GROUP BY without running the heavy aggregate", async () => {
|
||||
// v1.4.37.2 — the slim slice's rollup read is now a per-type
|
||||
// GROUP BY ($queryRaw) instead of a row-per-bucket findMany,
|
||||
// so the mock sequence is:
|
||||
// 1. per-type coverage probe — WEIGHT fully covered ⇒ happy path.
|
||||
// 2. narrow aggregate ($queryRaw) — windowed/regression only.
|
||||
// 3. latests ($queryRaw).
|
||||
// 2. narrow aggregate — windowed/regression only.
|
||||
// 3. latests.
|
||||
// 4. rollup GROUP BY — one row per type with count/min/max/mean
|
||||
// already composed server-side.
|
||||
RAW.mockResolvedValueOnce([{ type: "WEIGHT", has_buckets: true }])
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
@@ -248,31 +253,26 @@ describe("computeSummariesSlice", () => {
|
||||
])
|
||||
.mockResolvedValueOnce([
|
||||
{ type: "WEIGHT", value: 82.7, measured_at: new Date() },
|
||||
])
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
type: "WEIGHT",
|
||||
// pre-aggregated server-side: SUM(count), MIN(min),
|
||||
// MAX(max), weighted mean — equivalent to the two-bucket
|
||||
// fixture below.
|
||||
// bucket A: count=10, mean=81.0, min=79.5, max=82.0
|
||||
// bucket B: count=10, mean=83.0, min=81.5, max=84.0
|
||||
// ⇒ count=20, min=79.5, max=84.0, mean=82
|
||||
count: 20,
|
||||
min: 79.5,
|
||||
max: 84.0,
|
||||
mean: 82.0,
|
||||
},
|
||||
]);
|
||||
|
||||
ROLLUP_FIND_MANY.mockResolvedValueOnce([
|
||||
{
|
||||
type: "WEIGHT",
|
||||
bucketStart: new Date("2026-05-10T00:00:00.000Z"),
|
||||
count: 10,
|
||||
mean: 81.0,
|
||||
minValue: 79.5,
|
||||
maxValue: 82.0,
|
||||
},
|
||||
{
|
||||
type: "WEIGHT",
|
||||
bucketStart: new Date("2026-05-11T00:00:00.000Z"),
|
||||
count: 10,
|
||||
mean: 83.0,
|
||||
minValue: 81.5,
|
||||
maxValue: 84.0,
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await computeSummariesSlice("user-rollup");
|
||||
const weight = result.summaries.WEIGHT;
|
||||
|
||||
// count = 10 + 10 = 20; min/max across buckets; weighted mean.
|
||||
expect(weight.count).toBe(20);
|
||||
expect(weight.min).toBe(79.5);
|
||||
expect(weight.max).toBe(84.0);
|
||||
@@ -285,9 +285,10 @@ describe("computeSummariesSlice", () => {
|
||||
confidence: 0.5,
|
||||
});
|
||||
|
||||
// probe + narrow aggregate + latests — NOT the heavy aggregate.
|
||||
expect(RAW).toHaveBeenCalledTimes(3);
|
||||
expect(ROLLUP_FIND_MANY).toHaveBeenCalledTimes(1);
|
||||
// probe + narrow aggregate + latests + rollup GROUP BY
|
||||
// (v1.4.37.2 — the prior `findMany` is gone). No heavy aggregate.
|
||||
expect(RAW).toHaveBeenCalledTimes(4);
|
||||
expect(ROLLUP_FIND_MANY).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -58,7 +58,6 @@ import type { DataSummary } from "@/lib/analytics/trends";
|
||||
import { measurementTypeEnum } from "@/lib/validations/measurement";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { ensureUserRollupsFresh } from "@/lib/measurements/rollups";
|
||||
import { aggregateBuckets } from "@/lib/measurements/rollup-read";
|
||||
import {
|
||||
isFullyCovered,
|
||||
probeRollupCoverage,
|
||||
@@ -91,6 +90,20 @@ interface HeavyAggregateRow {
|
||||
* so the rollup-fresh read path's SQL projection is half as wide as
|
||||
* the legacy aggregate's.
|
||||
*/
|
||||
/**
|
||||
* v1.4.37.2 — shape returned by the per-type GROUP BY over
|
||||
* `measurement_rollups`. Replaces the v1.4.35 row-per-bucket transfer
|
||||
* that materialised ~306k rows on a power-user account; the SQL now
|
||||
* does the aggregation server-side and hands back one row per type.
|
||||
*/
|
||||
interface RollupAggregateRow {
|
||||
type: string;
|
||||
count: number;
|
||||
min: number;
|
||||
max: number;
|
||||
mean: number;
|
||||
}
|
||||
|
||||
interface NarrowAggregateRow {
|
||||
type: string;
|
||||
avg7: number | null;
|
||||
@@ -272,24 +285,32 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
|
||||
WHERE m."user_id" = ${userId}
|
||||
ORDER BY m."type", m."measured_at" DESC
|
||||
`,
|
||||
// v1.4.35 — every DAY rollup bucket for the user. The slim slice's
|
||||
// `count / min / max / mean` is all-time, so we read the bucket
|
||||
// table without a `bucketStart` window. The defensive parity
|
||||
// check below uses the live aggregate's `COUNT(*)` to decide
|
||||
// whether the rollup is complete for this account (post-backfill
|
||||
// → composed) or partial (pre-backfill → falls back to live).
|
||||
prisma.measurementRollup.findMany({
|
||||
where: { userId, granularity: "DAY" },
|
||||
orderBy: [{ type: "asc" }, { bucketStart: "asc" }],
|
||||
select: {
|
||||
type: true,
|
||||
bucketStart: true,
|
||||
count: true,
|
||||
mean: true,
|
||||
minValue: true,
|
||||
maxValue: true,
|
||||
},
|
||||
}),
|
||||
// v1.4.37.2 hotfix — the v1.4.35 implementation read EVERY DAY
|
||||
// rollup bucket for the user (`findMany` without a `bucketStart`
|
||||
// window) and then composed `count / min / max / mean` in JS.
|
||||
// On a power-user account that materialised as a 306k-row
|
||||
// transfer + a JS loop = ~3.85 s per cache miss, even with the
|
||||
// rollup table hot. The slim slice's contract is the all-time
|
||||
// count / min / max / mean per type — exactly what a SQL
|
||||
// `GROUP BY type` returns in a single round-trip. Returns 8 rows
|
||||
// instead of 306k, brings the cache-miss cost into the < 100 ms
|
||||
// budget. The downstream `aggregateBuckets` call is bypassed
|
||||
// because the per-type aggregate is already shaped server-side.
|
||||
prisma.$queryRaw<RollupAggregateRow[]>`
|
||||
SELECT
|
||||
"type"::text AS type,
|
||||
SUM("count")::int AS count,
|
||||
MIN("min_value")::double precision AS min,
|
||||
MAX("max_value")::double precision AS max,
|
||||
(
|
||||
SUM("count" * "mean")::double precision
|
||||
/ NULLIF(SUM("count")::double precision, 0)
|
||||
) AS mean
|
||||
FROM "measurement_rollups"
|
||||
WHERE "user_id" = ${userId}
|
||||
AND "granularity" = 'DAY'
|
||||
GROUP BY "type"
|
||||
`,
|
||||
]);
|
||||
|
||||
const latestByType = new Map<string, number>();
|
||||
@@ -328,48 +349,30 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
|
||||
};
|
||||
}
|
||||
|
||||
// v1.4.35 — partition the DAY buckets by type so the per-type loop
|
||||
// can compose `count / min / max / mean` from them in O(1).
|
||||
const bucketsByType = new Map<
|
||||
string,
|
||||
Array<{
|
||||
day: Date;
|
||||
count: number;
|
||||
mean: number;
|
||||
minValue: number;
|
||||
maxValue: number;
|
||||
}>
|
||||
>();
|
||||
for (const b of dayBuckets) {
|
||||
const list = bucketsByType.get(b.type) ?? [];
|
||||
list.push({
|
||||
day: b.bucketStart,
|
||||
count: b.count,
|
||||
mean: b.mean,
|
||||
minValue: b.minValue,
|
||||
maxValue: b.maxValue,
|
||||
});
|
||||
bucketsByType.set(b.type, list);
|
||||
}
|
||||
|
||||
// v1.4.37.2 hotfix — the rollup aggregate is now shaped server-side
|
||||
// (one row per type with count / min / max / mean already computed),
|
||||
// so we map straight through instead of partitioning + composing in
|
||||
// JS. The narrows query feeds the windowed avgs + slope columns the
|
||||
// GROUP BY cannot reconstruct.
|
||||
const narrowByType = new Map<string, NarrowAggregateRow>();
|
||||
for (const row of narrows) {
|
||||
narrowByType.set(row.type, row);
|
||||
}
|
||||
|
||||
let totalRows = 0;
|
||||
for (const [type, buckets] of bucketsByType.entries()) {
|
||||
const composed = aggregateBuckets(buckets);
|
||||
if (composed.count === 0) continue;
|
||||
totalRows += composed.count;
|
||||
const narrow = narrowByType.get(type);
|
||||
const latest = latestByType.get(type) ?? null;
|
||||
summaries[type] = {
|
||||
count: composed.count,
|
||||
let typeCount = 0;
|
||||
for (const row of dayBuckets) {
|
||||
if (!row.count || row.count <= 0) continue;
|
||||
typeCount += 1;
|
||||
totalRows += row.count;
|
||||
const narrow = narrowByType.get(row.type);
|
||||
const latest = latestByType.get(row.type) ?? null;
|
||||
summaries[row.type] = {
|
||||
count: row.count,
|
||||
latest,
|
||||
min: round2(composed.min),
|
||||
max: round2(composed.max),
|
||||
mean: round2(composed.mean),
|
||||
min: round2(row.min),
|
||||
max: round2(row.max),
|
||||
mean: round2(row.mean),
|
||||
avg7: round2(narrow?.avg7 ?? null),
|
||||
avg30: round2(narrow?.avg30 ?? null),
|
||||
slope7: buildSlope(narrow?.slope7 ?? null, narrow?.r2_7 ?? null),
|
||||
@@ -387,7 +390,7 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
|
||||
analytics: {
|
||||
slim_summaries: {
|
||||
row_count: totalRows,
|
||||
type_count: bucketsByType.size,
|
||||
type_count: typeCount,
|
||||
path: "rollup",
|
||||
},
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user