perf(analytics): slim summaries SQL aggregation, 306k rows to 8

The v1.4.37.1 fire-and-forget hotfix unblocked the event loop
but live HAR still showed /api/analytics?slice=summaries at
~3.1 s per cache miss on a power-user account. Root cause: the
v1.4.35 implementation read EVERY DAY rollup bucket for the
user without a bucketStart window and composed count/min/max/
mean in JavaScript. On the live account that materialised as
a 306k-row transfer plus a JS loop.

Replace the unwindowed prisma.measurementRollup.findMany with
a per-type $queryRaw GROUP BY. Returns 8 rows
(one per type with count/min/max/mean already shaped server-
side) instead of 306k bucket rows. The downstream JS aggregation
is bypassed because the per-type aggregate arrives pre-composed.
Cache-miss budget on Marc-sized accounts: ~3.1 s expected to
drop into the < 100 ms range.

The narrow aggregate query continues to carry the windowed
avg7/avg30 and regression slope/r2 columns that the GROUP BY
cannot reconstruct. Response shape is unchanged.
This commit is contained in:
Marc-André Bombeck committed 2026-05-17 16:57:13 +02:00
1 parent 98e078d359
commit 57795418fc
3 files changed
+114 -81

No files matched your search

+29
View File
@@ -1,5 +1,34 @@
# Changelog
## [1.4.37.2] — 2026-05-17 — Slim summaries SQL aggregation
Second hotfix on top of v1.4.37. The v1.4.37.1 fire-and-forget
release removed the 60 s event-loop block but live HAR still
showed `/api/analytics?slice=summaries` taking ~3.1 s per cache
miss. Root cause: the v1.4.35 implementation read EVERY DAY
rollup bucket for the user (`findMany` without a `bucketStart`
window) and composed `count / min / max / mean` in JavaScript.
On a power-user account that materialised as a ~306k-row
transfer plus a JS loop. The slim slice's contract is the
all-time per-type aggregate — exactly what a SQL `GROUP BY type`
returns in a single round-trip.
### Fixed
- `computeFromRollups` in `src/lib/analytics/summaries-slice.ts`
swaps the unwindowed `prisma.measurementRollup.findMany` for a
per-type `$queryRaw` GROUP BY that hands back 8 rows of
`count / min / max / mean` instead of ~306k bucket rows. The
downstream JS aggregation is bypassed because the per-type
aggregate is already shaped server-side. Cache-miss budget on
Marc-sized accounts: ~3.1 s → < 100 ms expected.
### Operator notes
- No schema change. No env-var change. No public API change
(the response body shape is identical to v1.4.37.1).
- Coolify auto-deploys main on tag push.
## [1.4.37.1] — 2026-05-17 — Event-loop unblock on the read path
Hotfix on top of v1.4.37. Post-deploy verification surfaced live
@@ -228,10 +228,15 @@ describe("computeSummariesSlice", () => {
});
describe("rollup-fresh happy path", () => {
it("composes count/min/max/mean from DAY buckets without running the heavy aggregate", async () => {
it("composes count/min/max/mean from the per-type rollup GROUP BY without running the heavy aggregate", async () => {
// v1.4.37.2 — the slim slice's rollup read is now a per-type
// GROUP BY ($queryRaw) instead of a row-per-bucket findMany,
// so the mock sequence is:
// 1. per-type coverage probe — WEIGHT fully covered ⇒ happy path.
// 2. narrow aggregate ($queryRaw) — windowed/regression only.
// 3. latests ($queryRaw).
// 2. narrow aggregate — windowed/regression only.
// 3. latests.
// 4. rollup GROUP BY — one row per type with count/min/max/mean
// already composed server-side.
RAW.mockResolvedValueOnce([{ type: "WEIGHT", has_buckets: true }])
.mockResolvedValueOnce([
{
@@ -248,31 +253,26 @@ describe("computeSummariesSlice", () => {
])
.mockResolvedValueOnce([
{ type: "WEIGHT", value: 82.7, measured_at: new Date() },
])
.mockResolvedValueOnce([
{
type: "WEIGHT",
// pre-aggregated server-side: SUM(count), MIN(min),
// MAX(max), weighted mean — equivalent to the two-bucket
// fixture below.
// bucket A: count=10, mean=81.0, min=79.5, max=82.0
// bucket B: count=10, mean=83.0, min=81.5, max=84.0
// ⇒ count=20, min=79.5, max=84.0, mean=82
count: 20,
min: 79.5,
max: 84.0,
mean: 82.0,
},
]);
ROLLUP_FIND_MANY.mockResolvedValueOnce([
{
type: "WEIGHT",
bucketStart: new Date("2026-05-10T00:00:00.000Z"),
count: 10,
mean: 81.0,
minValue: 79.5,
maxValue: 82.0,
},
{
type: "WEIGHT",
bucketStart: new Date("2026-05-11T00:00:00.000Z"),
count: 10,
mean: 83.0,
minValue: 81.5,
maxValue: 84.0,
},
]);
const result = await computeSummariesSlice("user-rollup");
const weight = result.summaries.WEIGHT;
// count = 10 + 10 = 20; min/max across buckets; weighted mean.
expect(weight.count).toBe(20);
expect(weight.min).toBe(79.5);
expect(weight.max).toBe(84.0);
@@ -285,9 +285,10 @@ describe("computeSummariesSlice", () => {
confidence: 0.5,
});
// probe + narrow aggregate + latests — NOT the heavy aggregate.
expect(RAW).toHaveBeenCalledTimes(3);
expect(ROLLUP_FIND_MANY).toHaveBeenCalledTimes(1);
// probe + narrow aggregate + latests + rollup GROUP BY
// (v1.4.37.2 — the prior `findMany` is gone). No heavy aggregate.
expect(RAW).toHaveBeenCalledTimes(4);
expect(ROLLUP_FIND_MANY).toHaveBeenCalledTimes(0);
});
});
});
+58 -55
View File
@@ -58,7 +58,6 @@ import type { DataSummary } from "@/lib/analytics/trends";
import { measurementTypeEnum } from "@/lib/validations/measurement";
import { annotate } from "@/lib/logging/context";
import { ensureUserRollupsFresh } from "@/lib/measurements/rollups";
import { aggregateBuckets } from "@/lib/measurements/rollup-read";
import {
isFullyCovered,
probeRollupCoverage,
@@ -91,6 +90,20 @@ interface HeavyAggregateRow {
* so the rollup-fresh read path's SQL projection is half as wide as
* the legacy aggregate's.
*/
/**
* v1.4.37.2 — shape returned by the per-type GROUP BY over
* `measurement_rollups`. Replaces the v1.4.35 row-per-bucket transfer
* that materialised ~306k rows on a power-user account; the SQL now
* does the aggregation server-side and hands back one row per type.
*/
interface RollupAggregateRow {
type: string;
count: number;
min: number;
max: number;
mean: number;
}
interface NarrowAggregateRow {
type: string;
avg7: number | null;
@@ -272,24 +285,32 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
WHERE m."user_id" = ${userId}
ORDER BY m."type", m."measured_at" DESC
`,
// v1.4.35 — every DAY rollup bucket for the user. The slim slice's
// `count / min / max / mean` is all-time, so we read the bucket
// table without a `bucketStart` window. The defensive parity
// check below uses the live aggregate's `COUNT(*)` to decide
// whether the rollup is complete for this account (post-backfill
// → composed) or partial (pre-backfill → falls back to live).
prisma.measurementRollup.findMany({
where: { userId, granularity: "DAY" },
orderBy: [{ type: "asc" }, { bucketStart: "asc" }],
select: {
type: true,
bucketStart: true,
count: true,
mean: true,
minValue: true,
maxValue: true,
},
}),
// v1.4.37.2 hotfix — the v1.4.35 implementation read EVERY DAY
// rollup bucket for the user (`findMany` without a `bucketStart`
// window) and then composed `count / min / max / mean` in JS.
// On a power-user account that materialised as a 306k-row
// transfer + a JS loop = ~3.85 s per cache miss, even with the
// rollup table hot. The slim slice's contract is the all-time
// count / min / max / mean per type — exactly what a SQL
// `GROUP BY type` returns in a single round-trip. Returns 8 rows
// instead of 306k, brings the cache-miss cost into the < 100 ms
// budget. The downstream `aggregateBuckets` call is bypassed
// because the per-type aggregate is already shaped server-side.
prisma.$queryRaw<RollupAggregateRow[]>`
SELECT
"type"::text AS type,
SUM("count")::int AS count,
MIN("min_value")::double precision AS min,
MAX("max_value")::double precision AS max,
(
SUM("count" * "mean")::double precision
/ NULLIF(SUM("count")::double precision, 0)
) AS mean
FROM "measurement_rollups"
WHERE "user_id" = ${userId}
AND "granularity" = 'DAY'
GROUP BY "type"
`,
]);
const latestByType = new Map<string, number>();
@@ -328,48 +349,30 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
};
}
// v1.4.35 — partition the DAY buckets by type so the per-type loop
// can compose `count / min / max / mean` from them in O(1).
const bucketsByType = new Map<
string,
Array<{
day: Date;
count: number;
mean: number;
minValue: number;
maxValue: number;
}>
>();
for (const b of dayBuckets) {
const list = bucketsByType.get(b.type) ?? [];
list.push({
day: b.bucketStart,
count: b.count,
mean: b.mean,
minValue: b.minValue,
maxValue: b.maxValue,
});
bucketsByType.set(b.type, list);
}
// v1.4.37.2 hotfix — the rollup aggregate is now shaped server-side
// (one row per type with count / min / max / mean already computed),
// so we map straight through instead of partitioning + composing in
// JS. The narrows query feeds the windowed avgs + slope columns the
// GROUP BY cannot reconstruct.
const narrowByType = new Map<string, NarrowAggregateRow>();
for (const row of narrows) {
narrowByType.set(row.type, row);
}
let totalRows = 0;
for (const [type, buckets] of bucketsByType.entries()) {
const composed = aggregateBuckets(buckets);
if (composed.count === 0) continue;
totalRows += composed.count;
const narrow = narrowByType.get(type);
const latest = latestByType.get(type) ?? null;
summaries[type] = {
count: composed.count,
let typeCount = 0;
for (const row of dayBuckets) {
if (!row.count || row.count <= 0) continue;
typeCount += 1;
totalRows += row.count;
const narrow = narrowByType.get(row.type);
const latest = latestByType.get(row.type) ?? null;
summaries[row.type] = {
count: row.count,
latest,
min: round2(composed.min),
max: round2(composed.max),
mean: round2(composed.mean),
min: round2(row.min),
max: round2(row.max),
mean: round2(row.mean),
avg7: round2(narrow?.avg7 ?? null),
avg30: round2(narrow?.avg30 ?? null),
slope7: buildSlope(narrow?.slope7 ?? null, narrow?.r2_7 ?? null),
@@ -387,7 +390,7 @@ async function computeFromRollups(userId: string): Promise<SummariesSlice> {
analytics: {
slim_summaries: {
row_count: totalRows,
type_count: bucketsByType.size,
type_count: typeCount,
path: "rollup",
},
},