mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-19 08:12:52 +00:00
perf(analytics): group before the canonical-source pick, and run without JIT
The shared canonical-source CTE sorted every raw reading in its window by a CASE expression no index can supply; on the benchmark account the pulse pick over ten years sorted 260,000 rows and spilled to disk. Its order depends only on (type, source), so the pick now groups the window to its distinct (type, day, source) triples first and runs DISTINCT ON over those. The picked source per day is the same by construction; an integration test runs the old statement beside the new one over a fixture with overlapping sources, sources off the ladder, tombstones, empty days and readings either side of midnight, under default and reordered ladders, for every reader shape, and requires identical rows. Every app session now sets jit=off. Postgres compiled the larger folds to machine code, which took 100 to 140 ms of a 220 ms statement and saved nothing for statements that run once and are bound by reading rows. JIT never changes a result.
This commit is contained in:
1 parent
b5f5a5a580
commit
850988c832
4 files changed
+264
-14
No files matched your search
@@ -152,14 +152,14 @@ describe("buildSessionOptions", () => {
|
||||
it("keeps work_mem when the timeout is disabled", () => {
|
||||
process.env.DATABASE_STATEMENT_TIMEOUT_MS = "0";
|
||||
delete process.env.DATABASE_WORK_MEM;
|
||||
expect(buildSessionOptions()).toBe("-c work_mem=16MB -c TimeZone=UTC");
|
||||
expect(buildSessionOptions()).toBe("-c work_mem=16MB -c jit=off -c TimeZone=UTC");
|
||||
});
|
||||
|
||||
it("sets the default work_mem beside the timeouts", () => {
|
||||
delete process.env.DATABASE_STATEMENT_TIMEOUT_MS;
|
||||
delete process.env.DATABASE_WORK_MEM;
|
||||
expect(buildSessionOptions()).toBe(
|
||||
"-c statement_timeout=60000 -c idle_in_transaction_session_timeout=60000 -c work_mem=16MB -c TimeZone=UTC",
|
||||
"-c statement_timeout=60000 -c idle_in_transaction_session_timeout=60000 -c work_mem=16MB -c jit=off -c TimeZone=UTC",
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -127,6 +127,16 @@ function assertInterval(value: string): string {
|
||||
* is whitelisted and, when given, scopes both the inner pick and the outer
|
||||
* filter to a trailing window. `onlyType`, when given, scopes both halves to
|
||||
* that one metric.
|
||||
*
|
||||
* The pick groups before it sorts. Its order (rank, then source name) depends
|
||||
* only on a row's `(type, source)`, so every reading of one source on one day
|
||||
* sorts alike, and the distinct `(type, day, source)` triples carry the whole
|
||||
* decision. A hash aggregate reduces the window's rows to those triples (a
|
||||
* few thousand) and `DISTINCT ON` sorts only them; sorting every raw row by
|
||||
* a CASE expression no index can supply spilled to disk on a large account.
|
||||
* The picked source per day is the same by construction;
|
||||
* `tests/integration/canonical-source-pick-parity.test.ts` holds the old
|
||||
* statement as its reference.
|
||||
*/
|
||||
export function canonicalMeasurementsCte(
|
||||
rankUnqualified: string,
|
||||
@@ -152,16 +162,20 @@ export function canonicalMeasurementsCte(
|
||||
SELECT mm.*
|
||||
FROM measurements mm
|
||||
JOIN (
|
||||
SELECT DISTINCT ON ("type", date_trunc('day', "measured_at"))
|
||||
"type" AS t,
|
||||
date_trunc('day', "measured_at") AS d,
|
||||
"source" AS canon
|
||||
FROM measurements
|
||||
WHERE "user_id" = $1
|
||||
AND "deleted_at" IS NULL
|
||||
${sinceInner}
|
||||
${typeInner}
|
||||
ORDER BY "type", date_trunc('day', "measured_at"), (${rankUnqualified}), "source"::text
|
||||
SELECT DISTINCT ON ("type", d)
|
||||
"type" AS t,
|
||||
d,
|
||||
"source" AS canon
|
||||
FROM (
|
||||
SELECT "type", date_trunc('day', "measured_at") AS d, "source"
|
||||
FROM measurements
|
||||
WHERE "user_id" = $1
|
||||
AND "deleted_at" IS NULL
|
||||
${sinceInner}
|
||||
${typeInner}
|
||||
GROUP BY "type", date_trunc('day', "measured_at"), "source"
|
||||
) g
|
||||
ORDER BY "type", d, (${rankUnqualified}), "source"::text
|
||||
) c
|
||||
ON c.t = mm."type"
|
||||
AND c.d = date_trunc('day', mm."measured_at")
|
||||
|
||||
+8
-2
@@ -146,7 +146,13 @@ export function getWorkMem(): string {
|
||||
* session carries them from the first query.
|
||||
*
|
||||
* The timeouts are left out when disabled (timeout 0); `work_mem` is always
|
||||
* set. Up to v1.41 the whole string was `undefined` at timeout 0, which would
|
||||
* set, and so is `jit=off`. Postgres compiles a statement to machine code
|
||||
* once its estimated cost passes `jit_above_cost`, which every analytics
|
||||
* fold over a year of readings does; on the benchmark account that
|
||||
* compilation took 100 to 140 ms of a 220 ms statement and saved nothing,
|
||||
* because these statements run once and are bound by reading rows, not by
|
||||
* evaluating expressions. JIT changes how a plan runs, never what it
|
||||
* returns. Up to v1.41 the whole string was `undefined` at timeout 0, which would
|
||||
* have taken `work_mem` with it. `undefined` now means exactly one thing:
|
||||
* the operator turned the startup options off for a connection pooler
|
||||
* ({@link sessionOptionsDisabled}).
|
||||
@@ -158,7 +164,7 @@ export function buildSessionOptions(): string | undefined {
|
||||
timeoutMs > 0
|
||||
? `-c statement_timeout=${timeoutMs} -c idle_in_transaction_session_timeout=${timeoutMs} `
|
||||
: "";
|
||||
return `${timeouts}-c work_mem=${getWorkMem()} ${TIME_ZONE_OPTION}`;
|
||||
return `${timeouts}-c work_mem=${getWorkMem()} -c jit=off ${TIME_ZONE_OPTION}`;
|
||||
}
|
||||
|
||||
export function prismaPoolConfig(): PoolConfig {
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
/**
|
||||
* The canonical-source pick, against a real Postgres: the shipped statement
|
||||
* (group the window to `(type, day, source)` first, then `DISTINCT ON`)
|
||||
* against the statement it replaced (`DISTINCT ON` over every raw row),
|
||||
* kept verbatim below as the reference.
|
||||
*
|
||||
* Both must return exactly the same rows, for every reader shape the helper
|
||||
* serves (all types or one type, a trailing window or none), under the
|
||||
* default ladders and under user ladders that reorder them, over a fixture
|
||||
* that carries what could tell them apart: overlapping sources on the same
|
||||
* day, a source off the ladder, a day where the winning source's only row is
|
||||
* tombstoned, readings either side of UTC midnight and of a DST change in
|
||||
* the user's zone, empty days, and several readings per source per day.
|
||||
*/
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
|
||||
import type {
|
||||
MeasurementSource,
|
||||
MeasurementType,
|
||||
} from "@/generated/prisma/client";
|
||||
import {
|
||||
buildSourceRankCase,
|
||||
canonicalMeasurementsCte,
|
||||
} from "@/lib/analytics/source-rank-sql";
|
||||
|
||||
import { makeUser } from "../kit/factories";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
|
||||
const USER_ID = "user-canonical-parity";
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
/** The pick as it stood before it grouped first. */
|
||||
function legacyCanonicalMeasurementsCte(
|
||||
rankUnqualified: string,
|
||||
sinceInterval?: string,
|
||||
onlyType?: MeasurementType,
|
||||
): string {
|
||||
const sinceInner = sinceInterval
|
||||
? `AND "measured_at" >= (NOW() AT TIME ZONE 'UTC') - INTERVAL '${sinceInterval}'`
|
||||
: "";
|
||||
const sinceOuter = sinceInterval
|
||||
? `AND mm."measured_at" >= (NOW() AT TIME ZONE 'UTC') - INTERVAL '${sinceInterval}'`
|
||||
: "";
|
||||
const typeInner = onlyType
|
||||
? `AND "type" = '${onlyType}'::"measurement_type"`
|
||||
: "";
|
||||
const typeOuter = onlyType
|
||||
? `AND mm."type" = '${onlyType}'::"measurement_type"`
|
||||
: "";
|
||||
return `
|
||||
SELECT mm.*
|
||||
FROM measurements mm
|
||||
JOIN (
|
||||
SELECT DISTINCT ON ("type", date_trunc('day', "measured_at"))
|
||||
"type" AS t,
|
||||
date_trunc('day', "measured_at") AS d,
|
||||
"source" AS canon
|
||||
FROM measurements
|
||||
WHERE "user_id" = $1
|
||||
AND "deleted_at" IS NULL
|
||||
${sinceInner}
|
||||
${typeInner}
|
||||
ORDER BY "type", date_trunc('day', "measured_at"), (${rankUnqualified}), "source"::text
|
||||
) c
|
||||
ON c.t = mm."type"
|
||||
AND c.d = date_trunc('day', mm."measured_at")
|
||||
AND c.canon = mm."source"
|
||||
WHERE mm."user_id" = $1
|
||||
AND mm."deleted_at" IS NULL
|
||||
${sinceOuter}
|
||||
${typeOuter}`;
|
||||
}
|
||||
|
||||
/** Small seeded generator, so the fixture is the same on every run. */
|
||||
function rng(seed: number) {
|
||||
let a = seed >>> 0;
|
||||
return () => {
|
||||
a = (a + 0x6d2b79f5) >>> 0;
|
||||
let t = a;
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
const SOURCES: MeasurementSource[] = [
|
||||
"APPLE_HEALTH",
|
||||
"WITHINGS",
|
||||
"WHOOP",
|
||||
"OURA",
|
||||
"MANUAL",
|
||||
// Off every ladder below: the name tiebreak decides.
|
||||
"NIGHTSCOUT",
|
||||
];
|
||||
const TYPES: MeasurementType[] = [
|
||||
"PULSE",
|
||||
"RESTING_HEART_RATE",
|
||||
"HEART_RATE_VARIABILITY",
|
||||
"WEIGHT",
|
||||
"ACTIVITY_STEPS",
|
||||
"BLOOD_GLUCOSE",
|
||||
];
|
||||
|
||||
async function seed(): Promise<number> {
|
||||
const prisma = getPrismaClient();
|
||||
await makeUser({ id: USER_ID, timezone: "Europe/Berlin" });
|
||||
const random = rng(42);
|
||||
const now = Date.now();
|
||||
const rows: Array<{
|
||||
userId: string;
|
||||
type: MeasurementType;
|
||||
source: MeasurementSource;
|
||||
value: number;
|
||||
unit: string;
|
||||
measuredAt: Date;
|
||||
deletedAt: Date | null;
|
||||
}> = [];
|
||||
// 140 days: across the 90-day window edge, and over the last DST change.
|
||||
for (let day = 0; day < 140; day++) {
|
||||
if (day % 11 === 5) continue; // an empty day
|
||||
const dayStart = Math.floor((now - day * DAY_MS) / DAY_MS) * DAY_MS;
|
||||
for (const type of TYPES) {
|
||||
const sourceCount = 1 + Math.floor(random() * 3);
|
||||
const sources = [...SOURCES]
|
||||
.sort(() => random() - 0.5)
|
||||
.slice(0, sourceCount);
|
||||
for (const source of sources) {
|
||||
const readings = 1 + Math.floor(random() * 4);
|
||||
for (let r = 0; r < readings; r++) {
|
||||
// Mostly mid-day, sometimes a minute either side of UTC midnight.
|
||||
const offset =
|
||||
r === 0 && random() < 0.2
|
||||
? random() < 0.5
|
||||
? 60_000
|
||||
: DAY_MS - 60_000
|
||||
: Math.floor(random() * DAY_MS);
|
||||
rows.push({
|
||||
userId: USER_ID,
|
||||
type,
|
||||
source,
|
||||
value: Math.round(random() * 1000) / 10,
|
||||
unit: "x",
|
||||
measuredAt: new Date(dayStart + offset),
|
||||
// A tombstone, sometimes the only row its source had that day.
|
||||
deletedAt: random() < 0.08 ? new Date() : null,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// The natural identity is unique; drop accidental collisions.
|
||||
const seen = new Set<string>();
|
||||
const unique = rows.filter((row) => {
|
||||
const key = `${row.type}|${row.source}|${row.measuredAt.getTime()}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
await prisma.measurement.createMany({ data: unique });
|
||||
return unique.length;
|
||||
}
|
||||
|
||||
const LADDERS: Array<[string, unknown]> = [
|
||||
["default ladders", null],
|
||||
[
|
||||
"user ladders that reorder them",
|
||||
{
|
||||
pulse: ["APPLE_HEALTH", "WITHINGS"],
|
||||
restingHeartRate: ["OURA", "APPLE_HEALTH", "WHOOP"],
|
||||
hrv: ["APPLE_HEALTH"],
|
||||
weight: ["MANUAL", "WITHINGS"],
|
||||
steps: ["WITHINGS", "APPLE_HEALTH"],
|
||||
},
|
||||
],
|
||||
];
|
||||
|
||||
const SHAPES: Array<[string, string | undefined, MeasurementType | undefined]> =
|
||||
[
|
||||
["every type, 90 days", "90 days", undefined],
|
||||
["every type, no window", undefined, undefined],
|
||||
["one type, 366 days", "366 days", "PULSE"],
|
||||
["one unranked type, 30 days", "30 days", "BLOOD_GLUCOSE"],
|
||||
["one type, 7 days", "7 days", "RESTING_HEART_RATE"],
|
||||
];
|
||||
|
||||
describe("canonical-source pick parity", () => {
|
||||
let seeded = 0;
|
||||
beforeAll(async () => {
|
||||
await truncateAllTables(getPrismaClient());
|
||||
seeded = await seed();
|
||||
});
|
||||
|
||||
it("seeds a fixture worth comparing", () => {
|
||||
expect(seeded).toBeGreaterThan(2000);
|
||||
});
|
||||
|
||||
for (const [ladderLabel, ladder] of LADDERS) {
|
||||
for (const [shapeLabel, since, onlyType] of SHAPES) {
|
||||
it(`returns the same rows: ${ladderLabel}, ${shapeLabel}`, async () => {
|
||||
const prisma = getPrismaClient();
|
||||
const rank = buildSourceRankCase(ladder, '"type"', '"source"');
|
||||
const select = (cte: string) =>
|
||||
prisma.$queryRawUnsafe<
|
||||
Array<{ id: string; type: string; source: string }>
|
||||
>(
|
||||
`WITH cm AS (${cte}) SELECT id, type::text AS type, source::text AS source FROM cm ORDER BY id`,
|
||||
USER_ID,
|
||||
);
|
||||
const [now, before] = await Promise.all([
|
||||
select(canonicalMeasurementsCte(rank, since, onlyType)),
|
||||
select(legacyCanonicalMeasurementsCte(rank, since, onlyType)),
|
||||
]);
|
||||
expect(now.length).toBeGreaterThan(0);
|
||||
expect(now).toEqual(before);
|
||||
// The pick dropped something, so the comparison is not trivial.
|
||||
if (!onlyType || onlyType === "PULSE") {
|
||||
const live = await prisma.measurement.count({
|
||||
where: {
|
||||
userId: USER_ID,
|
||||
deletedAt: null,
|
||||
...(onlyType ? { type: onlyType } : {}),
|
||||
},
|
||||
});
|
||||
if (!since) expect(now.length).toBeLessThan(live);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user