fix(measurements): keep per-type reads on the live index whatever the statistics say

The per-type reads descend measurements_live_covering_idx once per type and
stop at their cap, but the planner chose that descent only when its row
estimates favoured it. The type is a lateral parameter, so one generic
estimate covered every type; an account whose rows arrived after the last
ANALYZE was expected to hold about one row; and statistics taken while the
table held one type made the type condition look free. In those states the
planner walked the type-less (user_id, measured_at) index through every row
of the account once per type, or read every row of each type into a sort.
On a 9,300-row account that was 33,900 row visits for 6,300 rows returned,
1.66 million for the span probe and 845,000 for the dashboard's latest
reading per type.

One shared fragment now holds the descent: the account and the cap reach
the planner through scalar subqueries, and the type is matched as a closed
range it is also ordered on, so only an index leading (user_id, type,
measured_at) can return the rows without a sort. readNewestRowsPerType,
readLiveMeasurementSpan, the dashboard's readLatestEver and the latest-instant
probe of readLatestCanonical use it, and readLatestReadingsPerType reads
through readNewestRowsPerType.

The plan guard moves to its own file, which builds four statistics states on
purpose (none, predating the account over many types and over one type, and
analysed with other accounts) with autovacuum off, and holds every read to
its bound in each. The previous guard depended on statistics another file
had left behind, so it failed in the full suite and passed alone.

Refs #1398
This commit is contained in:
Marc-André Bombeck committed 2026-11-17 20:51:39 +01:00
1 parent 5bb77b975a
commit 8f1af43891
11 files changed
+536 -188

No files matched your search

@@ -55,9 +55,13 @@ describe("dashboard summary reads — latest reading per type", () => {
expect(lateral.length).toBeGreaterThan(0);
// The type list has to reach SQL as the driving relation, and the
// per-type branch has to stop at the first row — without the LIMIT the
// LATERAL still reads every row of the type partition.
// LATERAL still reads every row of the type partition. Its WHERE and
// ORDER BY are the shared live-index descent, which keeps the planner
// on the per-type index whatever the statistics say.
expect(sql).toMatch(/FROM\s+unnest\(\$\{/);
expect(sql).toMatch(/ORDER\s+BY\s+m\."measured_at"\s+DESC\s+LIMIT\s+1/);
expect(sql).toMatch(
/\$\{liveTypeDescent\(userId,[^}]*"newest",\s*since\)\}\s+LIMIT\s+1/,
);
});
});
@@ -332,12 +332,14 @@ describe("GET /api/insights/targets — mood-rollup tier swap", () => {
expect(latestCall).toBeDefined();
const [strings, ...values] = latestCall as [string[], ...unknown[]];
expect(strings.join("?")).not.toContain("DISTINCT ON");
expect(values).toContain("user-targets-1");
const floor = values.find(
// The probe's WHERE is the shared live-index descent, a nested fragment
// that carries the account and the floor.
const descent = values.find(
(v): v is { values: unknown[] } =>
typeof v === "object" && v !== null && !Array.isArray(v),
);
const flooredAt = floor?.values[0];
expect(descent?.values).toContain("user-targets-1");
const flooredAt = descent?.values.find((v) => v instanceof Date);
expect(flooredAt).toBeInstanceOf(Date);
const oneYearMs = 365 * 24 * 60 * 60 * 1000;
const ageMs = Date.now() - (flooredAt as Date).getTime();
+6 -8
View File
@@ -18,6 +18,7 @@
import { Prisma, type PrismaClient } from "@/generated/prisma/client";
import type { MeasurementType } from "@/generated/prisma/client";
import { ROLLUP_FOLD_WINDOW_MS } from "@/lib/rollups/measurement-rollups";
import { liveTypeDescent } from "@/lib/measurements/live-type-descent";
/** v1.4.38 — the single most recent reading per measurement
* type, irrespective of age. One row per metric the user has ever
@@ -83,6 +84,10 @@ export interface SparklineRow {
* is an inner join, so a type with no live reading drops out exactly as
* DISTINCT ON dropped it, and `ORDER BY mt."type"` reproduces the enum
* ordering the old `ORDER BY m."type"` emitted. Same rows, same order.
* The lookup's WHERE and ORDER BY are `liveTypeDescent`'s: written as a
* plain equality, an account whose rows arrived after the last ANALYZE had
* the planner walk the type-less `(user_id, measured_at)` index through
* every row of the account once per type.
*
* The driving relation is aliased `mt`, not `t`: a one-letter `t` alias
* makes `t("type")` in the SQL text look exactly like a `t()` translation
@@ -94,9 +99,6 @@ export function readLatestEver(
measurementTypes: readonly MeasurementType[],
since: Date | null = null,
): Promise<LatestEverRow[]> {
const floor = since
? Prisma.sql`AND m."measured_at" >= ${since}`
: Prisma.empty;
return db.$queryRaw<LatestEverRow[]>`
SELECT
mt."type" AS type,
@@ -108,11 +110,7 @@ export function readLatestEver(
m."value"::double precision AS value,
m."measured_at" AS measured_at
FROM measurements m
WHERE m."user_id" = ${userId}
AND m."type" = mt."type"
AND m."deleted_at" IS NULL
${floor}
ORDER BY m."measured_at" DESC
${liveTypeDescent(userId, Prisma.sql`mt."type"`, "newest", since)}
LIMIT 1
) l
ORDER BY mt."type"
@@ -19,10 +19,14 @@ function read(rel: string): string {
describe("prompt reads exclude tombstoned rows", () => {
it("general-status filters its measurement read", () => {
// The overview reads through the per-type latest-readings query.
// The overview reads through the per-type latest-readings query, which
// reads through the per-type newest-rows query.
expect(read("../general-status.ts")).toMatch(/readLatestReadingsPerType\(/);
expect(read("../recent-readings.ts")).toMatch(
/readLatestReadingsPerType[\s\S]*?m\."deleted_at" IS NULL/,
/function readLatestReadingsPerType[\s\S]*?readNewestRowsPerType\(/,
);
expect(read("../../measurements/live-type-descent.ts")).toMatch(
/function liveTypeDescent[\s\S]*?m\."deleted_at" IS NULL/,
);
});
@@ -30,7 +34,7 @@ describe("prompt reads exclude tombstoned rows", () => {
// The weight series reads through the per-type newest-rows query.
expect(read("../weight-status.ts")).toMatch(/readNewestRowsPerType\(/);
expect(read("../../measurements/newest-per-type.ts")).toMatch(
/readNewestRowsPerType[\s\S]*?m\."deleted_at" IS NULL/,
/function readNewestRowsPerType[\s\S]*?\$\{liveTypeDescent\(/,
);
});
+9 -21
View File
@@ -16,6 +16,7 @@
import type { MeasurementType } from "@/generated/prisma/client";
import { prisma } from "@/lib/db";
import { annotate } from "@/lib/logging/context";
import { readNewestRowsPerType } from "@/lib/measurements/newest-per-type";
export const RECENT_READINGS = 365;
export const RECENT_MIN_DAYS = 30;
@@ -70,9 +71,9 @@ export async function readRecentReadings(
* the multi-metric overview. It read the newest 5 000 rows across all types
* at once, so a dense stream (pulse every few minutes) filled the whole read
* within days and every other type fell out of the overview. One index
* descent per type now bounds each type on its own. The driving relation is
* aliased `mt`, the way `readLatestEver` aliases it, so the i18n call-site
* scanner does not read the SQL as a translation call.
* descent per type now bounds each type on its own; the descent is
* `readNewestRowsPerType`'s, so the statement that keeps the planner on it
* exists once.
*/
export async function readLatestReadingsPerType(
userId: string,
@@ -80,25 +81,12 @@ export async function readLatestReadingsPerType(
perType: number = RECENT_READINGS,
): Promise<Array<RecentReading & { type: MeasurementType }>> {
if (types.length === 0) return [];
const rows = await prisma.$queryRaw<
Array<{ type: MeasurementType; value: number; measured_at: Date }>
>`
SELECT mt."type" AS "type", x."value", x."measured_at"
FROM unnest(${[...types]}::text[]) AS mt("type")
CROSS JOIN LATERAL (
SELECT m."value", m."measured_at"
FROM measurements m
WHERE m."user_id" = ${userId}
AND m."type" = mt."type"::measurement_type
AND m."deleted_at" IS NULL
ORDER BY m."measured_at" DESC
LIMIT ${perType}
) x
ORDER BY x."measured_at" ASC
`;
const { rows } = await readNewestRowsPerType(userId, types, {
perTypeCap: perType,
});
return rows.map((r) => ({
type: r.type,
value: Number(r.value),
measuredAt: new Date(r.measured_at),
value: r.value,
measuredAt: r.measuredAt,
}));
}
+11 -3
View File
@@ -22,6 +22,13 @@ export interface LatestCanonicalRow {
* result: with only the lower bound it estimates a third of the type's rows
* per probe, and the summed estimate crosses `jit_above_cost`, whose
* compilation then costs more than the query itself.
*
* The latest-instant probe is `liveTypeDescent`
* (`src/lib/measurements/live-type-descent.ts`) written out positionally: the
* account through a scalar subquery, the type as a closed range it is also
* ordered on. As a plain equality, an account whose rows arrived after the
* last ANALYZE had the planner walk the type-less `(user_id, measured_at)`
* index through every row of the account once per type.
*/
export function readLatestCanonical(
userId: string,
@@ -39,11 +46,12 @@ export function readLatestCanonical(
CROSS JOIN LATERAL (
SELECT date_trunc('day', d."measured_at") AS day
FROM measurements d
WHERE d."user_id" = $1
AND d."type" = mt."type"
WHERE d."user_id" = (SELECT $1::text)
AND d."type" >= mt."type"
AND d."type" <= mt."type"
AND d."deleted_at" IS NULL
${since ? `AND d."measured_at" >= $3` : ""}
ORDER BY d."measured_at" DESC
ORDER BY d."type" ASC, d."measured_at" DESC
LIMIT 1
) last_day
CROSS JOIN LATERAL (
+65
View File
@@ -0,0 +1,65 @@
import { Prisma } from "@/generated/prisma/client";
/**
* The WHERE and ORDER BY of one type's descent of the live index
* `measurements_live_covering_idx (user_id, type, measured_at DESC) WHERE
* deleted_at IS NULL`, over the table aliased `m`: `typeSql` names the type,
* and the caller adds the LIMIT. The per-type reads that stop at a cap or at
* the first row descend through it (`newest-per-type.ts` and the dashboard's
* `readLatestEver`); `readLatestCanonical` writes the same statement out in
* its positional form.
*
* The descent is the right plan for every account and every type, but the
* planner picks it only when its row estimates say so, and for this read
* they cannot be trusted. The type is a lateral parameter, so one generic
* guess stands for the densest and the sparsest type alike, and it sits far
* below the cap of a dense stream, so the planner expects the LIMIT never to
* stop the scan and reads every row of the type into a sort. An account whose
* rows arrived after the last ANALYZE is absent from the statistics, so it is
* expected to hold about one row, and statistics taken while the table held a
* single type say the type condition removes nothing. In both the type-less
* `(user_id, measured_at)` index looks as good as the live one, and the
* planner walked it once per type through every row of the account. On the
* test's account of 9,300 rows that was 33,900 row visits for 6,300 rows
* returned, and 1.66 million for `readLiveMeasurementSpan`, which walks it
* at both ends of every type.
*
* So the statement withholds the values those estimates come from:
*
* - the account arrives through a scalar subquery, so the planner sizes it
* as an average account instead of the one row a new account gets;
* - the type is matched as a closed range on the one value and ordered on,
* so it is estimated by a fixed default rather than by column statistics,
* and only an index leading `(user_id, type, measured_at)` returns the
* rows in that order. The type-less index would need a full read and a
* sort, which it cannot win;
* - a cap above one row goes through a scalar subquery too: with a limit
* it cannot see, the planner costs the scan as stopping early, which a
* sort of every row cannot match. A LIMIT 1 probe needs no help there.
*
* Each of the three is needed in some state of the statistics, and
* `tests/integration/newest-per-type-plans.test.ts` builds each state and
* holds the plans to the cap. The range direction of the ORDER BY picks the
* live index's own order (`type` ascending, `measured_at` descending) for the
* newest end and its backward scan for the oldest end.
*/
export function liveTypeDescent(
userId: string,
typeSql: Prisma.Sql,
end: "newest" | "oldest",
since: Date | null,
): Prisma.Sql {
const sinceSql =
since === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${since}`;
const order =
end === "newest"
? Prisma.sql`ORDER BY m."type" ASC, m."measured_at" DESC`
: Prisma.sql`ORDER BY m."type" DESC, m."measured_at" ASC`;
return Prisma.sql`
WHERE m."user_id" = (SELECT ${userId}::text)
AND m."type" >= ${typeSql}
AND m."type" <= ${typeSql}
AND m."deleted_at" IS NULL
${sinceSql}
${order}`;
}
+15 -22
View File
@@ -26,11 +26,13 @@
* One statement: one index descent per type on the partial live index
* `measurements_live_covering_idx (user_id, type, measured_at DESC) WHERE
* deleted_at IS NULL`, each stopping at the cap, so the rows touched track the
* rows returned, never the length of a dense stream. The driving relation is
* aliased `mt`, as in `readLatestReadingsPerType`, so the i18n call-site
* scanner does not read the SQL as a translation call.
* rows returned, never the length of a dense stream (`liveTypeDescent` says how
* the statement keeps the planner on that path). The driving relation is
* aliased `mt` so the i18n call-site scanner does not read the SQL as a
* translation call.
*/
import { prisma } from "@/lib/db";
import { liveTypeDescent } from "@/lib/measurements/live-type-descent";
import {
Prisma,
type GlucoseContext,
@@ -74,8 +76,6 @@ export async function readNewestRowsPerType(
}
}
const since = options.since ?? null;
const sinceSql =
since === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${since}`;
// With a total, each row's rank within its type orders the cut, and the
// count per type says whether the cut reached it.
const fairShare = totalCap !== undefined;
@@ -115,12 +115,8 @@ export async function readNewestRowsPerType(
SELECT m."id", m."value", m."measured_at", m."source",
m."device_type", m."sleep_stage", m."glucose_context"
FROM measurements m
WHERE m."user_id" = ${userId}
AND m."type" = mt."type"::measurement_type
AND m."deleted_at" IS NULL
${sinceSql}
ORDER BY m."measured_at" DESC
LIMIT ${perTypeCap}
${liveTypeDescent(userId, Prisma.sql`mt."type"::measurement_type`, "newest", since)}
LIMIT (SELECT ${perTypeCap}::int)
) x
${shareCut}
) y
@@ -155,16 +151,15 @@ export async function readNewestRowsPerType(
* history when `since` is null), so the span a capped read reports is the
* account's, not the hours a dense stream fills of the read. One index probe
* at each end of every type on the partial live index, each stopping at its
* first row. The type-less `(user_id, measured_at)` index would answer in two
* probes, but on a table without statistics the planner took the live
* partial index instead and sorted every row of the account.
* first row (`liveTypeDescent`). The type-less `(user_id, measured_at)` index
* would answer in two probes, but on a table without statistics the planner
* took the live partial index instead and sorted every row of the account.
*/
export async function readLiveMeasurementSpan(
userId: string,
since: Date | null,
): Promise<{ oldest: Date | null; newest: Date | null }> {
const sinceSql =
since === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${since}`;
const type = Prisma.sql`mt."type"`;
const [row] = await prisma.$queryRaw<
Array<{ oldest: Date | null; newest: Date | null }>
>`
@@ -172,15 +167,13 @@ export async function readLiveMeasurementSpan(
FROM unnest(enum_range(NULL::measurement_type)) AS mt("type")
LEFT JOIN LATERAL (
SELECT m."measured_at" AS "at" FROM measurements m
WHERE m."user_id" = ${userId} AND m."type" = mt."type"
AND m."deleted_at" IS NULL ${sinceSql}
ORDER BY m."measured_at" ASC LIMIT 1
${liveTypeDescent(userId, type, "oldest", since)}
LIMIT 1
) o ON true
LEFT JOIN LATERAL (
SELECT m."measured_at" AS "at" FROM measurements m
WHERE m."user_id" = ${userId} AND m."type" = mt."type"
AND m."deleted_at" IS NULL ${sinceSql}
ORDER BY m."measured_at" DESC LIMIT 1
${liveTypeDescent(userId, type, "newest", since)}
LIMIT 1
) n ON true
`;
return {