mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-19 08:12:52 +00:00
fix(measurements): keep per-type reads on the live index whatever the statistics say
The per-type reads descend measurements_live_covering_idx once per type and stop at their cap, but the planner chose that descent only when its row estimates favoured it. The type is a lateral parameter, so one generic estimate covered every type; an account whose rows arrived after the last ANALYZE was expected to hold about one row; and statistics taken while the table held one type made the type condition look free. In those states the planner walked the type-less (user_id, measured_at) index through every row of the account once per type, or read every row of each type into a sort. On a 9,300-row account that was 33,900 row visits for 6,300 rows returned, 1.66 million for the span probe and 845,000 for the dashboard's latest reading per type. One shared fragment now holds the descent: the account and the cap reach the planner through scalar subqueries, and the type is matched as a closed range it is also ordered on, so only an index leading (user_id, type, measured_at) can return the rows without a sort. readNewestRowsPerType, readLiveMeasurementSpan, the dashboard's readLatestEver and the latest-instant probe of readLatestCanonical use it, and readLatestReadingsPerType reads through readNewestRowsPerType. The plan guard moves to its own file, which builds four statistics states on purpose (none, predating the account over many types and over one type, and analysed with other accounts) with autovacuum off, and holds every read to its bound in each. The previous guard depended on statistics another file had left behind, so it failed in the full suite and passed alone. Refs #1398
This commit is contained in:
1 parent
5bb77b975a
commit
8f1af43891
11 files changed
+536
-188
No files matched your search
@@ -41,6 +41,7 @@
|
||||
|
||||
- A lab marker created from a text result, such as "negative" on a scanned report or a CSV row without a unit, can be edited again. It opens with "No unit" ticked, a rename saves without touching the unit, and the unit can be set while the marker holds only text results. The unit stays locked once numbers are stored under it, and writing the same unit another way (`mg/dl` for `mg/dL`) no longer counts as a change. (#1393)
|
||||
- On an account with a watch sending heart rate every few seconds, weight, blood pressure and other occasional readings went missing from the MCP metric series, the Coach and the insights, because the heart rate filled every read before they were reached. Each kind of reading now gets its own share, so they all show up again. (#1398)
|
||||
- Right after a large first import, before the database had refreshed its statistics, the dashboard's latest readings and the insight reads could walk through every reading of the account once for each kind of measurement, which on a long history takes seconds or runs into the timeout. Each kind is now looked up on its own whatever the statistics say.
|
||||
- Large Apple Health exports, which the Health app writes as ZIP64 archives, no longer lose their ECGs and workout routes: the reader for those files refused ZIP64 and counted every one as failed.
|
||||
|
||||
### Security
|
||||
|
||||
@@ -55,9 +55,13 @@ describe("dashboard summary reads — latest reading per type", () => {
|
||||
expect(lateral.length).toBeGreaterThan(0);
|
||||
// The type list has to reach SQL as the driving relation, and the
|
||||
// per-type branch has to stop at the first row — without the LIMIT the
|
||||
// LATERAL still reads every row of the type partition.
|
||||
// LATERAL still reads every row of the type partition. Its WHERE and
|
||||
// ORDER BY are the shared live-index descent, which keeps the planner
|
||||
// on the per-type index whatever the statistics say.
|
||||
expect(sql).toMatch(/FROM\s+unnest\(\$\{/);
|
||||
expect(sql).toMatch(/ORDER\s+BY\s+m\."measured_at"\s+DESC\s+LIMIT\s+1/);
|
||||
expect(sql).toMatch(
|
||||
/\$\{liveTypeDescent\(userId,[^}]*"newest",\s*since\)\}\s+LIMIT\s+1/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -332,12 +332,14 @@ describe("GET /api/insights/targets — mood-rollup tier swap", () => {
|
||||
expect(latestCall).toBeDefined();
|
||||
const [strings, ...values] = latestCall as [string[], ...unknown[]];
|
||||
expect(strings.join("?")).not.toContain("DISTINCT ON");
|
||||
expect(values).toContain("user-targets-1");
|
||||
const floor = values.find(
|
||||
// The probe's WHERE is the shared live-index descent, a nested fragment
|
||||
// that carries the account and the floor.
|
||||
const descent = values.find(
|
||||
(v): v is { values: unknown[] } =>
|
||||
typeof v === "object" && v !== null && !Array.isArray(v),
|
||||
);
|
||||
const flooredAt = floor?.values[0];
|
||||
expect(descent?.values).toContain("user-targets-1");
|
||||
const flooredAt = descent?.values.find((v) => v instanceof Date);
|
||||
expect(flooredAt).toBeInstanceOf(Date);
|
||||
const oneYearMs = 365 * 24 * 60 * 60 * 1000;
|
||||
const ageMs = Date.now() - (flooredAt as Date).getTime();
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
import { Prisma, type PrismaClient } from "@/generated/prisma/client";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
import { ROLLUP_FOLD_WINDOW_MS } from "@/lib/rollups/measurement-rollups";
|
||||
import { liveTypeDescent } from "@/lib/measurements/live-type-descent";
|
||||
|
||||
/** v1.4.38 — the single most recent reading per measurement
|
||||
* type, irrespective of age. One row per metric the user has ever
|
||||
@@ -83,6 +84,10 @@ export interface SparklineRow {
|
||||
* is an inner join, so a type with no live reading drops out exactly as
|
||||
* DISTINCT ON dropped it, and `ORDER BY mt."type"` reproduces the enum
|
||||
* ordering the old `ORDER BY m."type"` emitted. Same rows, same order.
|
||||
* The lookup's WHERE and ORDER BY are `liveTypeDescent`'s: written as a
|
||||
* plain equality, an account whose rows arrived after the last ANALYZE had
|
||||
* the planner walk the type-less `(user_id, measured_at)` index through
|
||||
* every row of the account once per type.
|
||||
*
|
||||
* The driving relation is aliased `mt`, not `t`: a one-letter `t` alias
|
||||
* makes `t("type")` in the SQL text look exactly like a `t()` translation
|
||||
@@ -94,9 +99,6 @@ export function readLatestEver(
|
||||
measurementTypes: readonly MeasurementType[],
|
||||
since: Date | null = null,
|
||||
): Promise<LatestEverRow[]> {
|
||||
const floor = since
|
||||
? Prisma.sql`AND m."measured_at" >= ${since}`
|
||||
: Prisma.empty;
|
||||
return db.$queryRaw<LatestEverRow[]>`
|
||||
SELECT
|
||||
mt."type" AS type,
|
||||
@@ -108,11 +110,7 @@ export function readLatestEver(
|
||||
m."value"::double precision AS value,
|
||||
m."measured_at" AS measured_at
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = ${userId}
|
||||
AND m."type" = mt."type"
|
||||
AND m."deleted_at" IS NULL
|
||||
${floor}
|
||||
ORDER BY m."measured_at" DESC
|
||||
${liveTypeDescent(userId, Prisma.sql`mt."type"`, "newest", since)}
|
||||
LIMIT 1
|
||||
) l
|
||||
ORDER BY mt."type"
|
||||
|
||||
@@ -19,10 +19,14 @@ function read(rel: string): string {
|
||||
|
||||
describe("prompt reads exclude tombstoned rows", () => {
|
||||
it("general-status filters its measurement read", () => {
|
||||
// The overview reads through the per-type latest-readings query.
|
||||
// The overview reads through the per-type latest-readings query, which
|
||||
// reads through the per-type newest-rows query.
|
||||
expect(read("../general-status.ts")).toMatch(/readLatestReadingsPerType\(/);
|
||||
expect(read("../recent-readings.ts")).toMatch(
|
||||
/readLatestReadingsPerType[\s\S]*?m\."deleted_at" IS NULL/,
|
||||
/function readLatestReadingsPerType[\s\S]*?readNewestRowsPerType\(/,
|
||||
);
|
||||
expect(read("../../measurements/live-type-descent.ts")).toMatch(
|
||||
/function liveTypeDescent[\s\S]*?m\."deleted_at" IS NULL/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -30,7 +34,7 @@ describe("prompt reads exclude tombstoned rows", () => {
|
||||
// The weight series reads through the per-type newest-rows query.
|
||||
expect(read("../weight-status.ts")).toMatch(/readNewestRowsPerType\(/);
|
||||
expect(read("../../measurements/newest-per-type.ts")).toMatch(
|
||||
/readNewestRowsPerType[\s\S]*?m\."deleted_at" IS NULL/,
|
||||
/function readNewestRowsPerType[\s\S]*?\$\{liveTypeDescent\(/,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
import { prisma } from "@/lib/db";
|
||||
import { annotate } from "@/lib/logging/context";
|
||||
import { readNewestRowsPerType } from "@/lib/measurements/newest-per-type";
|
||||
|
||||
export const RECENT_READINGS = 365;
|
||||
export const RECENT_MIN_DAYS = 30;
|
||||
@@ -70,9 +71,9 @@ export async function readRecentReadings(
|
||||
* the multi-metric overview. It read the newest 5 000 rows across all types
|
||||
* at once, so a dense stream (pulse every few minutes) filled the whole read
|
||||
* within days and every other type fell out of the overview. One index
|
||||
* descent per type now bounds each type on its own. The driving relation is
|
||||
* aliased `mt`, the way `readLatestEver` aliases it, so the i18n call-site
|
||||
* scanner does not read the SQL as a translation call.
|
||||
* descent per type now bounds each type on its own; the descent is
|
||||
* `readNewestRowsPerType`'s, so the statement that keeps the planner on it
|
||||
* exists once.
|
||||
*/
|
||||
export async function readLatestReadingsPerType(
|
||||
userId: string,
|
||||
@@ -80,25 +81,12 @@ export async function readLatestReadingsPerType(
|
||||
perType: number = RECENT_READINGS,
|
||||
): Promise<Array<RecentReading & { type: MeasurementType }>> {
|
||||
if (types.length === 0) return [];
|
||||
const rows = await prisma.$queryRaw<
|
||||
Array<{ type: MeasurementType; value: number; measured_at: Date }>
|
||||
>`
|
||||
SELECT mt."type" AS "type", x."value", x."measured_at"
|
||||
FROM unnest(${[...types]}::text[]) AS mt("type")
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT m."value", m."measured_at"
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = ${userId}
|
||||
AND m."type" = mt."type"::measurement_type
|
||||
AND m."deleted_at" IS NULL
|
||||
ORDER BY m."measured_at" DESC
|
||||
LIMIT ${perType}
|
||||
) x
|
||||
ORDER BY x."measured_at" ASC
|
||||
`;
|
||||
const { rows } = await readNewestRowsPerType(userId, types, {
|
||||
perTypeCap: perType,
|
||||
});
|
||||
return rows.map((r) => ({
|
||||
type: r.type,
|
||||
value: Number(r.value),
|
||||
measuredAt: new Date(r.measured_at),
|
||||
value: r.value,
|
||||
measuredAt: r.measuredAt,
|
||||
}));
|
||||
}
|
||||
@@ -22,6 +22,13 @@ export interface LatestCanonicalRow {
|
||||
* result: with only the lower bound it estimates a third of the type's rows
|
||||
* per probe, and the summed estimate crosses `jit_above_cost`, whose
|
||||
* compilation then costs more than the query itself.
|
||||
*
|
||||
* The latest-instant probe is `liveTypeDescent`
|
||||
* (`src/lib/measurements/live-type-descent.ts`) written out positionally: the
|
||||
* account through a scalar subquery, the type as a closed range it is also
|
||||
* ordered on. As a plain equality, an account whose rows arrived after the
|
||||
* last ANALYZE had the planner walk the type-less `(user_id, measured_at)`
|
||||
* index through every row of the account once per type.
|
||||
*/
|
||||
export function readLatestCanonical(
|
||||
userId: string,
|
||||
@@ -39,11 +46,12 @@ export function readLatestCanonical(
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT date_trunc('day', d."measured_at") AS day
|
||||
FROM measurements d
|
||||
WHERE d."user_id" = $1
|
||||
AND d."type" = mt."type"
|
||||
WHERE d."user_id" = (SELECT $1::text)
|
||||
AND d."type" >= mt."type"
|
||||
AND d."type" <= mt."type"
|
||||
AND d."deleted_at" IS NULL
|
||||
${since ? `AND d."measured_at" >= $3` : ""}
|
||||
ORDER BY d."measured_at" DESC
|
||||
ORDER BY d."type" ASC, d."measured_at" DESC
|
||||
LIMIT 1
|
||||
) last_day
|
||||
CROSS JOIN LATERAL (
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { Prisma } from "@/generated/prisma/client";
|
||||
|
||||
/**
|
||||
* The WHERE and ORDER BY of one type's descent of the live index
|
||||
* `measurements_live_covering_idx (user_id, type, measured_at DESC) WHERE
|
||||
* deleted_at IS NULL`, over the table aliased `m`: `typeSql` names the type,
|
||||
* and the caller adds the LIMIT. The per-type reads that stop at a cap or at
|
||||
* the first row descend through it (`newest-per-type.ts` and the dashboard's
|
||||
* `readLatestEver`); `readLatestCanonical` writes the same statement out in
|
||||
* its positional form.
|
||||
*
|
||||
* The descent is the right plan for every account and every type, but the
|
||||
* planner picks it only when its row estimates say so, and for this read
|
||||
* they cannot be trusted. The type is a lateral parameter, so one generic
|
||||
* guess stands for the densest and the sparsest type alike, and it sits far
|
||||
* below the cap of a dense stream, so the planner expects the LIMIT never to
|
||||
* stop the scan and reads every row of the type into a sort. An account whose
|
||||
* rows arrived after the last ANALYZE is absent from the statistics, so it is
|
||||
* expected to hold about one row, and statistics taken while the table held a
|
||||
* single type say the type condition removes nothing. In both the type-less
|
||||
* `(user_id, measured_at)` index looks as good as the live one, and the
|
||||
* planner walked it once per type through every row of the account. On the
|
||||
* test's account of 9,300 rows that was 33,900 row visits for 6,300 rows
|
||||
* returned, and 1.66 million for `readLiveMeasurementSpan`, which walks it
|
||||
* at both ends of every type.
|
||||
*
|
||||
* So the statement withholds the values those estimates come from:
|
||||
*
|
||||
* - the account arrives through a scalar subquery, so the planner sizes it
|
||||
* as an average account instead of the one row a new account gets;
|
||||
* - the type is matched as a closed range on the one value and ordered on,
|
||||
* so it is estimated by a fixed default rather than by column statistics,
|
||||
* and only an index leading `(user_id, type, measured_at)` returns the
|
||||
* rows in that order. The type-less index would need a full read and a
|
||||
* sort, which it cannot win;
|
||||
* - a cap above one row goes through a scalar subquery too: with a limit
|
||||
* it cannot see, the planner costs the scan as stopping early, which a
|
||||
* sort of every row cannot match. A LIMIT 1 probe needs no help there.
|
||||
*
|
||||
* Each of the three is needed in some state of the statistics, and
|
||||
* `tests/integration/newest-per-type-plans.test.ts` builds each state and
|
||||
* holds the plans to the cap. The range direction of the ORDER BY picks the
|
||||
* live index's own order (`type` ascending, `measured_at` descending) for the
|
||||
* newest end and its backward scan for the oldest end.
|
||||
*/
|
||||
export function liveTypeDescent(
|
||||
userId: string,
|
||||
typeSql: Prisma.Sql,
|
||||
end: "newest" | "oldest",
|
||||
since: Date | null,
|
||||
): Prisma.Sql {
|
||||
const sinceSql =
|
||||
since === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${since}`;
|
||||
const order =
|
||||
end === "newest"
|
||||
? Prisma.sql`ORDER BY m."type" ASC, m."measured_at" DESC`
|
||||
: Prisma.sql`ORDER BY m."type" DESC, m."measured_at" ASC`;
|
||||
return Prisma.sql`
|
||||
WHERE m."user_id" = (SELECT ${userId}::text)
|
||||
AND m."type" >= ${typeSql}
|
||||
AND m."type" <= ${typeSql}
|
||||
AND m."deleted_at" IS NULL
|
||||
${sinceSql}
|
||||
${order}`;
|
||||
}
|
||||
@@ -26,11 +26,13 @@
|
||||
* One statement: one index descent per type on the partial live index
|
||||
* `measurements_live_covering_idx (user_id, type, measured_at DESC) WHERE
|
||||
* deleted_at IS NULL`, each stopping at the cap, so the rows touched track the
|
||||
* rows returned, never the length of a dense stream. The driving relation is
|
||||
* aliased `mt`, as in `readLatestReadingsPerType`, so the i18n call-site
|
||||
* scanner does not read the SQL as a translation call.
|
||||
* rows returned, never the length of a dense stream (`liveTypeDescent` says how
|
||||
* the statement keeps the planner on that path). The driving relation is
|
||||
* aliased `mt` so the i18n call-site scanner does not read the SQL as a
|
||||
* translation call.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import { liveTypeDescent } from "@/lib/measurements/live-type-descent";
|
||||
import {
|
||||
Prisma,
|
||||
type GlucoseContext,
|
||||
@@ -74,8 +76,6 @@ export async function readNewestRowsPerType(
|
||||
}
|
||||
}
|
||||
const since = options.since ?? null;
|
||||
const sinceSql =
|
||||
since === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${since}`;
|
||||
// With a total, each row's rank within its type orders the cut, and the
|
||||
// count per type says whether the cut reached it.
|
||||
const fairShare = totalCap !== undefined;
|
||||
@@ -115,12 +115,8 @@ export async function readNewestRowsPerType(
|
||||
SELECT m."id", m."value", m."measured_at", m."source",
|
||||
m."device_type", m."sleep_stage", m."glucose_context"
|
||||
FROM measurements m
|
||||
WHERE m."user_id" = ${userId}
|
||||
AND m."type" = mt."type"::measurement_type
|
||||
AND m."deleted_at" IS NULL
|
||||
${sinceSql}
|
||||
ORDER BY m."measured_at" DESC
|
||||
LIMIT ${perTypeCap}
|
||||
${liveTypeDescent(userId, Prisma.sql`mt."type"::measurement_type`, "newest", since)}
|
||||
LIMIT (SELECT ${perTypeCap}::int)
|
||||
) x
|
||||
${shareCut}
|
||||
) y
|
||||
@@ -155,16 +151,15 @@ export async function readNewestRowsPerType(
|
||||
* history when `since` is null), so the span a capped read reports is the
|
||||
* account's, not the hours a dense stream fills of the read. One index probe
|
||||
* at each end of every type on the partial live index, each stopping at its
|
||||
* first row. The type-less `(user_id, measured_at)` index would answer in two
|
||||
* probes, but on a table without statistics the planner took the live
|
||||
* partial index instead and sorted every row of the account.
|
||||
* first row (`liveTypeDescent`). The type-less `(user_id, measured_at)` index
|
||||
* would answer in two probes, but on a table without statistics the planner
|
||||
* took the live partial index instead and sorted every row of the account.
|
||||
*/
|
||||
export async function readLiveMeasurementSpan(
|
||||
userId: string,
|
||||
since: Date | null,
|
||||
): Promise<{ oldest: Date | null; newest: Date | null }> {
|
||||
const sinceSql =
|
||||
since === null ? Prisma.empty : Prisma.sql`AND m."measured_at" >= ${since}`;
|
||||
const type = Prisma.sql`mt."type"`;
|
||||
const [row] = await prisma.$queryRaw<
|
||||
Array<{ oldest: Date | null; newest: Date | null }>
|
||||
>`
|
||||
@@ -172,15 +167,13 @@ export async function readLiveMeasurementSpan(
|
||||
FROM unnest(enum_range(NULL::measurement_type)) AS mt("type")
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT m."measured_at" AS "at" FROM measurements m
|
||||
WHERE m."user_id" = ${userId} AND m."type" = mt."type"
|
||||
AND m."deleted_at" IS NULL ${sinceSql}
|
||||
ORDER BY m."measured_at" ASC LIMIT 1
|
||||
${liveTypeDescent(userId, type, "oldest", since)}
|
||||
LIMIT 1
|
||||
) o ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT m."measured_at" AS "at" FROM measurements m
|
||||
WHERE m."user_id" = ${userId} AND m."type" = mt."type"
|
||||
AND m."deleted_at" IS NULL ${sinceSql}
|
||||
ORDER BY m."measured_at" DESC LIMIT 1
|
||||
${liveTypeDescent(userId, type, "newest", since)}
|
||||
LIMIT 1
|
||||
) n ON true
|
||||
`;
|
||||
return {
|
||||
|
||||
@@ -18,15 +18,14 @@
|
||||
* snapshot. The scoped and the full extraction must also agree block for
|
||||
* block, which is what the single-metric read promises.
|
||||
*
|
||||
* The per-type read and the span probe are also run under `EXPLAIN ANALYZE`
|
||||
* on the unanalysed table: the rows their plans touch must track the rows
|
||||
* they return, not the size of the dense stream, so the cap still bounds the
|
||||
* I/O it exists for.
|
||||
* That the per-type read and the span probe touch only the rows they return,
|
||||
* whatever the planner statistics say, is held in
|
||||
* `newest-per-type-plans.test.ts`, which builds each statistics state on
|
||||
* purpose.
|
||||
*/
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
import { prisma as appPrisma } from "@/lib/db";
|
||||
import { MCP_TOOLS } from "@/lib/mcp/tools";
|
||||
import type { McpAuthContext } from "@/lib/mcp/auth";
|
||||
import {
|
||||
@@ -38,10 +37,6 @@ import {
|
||||
extractFeatures,
|
||||
type AggregatedFeatures,
|
||||
} from "@/lib/insights/features";
|
||||
import {
|
||||
readLiveMeasurementSpan,
|
||||
readNewestRowsPerType,
|
||||
} from "@/lib/measurements/newest-per-type";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
|
||||
vi.mock("@/lib/jobs/boss-instance", () => ({
|
||||
@@ -219,121 +214,4 @@ describe("a dense stream beside sparse history", () => {
|
||||
})) as { present: boolean; data?: { section?: unknown } };
|
||||
expect(JSON.stringify(single.data?.section)).toBe(JSON.stringify(weight));
|
||||
});
|
||||
|
||||
it("reads each type by its own index descent, bounded by the cap", async () => {
|
||||
const cap = 6_000;
|
||||
const types: MeasurementType[] = [
|
||||
"PULSE",
|
||||
"WEIGHT",
|
||||
"BLOOD_PRESSURE_SYS",
|
||||
"BLOOD_PRESSURE_DIA",
|
||||
];
|
||||
const [result, plans] = await withPlans(() =>
|
||||
readNewestRowsPerType(userId, types, { perTypeCap: cap }),
|
||||
);
|
||||
const counts = new Map<string, number>();
|
||||
for (const row of result.rows) {
|
||||
counts.set(row.type, (counts.get(row.type) ?? 0) + 1);
|
||||
}
|
||||
expect(Object.fromEntries(counts)).toEqual({
|
||||
PULSE: cap,
|
||||
WEIGHT: SPARSE_DAYS,
|
||||
BLOOD_PRESSURE_SYS: SPARSE_DAYS * 2,
|
||||
BLOOD_PRESSURE_DIA: SPARSE_DAYS * 2,
|
||||
});
|
||||
expect(result.cappedTypes).toEqual(["PULSE"]);
|
||||
// Ascending, the order every consumer folds in.
|
||||
for (let i = 1; i < result.rows.length; i++) {
|
||||
expect(result.rows[i].measuredAt.getTime()).toBeGreaterThanOrEqual(
|
||||
result.rows[i - 1].measuredAt.getTime(),
|
||||
);
|
||||
}
|
||||
|
||||
expect(plans).toHaveLength(1);
|
||||
expect(JSON.stringify(plans[0])).not.toContain("Seq Scan");
|
||||
// Every row the plan's measurement scans returned or threw away, over all
|
||||
// their loops: a plan that walked the dense stream would touch all 9,000
|
||||
// pulse rows, the descent stops at the cap.
|
||||
expect(scanRowVisits(plans[0])).toBeLessThanOrEqual(
|
||||
result.rows.length + types.length,
|
||||
);
|
||||
});
|
||||
|
||||
it("takes the overall span from index probes, not from the rows", async () => {
|
||||
const [span, plans] = await withPlans(() =>
|
||||
readLiveMeasurementSpan(userId, new Date(Date.now() - 400 * DAY_MS)),
|
||||
);
|
||||
expect(span.oldest).not.toBeNull();
|
||||
expect(span.newest).not.toBeNull();
|
||||
expect(
|
||||
(span.newest!.getTime() - span.oldest!.getTime()) / DAY_MS,
|
||||
).toBeGreaterThan(SPARSE_DAYS);
|
||||
expect(plans).toHaveLength(1);
|
||||
expect(JSON.stringify(plans[0])).not.toContain("Seq Scan");
|
||||
// One row at each end of each of the four seeded types.
|
||||
expect(scanRowVisits(plans[0])).toBeLessThanOrEqual(8);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Run `read`, then run every `$queryRaw` it issued again under `EXPLAIN
|
||||
* ANALYZE` and hand back the plans beside the reader's own result.
|
||||
*/
|
||||
async function withPlans<T>(read: () => Promise<T>): Promise<[T, unknown[]]> {
|
||||
const original = appPrisma.$queryRaw.bind(appPrisma) as (
|
||||
...args: unknown[]
|
||||
) => Promise<unknown>;
|
||||
const statements: Array<[TemplateStringsArray, unknown[]]> = [];
|
||||
vi.spyOn(appPrisma, "$queryRaw").mockImplementation(((
|
||||
strings: TemplateStringsArray,
|
||||
...values: unknown[]
|
||||
) => {
|
||||
statements.push([strings, values]);
|
||||
return original(strings, ...values);
|
||||
}) as never);
|
||||
let result: T;
|
||||
try {
|
||||
result = await read();
|
||||
} finally {
|
||||
vi.restoreAllMocks();
|
||||
}
|
||||
const plans: unknown[] = [];
|
||||
for (const [strings, values] of statements) {
|
||||
const head = "EXPLAIN (ANALYZE, FORMAT JSON) ";
|
||||
const explain = Object.assign([head + strings[0], ...strings.slice(1)], {
|
||||
raw: [head + strings.raw[0], ...strings.raw.slice(1)],
|
||||
});
|
||||
const [{ "QUERY PLAN": plan }] = await appPrisma.$queryRaw<
|
||||
Array<{ "QUERY PLAN": unknown }>
|
||||
>(explain, ...values);
|
||||
plans.push(plan);
|
||||
}
|
||||
return [result, plans];
|
||||
}
|
||||
|
||||
interface PlanNode {
|
||||
"Node Type"?: string;
|
||||
"Relation Name"?: string;
|
||||
"Actual Rows"?: number;
|
||||
"Actual Loops"?: number;
|
||||
"Rows Removed by Filter"?: number;
|
||||
"Rows Removed by Index Recheck"?: number;
|
||||
Plans?: PlanNode[];
|
||||
Plan?: PlanNode;
|
||||
}
|
||||
|
||||
/** Rows the scans of `measurements` produced or discarded, over all loops. */
|
||||
function scanRowVisits(plan: unknown): number {
|
||||
const walk = (node: PlanNode): number => {
|
||||
const own =
|
||||
node["Relation Name"] === "measurements"
|
||||
? ((node["Actual Rows"] ?? 0) +
|
||||
(node["Rows Removed by Filter"] ?? 0) +
|
||||
(node["Rows Removed by Index Recheck"] ?? 0)) *
|
||||
(node["Actual Loops"] ?? 1)
|
||||
: 0;
|
||||
return own + (node.Plans ?? []).reduce((sum, p) => sum + walk(p), 0);
|
||||
};
|
||||
const root = (plan as PlanNode[])[0];
|
||||
return walk(root.Plan ?? root);
|
||||
}
|
||||
@@ -0,0 +1,407 @@
|
||||
/**
|
||||
* The per-type reads must descend the live index once per type and stop at
|
||||
* the cap, whatever the planner statistics say.
|
||||
*
|
||||
* `readNewestRowsPerType`, `readLiveMeasurementSpan`,
|
||||
* `readLatestReadingsPerType` and the dashboard's latest-reading probes
|
||||
* (`readLatestEver`, `readLatestCanonical`) exist so the rows a dense stream
|
||||
* makes the database touch track the rows returned, not the length of the
|
||||
* stream. That
|
||||
* only holds while the planner picks the per-type descent, and its pick
|
||||
* follows its row estimates. The estimates go wrong in exactly the states a
|
||||
* dense account passes through, so each state is built here on purpose and
|
||||
* the plans are held to the bound in every one of them:
|
||||
*
|
||||
* - no statistics at all (a fresh install);
|
||||
* - statistics gathered before the account's rows arrived, over other
|
||||
* accounts with many types: the account is absent from them, so the
|
||||
* planner expects it to hold almost nothing (a first sync on a running
|
||||
* instance, until the next ANALYZE);
|
||||
* - the same, but the other accounts held a single type, so the type
|
||||
* condition looks as if it removes nothing;
|
||||
* - analysed with the account among the others: its per-type estimate is
|
||||
* one generic share of its rows, far below the cap of a dense type.
|
||||
*
|
||||
* Under the old statement the second and third state walked the
|
||||
* type-less `(user_id, measured_at)` index once per type through every row
|
||||
* of the account, and the fourth read every row of every type into a sort.
|
||||
* The full suite reached the second state by accident (another file's
|
||||
* statistics survive its TRUNCATE), which is how the bound came to fail there
|
||||
* and pass alone. Autovacuum is off for the file so no state drifts into
|
||||
* another mid-test, and every table stays under the ANALYZE sample size, so
|
||||
* the statistics are the whole table rather than a random sample of it.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { getPrismaClient, truncateAllTables } from "./setup";
|
||||
import { prisma as appPrisma } from "@/lib/db";
|
||||
import { readLatestEver } from "@/lib/dashboard/summary-reads";
|
||||
import { readLatestReadingsPerType } from "@/lib/insights/recent-readings";
|
||||
import { readLatestCanonical } from "@/lib/measurements/latest-canonical";
|
||||
import {
|
||||
readLiveMeasurementSpan,
|
||||
readNewestRowsPerType,
|
||||
} from "@/lib/measurements/newest-per-type";
|
||||
import { measurementTypeEnum } from "@/lib/validations/measurement";
|
||||
import type { MeasurementType } from "@/generated/prisma/client";
|
||||
|
||||
vi.mock("@/lib/jobs/boss-instance", () => ({
|
||||
getGlobalBoss: vi.fn(() => null),
|
||||
}));
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const ACCOUNT = "dense-account";
|
||||
/** Above the 6,000-row cap on its own. */
|
||||
const PULSE_ROWS = 9_000;
|
||||
const SPARSE_DAYS = 60;
|
||||
const OTHER_ACCOUNTS = 20;
|
||||
/** Rows per other account; with the dense account, below 30,000 in all. */
|
||||
const OTHER_ROWS = 1_000;
|
||||
const CAP = 6_000;
|
||||
const TYPES: MeasurementType[] = [
|
||||
"PULSE",
|
||||
"WEIGHT",
|
||||
"BLOOD_PRESSURE_SYS",
|
||||
"BLOOD_PRESSURE_DIA",
|
||||
];
|
||||
|
||||
type StatisticsState = "none" | "predate-many" | "predate-one" | "analysed";
|
||||
|
||||
const prisma = getPrismaClient();
|
||||
/** The overview reads every type the API accepts. */
|
||||
const MEASUREMENT_TYPES = measurementTypeEnum.options;
|
||||
|
||||
/** The dense account: heart rate every ten seconds, sparse history behind. */
|
||||
async function seedAccount(): Promise<void> {
|
||||
await prisma.$executeRawUnsafe(
|
||||
`INSERT INTO measurements
|
||||
(id, user_id, type, value, unit, source, measured_at, updated_at)
|
||||
SELECT 'p' || i, $1, 'PULSE'::measurement_type, 60 + (i % 25), 'bpm',
|
||||
'APPLE_HEALTH'::measurement_source,
|
||||
now() - interval '1 minute' - i * interval '10 seconds', now()
|
||||
FROM generate_series(0, ${PULSE_ROWS - 1}) i
|
||||
UNION ALL
|
||||
SELECT 'w' || d, $1, 'WEIGHT'::measurement_type, 81, 'kg',
|
||||
'WITHINGS'::measurement_source,
|
||||
now() - d * interval '1 day' - interval '2 hours', now()
|
||||
FROM generate_series(2, ${SPARSE_DAYS + 1}) d
|
||||
UNION ALL
|
||||
SELECT t || d || '-' || h, $1, t::measurement_type, 120, 'mmHg',
|
||||
'WITHINGS'::measurement_source,
|
||||
now() - d * interval '1 day' - h * interval '1 hour', now()
|
||||
FROM generate_series(2, ${SPARSE_DAYS + 1}) d,
|
||||
unnest(ARRAY[8, 20]) h,
|
||||
unnest(ARRAY['BLOOD_PRESSURE_SYS', 'BLOOD_PRESSURE_DIA']) t`,
|
||||
ACCOUNT,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Other accounts' rows: twenty types each, or WEIGHT alone. Twenty types
|
||||
* cycle through the enum so none of them is one the dense account holds
|
||||
* more of than the rest.
|
||||
*/
|
||||
async function seedOthers(oneType: boolean): Promise<void> {
|
||||
await prisma.$executeRawUnsafe(
|
||||
`INSERT INTO measurements
|
||||
(id, user_id, type, value, unit, source, measured_at, updated_at)
|
||||
SELECT 'o' || u || '-' || i, 'other-' || u,
|
||||
${
|
||||
oneType
|
||||
? `'WEIGHT'::measurement_type`
|
||||
: `(enum_range(NULL::measurement_type))[1 + (i % 20)]`
|
||||
},
|
||||
1, 'x', 'MANUAL', now() - i * interval '1 hour', now()
|
||||
FROM generate_series(1, ${OTHER_ACCOUNTS}) u,
|
||||
generate_series(0, ${OTHER_ROWS - 1}) i`,
|
||||
);
|
||||
}
|
||||
|
||||
async function buildState(state: StatisticsState): Promise<void> {
|
||||
await prisma.$executeRawUnsafe("TRUNCATE measurements CASCADE");
|
||||
await prisma.$executeRawUnsafe(
|
||||
"DELETE FROM pg_statistic WHERE starelid = 'measurements'::regclass",
|
||||
);
|
||||
switch (state) {
|
||||
case "none":
|
||||
await seedAccount();
|
||||
return;
|
||||
case "predate-many":
|
||||
case "predate-one":
|
||||
await seedOthers(state === "predate-one");
|
||||
await prisma.$executeRawUnsafe("ANALYZE measurements");
|
||||
await seedAccount();
|
||||
return;
|
||||
case "analysed":
|
||||
await seedOthers(false);
|
||||
await seedAccount();
|
||||
await prisma.$executeRawUnsafe("ANALYZE measurements");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/** What the statistics say, so a state cannot pass for one it is not. */
|
||||
async function readStatistics(): Promise<{
|
||||
columns: number;
|
||||
accountKnown: boolean;
|
||||
typesKnown: number;
|
||||
}> {
|
||||
const [row] = await prisma.$queryRawUnsafe<
|
||||
Array<{ columns: number; account_known: boolean; types_known: number }>
|
||||
>(
|
||||
`SELECT
|
||||
(SELECT COUNT(*) FROM pg_statistic
|
||||
WHERE starelid = 'measurements'::regclass)::int AS columns,
|
||||
COALESCE((SELECT $1 = ANY (most_common_vals::text::text[])
|
||||
FROM pg_stats WHERE tablename = 'measurements'
|
||||
AND attname = 'user_id'), false) AS account_known,
|
||||
COALESCE((SELECT array_length(most_common_vals::text::text[], 1)
|
||||
FROM pg_stats WHERE tablename = 'measurements'
|
||||
AND attname = 'type'), 0)::int AS types_known`,
|
||||
ACCOUNT,
|
||||
);
|
||||
return {
|
||||
columns: row.columns,
|
||||
accountKnown: row.account_known,
|
||||
typesKnown: row.types_known,
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
await truncateAllTables(prisma);
|
||||
await prisma.$executeRawUnsafe(
|
||||
"ALTER TABLE measurements SET (autovacuum_enabled = false)",
|
||||
);
|
||||
await prisma.user.createMany({
|
||||
data: [
|
||||
{ id: ACCOUNT, username: ACCOUNT },
|
||||
...Array.from({ length: OTHER_ACCOUNTS }, (_, i) => ({
|
||||
id: `other-${i + 1}`,
|
||||
username: `other-${i + 1}`,
|
||||
})),
|
||||
],
|
||||
});
|
||||
}, 120_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await prisma.$executeRawUnsafe(
|
||||
"ALTER TABLE measurements RESET (autovacuum_enabled)",
|
||||
);
|
||||
});
|
||||
|
||||
describe.each<
|
||||
[
|
||||
StatisticsState,
|
||||
{ columns: boolean; accountKnown: boolean; typesKnown: number | null },
|
||||
]
|
||||
>([
|
||||
["none", { columns: false, accountKnown: false, typesKnown: 0 }],
|
||||
["predate-many", { columns: true, accountKnown: false, typesKnown: 20 }],
|
||||
["predate-one", { columns: true, accountKnown: false, typesKnown: 1 }],
|
||||
["analysed", { columns: true, accountKnown: true, typesKnown: null }],
|
||||
])("per-type reads with statistics: %s", (state, expected) => {
|
||||
beforeAll(async () => {
|
||||
await buildState(state);
|
||||
}, 120_000);
|
||||
|
||||
it("runs against the statistics the state names", async () => {
|
||||
const stats = await readStatistics();
|
||||
expect(stats.columns > 0).toBe(expected.columns);
|
||||
expect(stats.accountKnown).toBe(expected.accountKnown);
|
||||
if (expected.typesKnown !== null) {
|
||||
expect(stats.typesKnown).toBe(expected.typesKnown);
|
||||
} else {
|
||||
expect(stats.typesKnown).toBeGreaterThan(TYPES.length);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
["the whole history", null],
|
||||
["a 400-day window", 400],
|
||||
] as const)(
|
||||
"reads each type by its own descent, bounded by the cap: %s",
|
||||
async (_label, sinceDays) => {
|
||||
const since =
|
||||
sinceDays === null ? null : new Date(Date.now() - sinceDays * DAY_MS);
|
||||
const [result, plans] = await withPlans(() =>
|
||||
readNewestRowsPerType(ACCOUNT, TYPES, { perTypeCap: CAP, since }),
|
||||
);
|
||||
const counts = new Map<string, number>();
|
||||
for (const row of result.rows) {
|
||||
counts.set(row.type, (counts.get(row.type) ?? 0) + 1);
|
||||
}
|
||||
expect(Object.fromEntries(counts)).toEqual({
|
||||
PULSE: CAP,
|
||||
WEIGHT: SPARSE_DAYS,
|
||||
BLOOD_PRESSURE_SYS: SPARSE_DAYS * 2,
|
||||
BLOOD_PRESSURE_DIA: SPARSE_DAYS * 2,
|
||||
});
|
||||
expect(result.cappedTypes).toEqual(["PULSE"]);
|
||||
expectBounded(plans, result.rows.length + TYPES.length);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the descent under a shared total", async () => {
|
||||
const [result, plans] = await withPlans(() =>
|
||||
readNewestRowsPerType(ACCOUNT, TYPES, {
|
||||
perTypeCap: CAP,
|
||||
totalCap: 2_000,
|
||||
}),
|
||||
);
|
||||
expect(result.rows).toHaveLength(2_000);
|
||||
// The descent reads at most the cap per type before the total cuts.
|
||||
expectBounded(plans, CAP + SPARSE_DAYS * 5 + TYPES.length);
|
||||
});
|
||||
|
||||
it("takes the span from one probe at each end of each type", async () => {
|
||||
const [span, plans] = await withPlans(() =>
|
||||
readLiveMeasurementSpan(ACCOUNT, new Date(Date.now() - 400 * DAY_MS)),
|
||||
);
|
||||
expect(span.oldest).not.toBeNull();
|
||||
expect(span.newest).not.toBeNull();
|
||||
expect(
|
||||
(span.newest!.getTime() - span.oldest!.getTime()) / DAY_MS,
|
||||
).toBeGreaterThan(SPARSE_DAYS);
|
||||
// One row at each end of each of the four seeded types.
|
||||
expectBounded(plans, 2 * TYPES.length);
|
||||
});
|
||||
|
||||
it("bounds the overview's latest readings of every type", async () => {
|
||||
const perType = 365;
|
||||
const [rows, plans] = await withPlans(() =>
|
||||
readLatestReadingsPerType(ACCOUNT, [...MEASUREMENT_TYPES], perType),
|
||||
);
|
||||
expect(rows).toHaveLength(perType + SPARSE_DAYS * 5);
|
||||
expectBounded(plans, rows.length + MEASUREMENT_TYPES.length);
|
||||
});
|
||||
|
||||
it("finds the dashboard's latest reading of every type by one probe each", async () => {
|
||||
const [rows, plans] = await withPlans(() =>
|
||||
readLatestEver(appPrisma, ACCOUNT, [...MEASUREMENT_TYPES]),
|
||||
);
|
||||
expect(rows.map((r) => r.type).sort()).toEqual([...TYPES].sort());
|
||||
expectBounded(plans, TYPES.length);
|
||||
});
|
||||
|
||||
it("finds the dashboard's canonical latest instant by one probe per type", async () => {
|
||||
const [rows, plans] = await withPlans(() =>
|
||||
readLatestCanonical(ACCOUNT, null),
|
||||
);
|
||||
expect(rows.map((r) => r.type).sort()).toEqual([...TYPES].sort());
|
||||
// Only the latest-instant probe (`d`) is held here; the read of that
|
||||
// one day that follows ranks the day's rows by source and is not a
|
||||
// per-type descent.
|
||||
expectBounded(plans, TYPES.length, "d");
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* One statement, no sequential scan, and every row its measurement scans
|
||||
* (those aliased `alias`, when given) produced or threw away within `bound`.
|
||||
* A plan with no such scan fails rather than passing on zero.
|
||||
*/
|
||||
function expectBounded(plans: unknown[], bound: number, alias?: string): void {
|
||||
expect(plans).toHaveLength(1);
|
||||
const text = JSON.stringify(plans[0]);
|
||||
expect(text).not.toContain("Seq Scan");
|
||||
const { scans, visits } = scanRowVisits(plans[0], alias);
|
||||
expect(scans).toBeGreaterThan(0);
|
||||
expect(
|
||||
visits,
|
||||
`${visits} row visits against a bound of ${bound}:\n${JSON.stringify(plans[0], null, 1)}`,
|
||||
).toBeLessThanOrEqual(bound);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `read`, then run every `$queryRaw` and `$queryRawUnsafe` it issued
|
||||
* again under `EXPLAIN ANALYZE` and hand back the plans beside the reader's
|
||||
* own result.
|
||||
*/
|
||||
async function withPlans<T>(read: () => Promise<T>): Promise<[T, unknown[]]> {
|
||||
const head = "EXPLAIN (ANALYZE, FORMAT JSON) ";
|
||||
const tagged = appPrisma.$queryRaw.bind(appPrisma) as (
|
||||
...args: unknown[]
|
||||
) => Promise<unknown>;
|
||||
const unsafe = appPrisma.$queryRawUnsafe.bind(appPrisma) as (
|
||||
...args: unknown[]
|
||||
) => Promise<unknown>;
|
||||
const explains: Array<() => Promise<Array<{ "QUERY PLAN": unknown }>>> = [];
|
||||
vi.spyOn(appPrisma, "$queryRaw").mockImplementation(((
|
||||
strings: TemplateStringsArray,
|
||||
...values: unknown[]
|
||||
) => {
|
||||
const explain = Object.assign([head + strings[0], ...strings.slice(1)], {
|
||||
raw: [head + strings.raw[0], ...strings.raw.slice(1)],
|
||||
});
|
||||
explains.push(
|
||||
() =>
|
||||
tagged(explain, ...values) as Promise<Array<{ "QUERY PLAN": unknown }>>,
|
||||
);
|
||||
return tagged(strings, ...values);
|
||||
}) as never);
|
||||
vi.spyOn(appPrisma, "$queryRawUnsafe").mockImplementation(((
|
||||
sql: string,
|
||||
...values: unknown[]
|
||||
) => {
|
||||
explains.push(
|
||||
() =>
|
||||
unsafe(head + sql, ...values) as Promise<
|
||||
Array<{ "QUERY PLAN": unknown }>
|
||||
>,
|
||||
);
|
||||
return unsafe(sql, ...values);
|
||||
}) as never);
|
||||
let result: T;
|
||||
try {
|
||||
result = await read();
|
||||
} finally {
|
||||
vi.restoreAllMocks();
|
||||
}
|
||||
const plans: unknown[] = [];
|
||||
for (const explain of explains) {
|
||||
const [{ "QUERY PLAN": plan }] = await explain();
|
||||
plans.push(plan);
|
||||
}
|
||||
return [result, plans];
|
||||
}
|
||||
|
||||
interface PlanNode {
|
||||
"Node Type"?: string;
|
||||
"Relation Name"?: string;
|
||||
Alias?: string;
|
||||
"Actual Rows"?: number;
|
||||
"Actual Loops"?: number;
|
||||
"Rows Removed by Filter"?: number;
|
||||
"Rows Removed by Index Recheck"?: number;
|
||||
Plans?: PlanNode[];
|
||||
Plan?: PlanNode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rows the scans of `measurements` (only those aliased `alias`, when given)
|
||||
* produced or discarded, over all loops.
|
||||
*/
|
||||
function scanRowVisits(
|
||||
plan: unknown,
|
||||
alias?: string,
|
||||
): { scans: number; visits: number } {
|
||||
let scans = 0;
|
||||
const walk = (node: PlanNode): number => {
|
||||
let own = 0;
|
||||
if (
|
||||
node["Relation Name"] === "measurements" &&
|
||||
(alias === undefined || node.Alias === alias)
|
||||
) {
|
||||
scans += 1;
|
||||
own =
|
||||
((node["Actual Rows"] ?? 0) +
|
||||
(node["Rows Removed by Filter"] ?? 0) +
|
||||
(node["Rows Removed by Index Recheck"] ?? 0)) *
|
||||
(node["Actual Loops"] ?? 1);
|
||||
}
|
||||
return own + (node.Plans ?? []).reduce((sum, p) => sum + walk(p), 0);
|
||||
};
|
||||
const root = (plan as PlanNode[])[0];
|
||||
const visits = walk(root.Plan ?? root);
|
||||
return { scans, visits };
|
||||
}
|
||||
Reference in new issue
Block a user