tom/HealthLog
1
0 Fork 0
mirror of https://github.com/MBombeck/HealthLog.git synced 2026-11-09 13:01:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(measurements): fold per-source days without joining a regrouping back

Browse Source 
Three readers grouped the same rows twice and joined the second grouping
back to the first: the live buckets joined each hourly-mean day's hour
means to the per-source days, the all-time extremes joined the day
weights the same way, and the doctor report's dense buckets joined each
bucket's latest reading and hour means to the bucket aggregates. On a
measurements table without planner statistics (a new install, or an
import under the autoanalyze threshold) the planner estimated one row on
each side and ran every one of those joins as a nested loop that re-ran
the inner grouping for each outer row.

On ten years of readings over six types (51,000 rows, no statistics) the
dense buckets and the every-type live buckets ran into the 60 s statement
timeout and the all-time extremes took 11 s. With statistics all three
answered in 0.1 to 0.2 s.

Each is now one chain of folds over one scan. The live buckets fold the
readings per local hour and the hours per local day, carrying the day of
hours beside the counts. The all-time extremes take the day weights from
the same windowed rows they count, the shape the pre-fold remainder
already has. The dense buckets carry each bucket's latest reading and each
hour's count and mean on the rows as window values, and one grouping folds
them all. Same fixture without statistics: 0.2 s, 0.1 s and 0.1 s, the same
as with them.

A parity suite runs the joined statements as the reference against the
readers across ladders, zones with whole, half-hour and negative offsets,
grains, overlapping and off-ladder sources, glucose contexts and device
types, tombstones and both DST changes; sums and means agree to
floating-point summation order, everything else exactly. The
unanalysed-table guard now runs the three readers under EXPLAIN ANALYZE:
the joins measured 200 to 3,000 row visits per row stored, the folds about
10.

The dense-bucket loader drops under the 200-line function cap, so its
baseline entry goes.
This commit is contained in:
Marc-André Bombeck committed 2026-10-25 06:27:44 +01:00
1 parent 6d338760d8
commit 4e79098cc0
6 files changed
+965 -148
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/__tests__/baselines/source-function-lines.json
+2 -3
View File
@@ -1,7 +1,7 @@
{
"$comment": "Production functions over 200 lines, keyed \"file :: name\", and their length. Maintained by src/__tests__/source-size-ratchet.test.ts.",
"count": 349,
"total": 113048,
"count": 348,
"total": 112838,
"entries": {
"app/achievements/page.tsx :: AchievementsPage": 206,
"app/api/admin/backups/upload/route.ts :: apiHandler(arg)": 314,
@@ -275,7 +275,6 @@
"lib/doctor-report-pdf/measurements-charts-section.ts :: buildMeasurementsChartsSection": 384,
"lib/doctor-report-pdf/medication-mood-wellness-section.ts :: buildMedicationMoodWellnessSection": 509,
"lib/doctor-report/collect.ts :: collectDoctorReportData": 667,
"lib/doctor-report/dense-buckets.ts :: loadDenseMeasurementBuckets": 210,
"lib/documents/fenced-chat.ts :: streamFencedReply": 309,
"lib/documents/ingest.ts :: ingestDocument": 272,
"lib/export/backup-json-scan.ts :: scanBackupJson": 225,
src/lib/doctor-report/dense-buckets.ts
+32 -80
View File
@@ -81,6 +81,13 @@ export async function loadDenseMeasurementBuckets(params: {
const hourlyMeanType = Prisma.raw(isHourlyMeanTypeSql('"type"'));
// Each hour's weight under the active pulse day-mean method (code-built).
const hourWeight = Prisma.raw(hourWeightSql("n"));
// One grouping over one scan, no join. The latest reading and the hour
// means used to be groupings of their own, joined back to the buckets; on
// a measurements table without planner statistics (a new install, an
// import under the autoanalyze threshold) those joins ran as nested loops
// that re-ran a grouping for every bucket, and a long report window ran
// into the statement timeout. `tests/integration/per-source-fold-parity.test.ts`
// holds the joined statement as the reference.
return prisma.$queryRaw<DenseMeasurementBucket[]>`
WITH localized AS (
SELECT
@@ -106,6 +113,9 @@ export async function loadDenseMeasurementBuckets(params: {
AND m."deleted_at" IS NULL
AND m."type" IN (${typeList})
),
-- Every row carries its bucket's latest reading and its local hour's
-- reading count and mean, with one row per hour marked, so the one
-- grouping below folds them all.
scored AS (
SELECT
localized.*,
@@ -113,72 +123,18 @@ export async function loadDenseMeasurementBuckets(params: {
WHEN "value" > 0
THEN 1.509 * (power(ln("value"), 1.084) - 5.381)
ELSE NULL
END AS risk_transform
END AS risk_transform,
FIRST_VALUE("value") OVER (
PARTITION BY local_day, "type", "source", "device_type", "glucose_context"
ORDER BY "measured_at" DESC, "id" DESC
) AS latest_value,
COUNT(*) OVER hour AS n,
AVG("value") OVER hour AS hour_mean,
ROW_NUMBER() OVER hour = 1 AS hour_first
FROM localized
),
latest_rows AS (
SELECT DISTINCT ON (
local_day,
"type",
"source",
"device_type",
"glucose_context"
WINDOW hour AS (
PARTITION BY local_day, local_hour, "type", "source", "device_type", "glucose_context"
)
local_day,
"type",
"source",
"device_type",
"glucose_context",
"measured_at" AS latest_at,
"value" AS latest_value
FROM localized
ORDER BY
local_day,
"type",
"source",
"device_type",
"glucose_context",
"measured_at" DESC,
"id" DESC
),
-- The hourly-mean day (pulse): each local hour's mean and reading
-- count, then the weighted sum of the means and the sum of the hour
-- weights per bucket. Empty for every other type.
hour_means AS (
SELECT
local_day,
"type",
"source",
"device_type",
"glucose_context",
AVG("value") AS hour_mean,
COUNT(*) AS n
FROM localized
WHERE ${hourlyMeanType}
GROUP BY
local_day,
local_hour,
"type",
"source",
"device_type",
"glucose_context"
),
hour_agg AS (
SELECT
local_day,
"type",
"source",
"device_type",
"glucose_context",
SUM(hour_mean * ${hourWeight})::double precision AS hour_weighted_sum,
SUM(${hourWeight})::double precision AS hour_weight
FROM hour_means
GROUP BY
local_day,
"type",
"source",
"device_type",
"glucose_context"
),
grouped AS (
SELECT
@@ -188,6 +144,14 @@ export async function loadDenseMeasurementBuckets(params: {
"device_type",
"glucose_context",
COUNT(*)::int AS sample_count,
MAX("measured_at") AS latest_at,
MAX(latest_value)::double precision AS latest_value,
SUM(hour_mean * ${hourWeight})
FILTER (WHERE hour_first AND ${hourlyMeanType})::double precision
AS hour_weighted_sum,
SUM(${hourWeight})
FILTER (WHERE hour_first AND ${hourlyMeanType})::double precision
AS hour_weight,
SUM("value")::double precision AS sum_value,
MIN("value")::double precision AS min_value,
MAX("value")::double precision AS max_value,
@@ -242,8 +206,8 @@ export async function loadDenseMeasurementBuckets(params: {
grouped.sum_value AS "sumValue",
grouped.min_value AS "minValue",
grouped.max_value AS "maxValue",
latest_rows.latest_at AS "latestAt",
latest_rows.latest_value::double precision AS "latestValue",
grouped.latest_at AS "latestAt",
grouped.latest_value AS "latestValue",
grouped.clinical_count AS "clinicalCount",
grouped.clinical_sum AS "clinicalSum",
grouped.clinical_sum_squares AS "clinicalSumSquares",
@@ -256,21 +220,9 @@ export async function loadDenseMeasurementBuckets(params: {
grouped.clinical_tar2_count AS "clinicalTar2Count",
grouped.clinical_low_risk_sum AS "clinicalLowRiskSum",
grouped.clinical_high_risk_sum AS "clinicalHighRiskSum",
hour_agg.hour_weighted_sum AS "hourWeightedSum",
hour_agg.hour_weight AS "hourWeight"
grouped.hour_weighted_sum AS "hourWeightedSum",
grouped.hour_weight AS "hourWeight"
FROM grouped
INNER JOIN latest_rows
ON latest_rows.local_day = grouped.local_day
AND latest_rows."type" = grouped."type"
AND latest_rows."source" = grouped."source"
AND latest_rows."device_type" IS NOT DISTINCT FROM grouped."device_type"
AND latest_rows."glucose_context" IS NOT DISTINCT FROM grouped."glucose_context"
LEFT JOIN hour_agg
ON hour_agg.local_day = grouped.local_day
AND hour_agg."type" = grouped."type"
AND hour_agg."source" = grouped."source"
AND hour_agg."device_type" IS NOT DISTINCT FROM grouped."device_type"
AND hour_agg."glucose_context" IS NOT DISTINCT FROM grouped."glucose_context"
ORDER BY "bucketStart" ASC, grouped."type" ASC
`;
src/lib/insights/feature-blocks.ts
+22 -31
View File
@@ -12,11 +12,7 @@
import { prisma } from "@/lib/db";
import { Prisma } from "@/generated/prisma/client";
import { buildSourceRankCase } from "@/lib/analytics/source-rank-sql";
import {
dayWeightedRowsSql,
foldMeanSql,
isHourlyMeanTypeSql,
} from "@/lib/measurements/day-mean";
import { dayWeightedRowsSql, foldMeanSql } from "@/lib/measurements/day-mean";
import {
loadUserSourcePriority,
utcBucketEnd,
@@ -207,9 +203,18 @@ export async function readAllTimeExtremes(
ORDER BY r."type", r."bucket_start", (${rank("r")}), r."source"::text
) tier`;
// A day of pulse is the mean of its hours' means and the full history the
// mean of its days (`day-mean.ts`). `weighted` carries, per source and day,
// the day-weighted sum and the day's weight (one) for those types only;
// every other type keeps the reading-weighted mean below, unchanged.
// mean of its days (`day-mean.ts`). `per_source` carries, per source and
// day, the day-weighted sum and the day's weight (one) beside the reading
// count and sum; the mean below reads the weighted pair for those types
// only, and every other type keeps the reading-weighted mean, unchanged.
//
// One fold of one scan, no join. The weighted pair used to be a second
// grouping of the same rows, joined back to the per-source days before the
// pick; on a measurements table without planner statistics (a new install,
// an import under the autoanalyze threshold) that join ran as a nested
// loop that re-ran the weighting for every source day, and ten years of
// readings took seconds. `tests/integration/per-source-fold-parity.test.ts`
// holds the joined statement as the reference.
const mean = Prisma.raw(
foldMeanSql({
typeColumn: 'c."type"',
@@ -223,30 +228,18 @@ export async function readAllTimeExtremes(
Array<{ type: string; mean: number; min: number; max: number }>
>`
WITH live AS (${Prisma.join(rawArms, " UNION ALL ")}),
hourly_src AS (
SELECT * FROM live l
WHERE ${Prisma.raw(isHourlyMeanTypeSql('l."type"'))}
),
weighted AS (
SELECT
w."type",
w."source",
date_trunc('day', w."measured_at") AS day,
SUM(w."value" * w.day_weight)::double precision AS wsum,
SUM(w.day_weight)::double precision AS wdays
FROM ${dayWeightedRowsSql("hourly_src", null, { bySource: true })} w
GROUP BY w."type", w."source", 3
),
per_source AS (
SELECT
l."type",
l."source",
date_trunc('day', l."measured_at") AS day,
COUNT(*)::int AS cnt,
SUM(l."value")::double precision AS total,
MIN(l."value")::double precision AS min_value,
MAX(l."value")::double precision AS max_value
FROM live l
date_trunc('day', l."measured_at") AS day,
COUNT(*)::int AS cnt,
SUM(l."value")::double precision AS total,
MIN(l."value")::double precision AS min_value,
MAX(l."value")::double precision AS max_value,
SUM(l."value" * l.day_weight)::double precision AS wsum,
SUM(l.day_weight)::double precision AS wdays
FROM ${dayWeightedRowsSql("live", null, { bySource: true })} l
GROUP BY l."type", l."source", 3
)
SELECT
@@ -257,10 +250,8 @@ export async function readAllTimeExtremes(
FROM (
SELECT * FROM (
SELECT DISTINCT ON (p."type", p.day)
p."type", p.cnt, p.total, p.min_value, p.max_value, w.wsum, w.wdays
p."type", p.cnt, p.total, p.min_value, p.max_value, p.wsum, p.wdays
FROM per_source p
LEFT JOIN weighted w
ON w."type" = p."type" AND w."source" = p."source" AND w.day = p.day
ORDER BY p."type", p.day, (${rank("p")}), p."source"::text
) raw
${tierCanon}
src/lib/measurements/daily-series-read.ts
+34 -33
View File
@@ -217,13 +217,22 @@ export async function readLiveBuckets(opts: {
// the user's zone and truncate there.
//
// The canonical source is chosen per local DAY, whatever the grain: each
// source is folded per day in one hashed pass (a few rows per day, however
// dense the stream), each day keeps its ladder-canonical source, and only
// then are the days folded into the week or month. Picking the source per
// source is folded per hour and then per day in hashed passes (a few rows
// per day, however dense the stream), each day keeps its ladder-canonical
// source, and only then are the days folded into the week or month. Picking the source per
// week or month instead dropped every day the losing source carried, so a
// month of manual readings plus one synced reading reported one day.
// Folding first and picking second is the same result as collapsing the
// raw rows first; it just never sorts more than one row per source per day.
//
// One chain of folds over one scan, no join. The hour means used to be a
// second scan joined back to the per-source days before the pick; on a
// measurements table without planner statistics (a new install, an import
// under the autoanalyze threshold) that join ran as a nested loop that
// re-ran the hour fold for every source day, and a ten-year read of every
// type ran into the statement timeout.
// `tests/integration/per-source-fold-parity.test.ts` holds the joined
// statement as the reference.
const buckets = await prisma.$queryRaw<
Array<{
type: string;
@@ -234,13 +243,20 @@ export async function readLiveBuckets(opts: {
max_value: number | null;
}>
>`
WITH per_source AS (
-- Each source's readings fold per local hour, then the hours per local
-- day. A day of a type with an activity-dependent sampling rate is the
-- weighted mean of its local hours' means, not the mean of its readings:
-- a workout hour holds hundreds of times as many readings as a resting
-- one. NULL for every other type.
WITH per_hour AS (
SELECT
m."type",
m."source",
date_trunc('day', (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}) AS day,
date_trunc('hour', (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}) AS hr,
SUM(m."value")::double precision AS total,
COUNT(*)::int AS cnt,
COUNT(*) AS n,
AVG(m."value") AS hour_mean,
MIN(m."value") AS min_value,
MAX(m."value") AS max_value
FROM measurements m
@@ -249,40 +265,25 @@ export async function readLiveBuckets(opts: {
AND m."measured_at" <= ${to}
AND m."deleted_at" IS NULL
${typeFilter}
GROUP BY m."type", m."source", 3
GROUP BY m."type", m."source", 3, 4
),
-- A day of a type with an activity-dependent sampling rate is the weighted
-- mean of its local hours' means, not the mean of its readings: a workout
-- hour holds hundreds of times as many readings as a resting one.
-- Empty for every other type.
hour_mean AS (
SELECT h."type", h."source", h.day,
${dayOfHours}::double precision AS day_mean
FROM (
SELECT
m."type",
m."source",
date_trunc('day', (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}) AS day,
date_trunc('hour', (m."measured_at" AT TIME ZONE 'UTC') AT TIME ZONE ${timeZone}) AS hr,
AVG(m."value") AS hour_mean,
COUNT(*) AS n
FROM measurements m
WHERE m."user_id" = ${userId}
AND m."measured_at" >= ${from}
AND m."measured_at" <= ${to}
AND m."deleted_at" IS NULL
AND m."type" IN (${hourlyMeanList})
${typeFilter}
GROUP BY m."type", m."source", 3, 4
) h
per_source AS (
SELECT
h."type",
h."source",
h.day,
SUM(h.total)::double precision AS total,
SUM(h.n)::int AS cnt,
MIN(h.min_value) AS min_value,
MAX(h.max_value) AS max_value,
(CASE WHEN h."type" IN (${hourlyMeanList}) THEN ${dayOfHours} END)::double precision AS day_mean
FROM per_hour h
GROUP BY h."type", h."source", h.day
),
canon AS (
SELECT DISTINCT ON (p."type", p.day)
p."type", p.day, p.total, p.cnt, p.min_value, p.max_value, hm.day_mean
p."type", p.day, p.total, p.cnt, p.min_value, p.max_value, p.day_mean
FROM per_source p
LEFT JOIN hour_mean hm
ON hm."type" = p."type" AND hm."source" = p."source" AND hm.day = p.day
ORDER BY p."type", p.day, (${rankRaw}), p."source"::text
),
folded AS (