mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 13:22:14 +00:00
feat(rollups): MeasurementRollup foundation — populator + write hooks + warm-on-read
WIP toward v1.4.35. Lands the persistent rollup foundation from the
v1.4.34 cache-aggregation research §9. Reads still served by the
v1.4.34.1 live-SQL aggregator — the parity-safe read-swap
(count/min/max/mean + dailyByType from buckets, slope/r2/sd stay on
live aggregation) ships in a follow-up commit so v1.4.35 is the
single release that flips both foundation + read-swap.
Schema:
- new MeasurementRollup model with composite PK
(userId, type, granularity, bucketStart) and a descending
bucketStart index. RollupGranularity enum (DAY/WEEK/MONTH/YEAR).
- Prisma migration 0067_v1434_measurement_rollups is additive:
CREATE TYPE / CREATE TABLE IF NOT EXISTS / CREATE INDEX IF NOT
EXISTS / ADD CONSTRAINT FK. No ALTER on existing tables.
- db-compat schema-bootstrap path mirrored.
Populator (src/lib/measurements/rollups.ts):
- recomputeBucketsForMeasurement: sync DAY; WEEK/MONTH/YEAR via
pg-boss (singleton-keyed).
- recomputeUserRollups: bounded full-range, used by Apple Health
import + backfill.
- ensureUserRollupsFresh: cheap stale-detection + bounded DAY-90d
recompute. Error-swallowing.
- SQL via $queryRawUnsafe + strict enum whitelist for date_trunc
unit (Postgres 42803 forbids binding the unit literal).
Write hooks on every measurement-mutating endpoint; each wrapped
in try/catch so a populator hiccup never fails the write.
Apple Health import worker recomputes the full range once at
completion (per-row hooks would multiply cost).
Backfill (scripts/backfill-rollups.ts): idempotent, single-user
serial, single-digit minutes for a 100k-row account.
Tests: +14 unit, +3 integration. setup.ts.truncateAllTables
extends to the new table.
Read-swap follow-up:
- comprehensive-aggregator + analytics/route + summaries-slice
each call ensureUserRollupsFresh as a read side-effect (warms
the table). The response stays on the live-SQL aggregator so
byte-shape parity with v1.4.34.5 is preserved.
- Next commit lands the partial read-swap: count/min/max/mean
and dailyByType from buckets (math-identical compose), while
slope/r2/sd stay on indexed live aggregation. Together they
ship as v1.4.35.
Pre-shipping state — do NOT tag v1.4.35 from this commit alone.
This commit is contained in:
1 parent
2982ed1f0b
commit
9872081d0e
19 files changed
+1821
-4
No files matched your search
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* v1.5.0 — one-shot backfill for `measurement_rollups`.
|
||||
*
|
||||
* Walks every user and folds their measurement history into the
|
||||
* persistent rollup table across all four granularities (DAY, WEEK,
|
||||
* MONTH, YEAR). Bounded — one user at a time, no parallel users — so
|
||||
* the script never crowds the Prisma pool. Idempotent — re-running it
|
||||
* upserts the same rows.
|
||||
*
|
||||
* Usage
|
||||
* -----
|
||||
* pnpm tsx scripts/backfill-rollups.ts
|
||||
* pnpm tsx scripts/backfill-rollups.ts --user <userId>
|
||||
*
|
||||
* The `--user` flag targets a single user; without it the script
|
||||
* iterates every account in the database.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import {
|
||||
ALL_GRANULARITIES,
|
||||
recomputeUserRollups,
|
||||
} from "@/lib/measurements/rollups";
|
||||
|
||||
interface CliOptions {
|
||||
userId: string | null;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): CliOptions {
|
||||
const opts: CliOptions = { userId: null };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i];
|
||||
if (arg === "--user" && argv[i + 1]) {
|
||||
opts.userId = argv[i + 1];
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const opts = parseArgs(process.argv.slice(2));
|
||||
const where = opts.userId ? { id: opts.userId } : {};
|
||||
|
||||
const users = await prisma.user.findMany({
|
||||
where,
|
||||
select: { id: true, username: true },
|
||||
orderBy: { createdAt: "asc" },
|
||||
});
|
||||
|
||||
if (users.length === 0) {
|
||||
console.log(
|
||||
opts.userId
|
||||
? `No user matched id=${opts.userId}`
|
||||
: "No users in database — nothing to backfill",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[backfill-rollups] Folding ${users.length} user(s) across ` +
|
||||
`${ALL_GRANULARITIES.join(", ")} granularities`,
|
||||
);
|
||||
|
||||
let totalRowsUpserted = 0;
|
||||
const totalStarted = Date.now();
|
||||
for (const [index, user] of users.entries()) {
|
||||
const userStarted = Date.now();
|
||||
try {
|
||||
const { rowsUpserted, durationMs } = await recomputeUserRollups(
|
||||
user.id,
|
||||
{
|
||||
granularities: ALL_GRANULARITIES,
|
||||
},
|
||||
);
|
||||
totalRowsUpserted += rowsUpserted;
|
||||
console.log(
|
||||
`[${index + 1}/${users.length}] user=${user.username} ` +
|
||||
`(${user.id}) rows=${rowsUpserted} duration=${durationMs}ms`,
|
||||
);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(
|
||||
`[${index + 1}/${users.length}] user=${user.username} ` +
|
||||
`(${user.id}) FAILED after ${Date.now() - userStarted}ms: ${message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[backfill-rollups] DONE — ${totalRowsUpserted} rows upserted in ` +
|
||||
`${Date.now() - totalStarted}ms across ${users.length} user(s)`,
|
||||
);
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((err) => {
|
||||
console.error("[backfill-rollups] fatal error:", err);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(async () => {
|
||||
await prisma.$disconnect();
|
||||
});
|
||||
Reference in new issue
Block a user