mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-09 04:50:59 +00:00
perf(bench): add a large-account seed and a request benchmark
scripts/bench/seed-large-account.ts (moved from scripts/, extended) writes one synthetic account of ten years: 1,265,561 measurement rows across the main types and five sources (Apple Health dense spot rows, Withings blood pressure and scale, a weekly manual weight, three years of Oura nightly values, 90 days of Nightscout CGM), `stats:` and per-sample cumulative rows, nightly sleep stages, 3,020 workouts, 1,095 mood entries and three medications with 3,285 intake events. Everything goes in through set-based INSERT … SELECT generate_series; values and timestamps are functions of the row index and `--anchor`, so a run is reproducible. The rollup tier is folded with recomputeUserRollups, as the boot backfill does, and the tables are analysed. About 80 s on the Mac mini. scripts/bench/run.ts boots the production build once with every role (so the queues exist), then measures against a web-only process under the production heap ceiling: p50, p95 and max latency and SQL statements per request (pg_stat_statements, read once nothing is in flight and the count holds still) for /api/auth/me, the dashboard summary and snapshot, the pulse series over 30, 365 and 3650 days, the day view, insights health status and correlations, each cold (caches evicted by a one-row manual-sync ingest before every request) and warm, plus a 5,000-row batch ingest as ten requests of 500. It writes JSON and a markdown table. docs/contributing/benchmarks.md describes the setup against a throwaway Postgres in Docker with the production pg_stat_statements settings.
This commit is contained in:
Marc-André Bombeck committed
1 parent
45ea61261e
commit
375ca12b05
7 files changed
+1225
-297
No files matched your search
@@ -28,6 +28,7 @@ typecheck.
|
||||
- **`repair-intake-anomalies.ts`** — repair historic medication-ledger defects (duplicate dose-slot rows, implausible `taken_at`); dry-run by default, see `docs/ops/intake-repair.md`.
|
||||
- **`backfill-rollups.ts`**, **`drain-per-sample-cumulative.ts`**, **`backfill-mood-note-column.ts`** — historical data backfills (note: the boot-time `rollup-full-backfill` queue handles new accounts automatically; the CLI is the manual fallback).
|
||||
- **`seed-demo.ts`** + **`seed-demo-v15.sql`** — populate a demo tenant.
|
||||
- **`bench/`** — `seed-large-account.ts` (a synthetic ten-year account of about 1.26 million measurement rows) and `run.ts` (latency and SQL statements per request against it); see `docs/contributing/benchmarks.md`. Results land in `bench/results/`.
|
||||
- **`test-notifications.ts`** — exercise the notification dispatcher + reminder check.
|
||||
- **`fetch-geolite2.sh`** — download the MaxMind GeoLite2 MMDBs (needs a licence key) before `docker build`.
|
||||
- **`install-hooks.sh`** — install the local git hooks.
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
/**
|
||||
* Request benchmark against a seeded account: latency (p50, p95, max) and
|
||||
* SQL statements per request for the hot read and write paths.
|
||||
*
|
||||
* DATABASE_URL=postgresql://… API_TOKEN_HMAC_KEY=… ENCRYPTION_KEYS=… \
|
||||
* ENCRYPTION_ACTIVE_KEY_ID=… pnpm dlx tsx scripts/bench/run.ts \
|
||||
* [--user bench-large] [--iterations 20] [--warmup 3] \
|
||||
* [--base-url http://127.0.0.1:3100] [--out scripts/bench/results/<label>]
|
||||
*
|
||||
* See docs/contributing/benchmarks.md for the whole setup (a throwaway
|
||||
* Postgres with pg_stat_statements, migrations, the seed, a production
|
||||
* build). Never point it at a database that holds real data: it mints a
|
||||
* token for the bench user and writes measurements under it.
|
||||
*
|
||||
* Without `--base-url` the script starts the production build itself
|
||||
* (`next start` on `.next`) as a web-only process under the production heap
|
||||
* ceiling, and stops it at the end.
|
||||
*
|
||||
* Statements are counted with pg_stat_statements: the sum of `calls` over
|
||||
* the database, read before and after each request, once no statement is
|
||||
* in flight and the count has stopped moving (so fire-and-forget writes a
|
||||
* request starts are counted with it). The bench's own reads of the two
|
||||
* views are excluded. Requests run one at a time, so nothing else is in the
|
||||
* delta except the server's idle background, which is measured first and
|
||||
* reported. Before the measurement the build boots once with every role, so
|
||||
* the worker creates its queues as it does in production; whatever that
|
||||
* boot scheduled is dropped.
|
||||
*
|
||||
* Each read runs twice: cold, with the user's caches evicted before every
|
||||
* request by a one-row manual-sync ingest (excluded from the numbers), so
|
||||
* the request rebuilds what it serves; and warm, back to back, which is
|
||||
* what a cache hit costs. The batch ingest posts 5,000 fresh rows as ten
|
||||
* requests of 500, the route's per-request limit, and removes them again.
|
||||
*
|
||||
* Output: `<out>.json` and `<out>.md`. Exit 2 on any non-2xx response.
|
||||
*/
|
||||
import { execSync, spawn, type ChildProcess } from "node:child_process";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { cpus, loadavg } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
|
||||
import { Pool } from "pg";
|
||||
|
||||
import { issueApiToken } from "@/lib/auth/issue-token";
|
||||
import { prisma } from "@/lib/db";
|
||||
|
||||
function arg(name: string, fallback: string): string {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
return i >= 0 && process.argv[i + 1] ? process.argv[i + 1]! : fallback;
|
||||
}
|
||||
|
||||
const USER_ID = arg("user", "bench-large");
|
||||
const ITERATIONS = Number(arg("iterations", "20"));
|
||||
const WARMUP = Number(arg("warmup", "3"));
|
||||
const PORT = Number(arg("port", "3100"));
|
||||
const BASE_URL = arg("base-url", "");
|
||||
const OUT = arg("out", join("scripts", "bench", "results", "latest"));
|
||||
const HEAP_MB = Number(arg("heap-mb", "792"));
|
||||
|
||||
const pool = new Pool({ connectionString: process.env.DATABASE_URL, max: 1 });
|
||||
|
||||
async function statementCount(): Promise<number> {
|
||||
const { rows } = await pool.query<{ n: string | null }>(
|
||||
`SELECT sum(calls)::bigint AS n FROM pg_stat_statements
|
||||
WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database())
|
||||
AND query NOT ILIKE '%pg_stat_statements%'
|
||||
AND query NOT ILIKE '%pg_stat_activity%'`,
|
||||
);
|
||||
return Number(rows[0]?.n ?? 0);
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
/**
|
||||
* Wait until the server has no statement in flight and the counter holds
|
||||
* still for 150 ms (or 10 s pass), then read it. pg_stat_statements counts a
|
||||
* statement when it ends, so a long fold in a background refresh would look
|
||||
* settled by the counter alone; pg_stat_activity says it is still running.
|
||||
*/
|
||||
async function settledCount(): Promise<number> {
|
||||
let last = -1;
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (Date.now() < deadline) {
|
||||
const { rows } = await pool.query<{ active: string }>(
|
||||
`SELECT count(*)::int AS active FROM pg_stat_activity
|
||||
WHERE datname = current_database() AND pid <> pg_backend_pid()
|
||||
AND state NOT IN ('idle')`,
|
||||
);
|
||||
const now = await statementCount();
|
||||
if (Number(rows[0]?.active ?? 0) === 0 && now === last) return now;
|
||||
last = now;
|
||||
await sleep(150);
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
function percentile(sorted: number[], p: number): number {
|
||||
if (sorted.length === 0) return 0;
|
||||
const i = Math.ceil((p / 100) * sorted.length) - 1;
|
||||
return sorted[Math.min(sorted.length - 1, Math.max(0, i))]!;
|
||||
}
|
||||
|
||||
interface Sample {
|
||||
ms: number;
|
||||
statements: number;
|
||||
status: number;
|
||||
}
|
||||
|
||||
interface ScenarioResult {
|
||||
name: string;
|
||||
mode: "cold" | "warm" | "write";
|
||||
request: string;
|
||||
n: number;
|
||||
p50Ms: number;
|
||||
p95Ms: number;
|
||||
maxMs: number;
|
||||
statementsMedian: number;
|
||||
statementsMax: number;
|
||||
failures: number;
|
||||
}
|
||||
|
||||
let token = "";
|
||||
const failures: string[] = [];
|
||||
|
||||
async function call(
|
||||
method: string,
|
||||
path: string,
|
||||
body?: unknown,
|
||||
): Promise<Sample> {
|
||||
const before = await settledCount();
|
||||
const started = performance.now();
|
||||
const res = await fetch(`${base()}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
...(body ? { "content-type": "application/json" } : {}),
|
||||
},
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
await res.arrayBuffer();
|
||||
const ms = performance.now() - started;
|
||||
const after = await settledCount();
|
||||
if (!res.ok) failures.push(`${method} ${path}: ${res.status}`);
|
||||
return { ms, statements: after - before, status: res.status };
|
||||
}
|
||||
|
||||
function summarise(
|
||||
name: string,
|
||||
mode: ScenarioResult["mode"],
|
||||
request: string,
|
||||
samples: Sample[],
|
||||
): ScenarioResult {
|
||||
const ms = samples.map((s) => s.ms).sort((a, b) => a - b);
|
||||
const st = samples.map((s) => s.statements).sort((a, b) => a - b);
|
||||
const r = (n: number) => Math.round(n * 10) / 10;
|
||||
return {
|
||||
name,
|
||||
mode,
|
||||
request,
|
||||
n: samples.length,
|
||||
p50Ms: r(percentile(ms, 50)),
|
||||
p95Ms: r(percentile(ms, 95)),
|
||||
maxMs: r(ms[ms.length - 1] ?? 0),
|
||||
statementsMedian: percentile(st, 50),
|
||||
statementsMax: st[st.length - 1] ?? 0,
|
||||
failures: samples.filter((s) => s.status >= 300).length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The per-user rate limits would refuse a benchmark long before a person
|
||||
* meets them (one run sends several hundred requests in minutes), so the
|
||||
* bench user's buckets are cleared before each scenario. The limiter's own
|
||||
* cost stays in every count.
|
||||
*/
|
||||
async function resetRateLimits(): Promise<void> {
|
||||
await pool.query(`DELETE FROM rate_limits WHERE key LIKE '%:' || $1`, [
|
||||
USER_ID,
|
||||
]);
|
||||
}
|
||||
|
||||
let bustCounter = 0;
|
||||
/**
|
||||
* Evict the user's caches with a real change, outside the numbers: a one-row
|
||||
* ingest marked as a manual sync, which hard-evicts instead of marking
|
||||
* stale, so the next read is a miss and rebuilds in the request.
|
||||
*/
|
||||
async function bust(anchorDay: string): Promise<void> {
|
||||
bustCounter += 1;
|
||||
const res = await fetch(`${base()}/api/measurements/batch`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
syncTrigger: "manual",
|
||||
entries: [
|
||||
{
|
||||
hkIdentifier: "HKQuantityTypeIdentifierFlightsClimbed",
|
||||
value: 10 + (bustCounter % 2),
|
||||
unit: "count",
|
||||
startDate: `${anchorDay}T08:00:00.000Z`,
|
||||
endDate: `${anchorDay}T18:00:00.000Z`,
|
||||
externalId: `stats:HKQuantityTypeIdentifierFlightsClimbed:${anchorDay}`,
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
const body = (await res.json()) as {
|
||||
data?: { inserted?: number; updated?: number };
|
||||
};
|
||||
// A bust that wrote nothing evicted nothing, and every "cold" number
|
||||
// after it would be a cache hit.
|
||||
if ((body.data?.inserted ?? 0) + (body.data?.updated ?? 0) !== 1) {
|
||||
throw new Error(`cache bust wrote no row: ${JSON.stringify(body)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function base(): string {
|
||||
return BASE_URL || `http://127.0.0.1:${PORT}`;
|
||||
}
|
||||
|
||||
async function waitForHealth(
|
||||
timeoutMs: number,
|
||||
server: ChildProcess | null,
|
||||
): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (server && server.exitCode !== null) {
|
||||
throw new Error(`the server exited with code ${server.exitCode}`);
|
||||
}
|
||||
try {
|
||||
const res = await fetch(`${base()}/api/health`);
|
||||
if (res.ok) return;
|
||||
} catch {
|
||||
// not listening yet
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
throw new Error(`server at ${base()} did not answer /api/health`);
|
||||
}
|
||||
|
||||
function startServer(role: "web" | "all"): ChildProcess {
|
||||
// `next start` rather than the standalone server: the standalone tree
|
||||
// lacks the externalised runtime packages the image installs separately
|
||||
// (see the Dockerfile), while the checkout has them. It serves the same
|
||||
// build output.
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"node_modules/next/dist/bin/next",
|
||||
"start",
|
||||
"-p",
|
||||
String(PORT),
|
||||
"-H",
|
||||
"127.0.0.1",
|
||||
],
|
||||
{
|
||||
env: {
|
||||
...process.env,
|
||||
NODE_ENV: "production",
|
||||
PORT: String(PORT),
|
||||
HOSTNAME: "127.0.0.1",
|
||||
HEALTHLOG_PROCESS_TYPE: role,
|
||||
SESSION_COOKIE_SECURE: "false",
|
||||
NODE_OPTIONS: `--max-old-space-size=${HEAP_MB}`,
|
||||
},
|
||||
stdio: ["ignore", "ignore", "inherit"],
|
||||
},
|
||||
);
|
||||
return child;
|
||||
}
|
||||
|
||||
async function stopServer(child: ChildProcess): Promise<void> {
|
||||
if (child.exitCode !== null) return;
|
||||
const exited = new Promise((resolve) => child.once("exit", resolve));
|
||||
child.kill("SIGTERM");
|
||||
await Promise.race([exited, sleep(20_000)]);
|
||||
if (child.exitCode === null) child.kill("SIGKILL");
|
||||
}
|
||||
|
||||
function freshBatch(run: string, chunk: number): unknown[] {
|
||||
// 500 heart-rate readings a minute apart, on a day inside the seeded
|
||||
// window, with ids no earlier run used.
|
||||
const day = Date.UTC(2026, 0, 1) + chunk * 86_400_000;
|
||||
return Array.from({ length: 500 }, (_, i) => {
|
||||
const at = new Date(day + i * 60_000).toISOString();
|
||||
return {
|
||||
hkIdentifier: "HKQuantityTypeIdentifierHeartRate",
|
||||
value: 55 + (i % 40),
|
||||
unit: "count/min",
|
||||
startDate: at,
|
||||
endDate: at,
|
||||
externalId: `bench-${run}-${chunk}-${i}`,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function markdown(meta: Record<string, unknown>, rows: ScenarioResult[]) {
|
||||
const lines = [
|
||||
`# Benchmark ${String(meta.label)}`,
|
||||
"",
|
||||
`Commit ${String(meta.commit)}, ${String(meta.date)}. ${String(meta.measurements)} measurement rows on the bench account, ${ITERATIONS} measured requests per row after ${WARMUP} warm-up requests, one at a time. Server: production build, web process, heap ceiling ${HEAP_MB} MB. Postgres ${String(meta.postgres)}, ${String(meta.cpus)} cores, load average ${String(meta.loadAverage)} at the end. Idle background: ${String(meta.idleStatementsPerSecond)} statements/s.`,
|
||||
"",
|
||||
"| Request | Mode | n | p50 ms | p95 ms | max ms | statements (median) | statements (max) |",
|
||||
"| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
...rows.map(
|
||||
(r) =>
|
||||
`| ${r.name} | ${r.mode} | ${r.n} | ${r.p50Ms} | ${r.p95Ms} | ${r.maxMs} | ${r.statementsMedian} | ${r.statementsMax} |`,
|
||||
),
|
||||
"",
|
||||
];
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
if (!process.env.DATABASE_URL) throw new Error("DATABASE_URL is required");
|
||||
await pool.query("CREATE EXTENSION IF NOT EXISTS pg_stat_statements");
|
||||
await statementCount(); // fails here when the library is not preloaded
|
||||
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: USER_ID },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!user) throw new Error(`no user ${USER_ID}; run the seed first`);
|
||||
const measurements = await prisma.measurement.count({
|
||||
where: { userId: USER_ID },
|
||||
});
|
||||
token = (
|
||||
await issueApiToken({
|
||||
userId: USER_ID,
|
||||
name: "benchmark",
|
||||
permissions: ["*"],
|
||||
expiresInDays: 1,
|
||||
})
|
||||
).token;
|
||||
|
||||
// A web-only process only sends jobs; the queues are created by the
|
||||
// worker role. Boot the full process once so every queue exists (as it
|
||||
// does in production), then run the measurement against a web-only
|
||||
// process, so the worker's polling is not in every delta.
|
||||
if (!BASE_URL) {
|
||||
const full = startServer("all");
|
||||
await waitForHealth(180_000, full);
|
||||
await stopServer(full);
|
||||
// Drop whatever that boot scheduled: nothing in the web-only process
|
||||
// runs jobs, and a fold left queued would change the fixture on the
|
||||
// next full boot.
|
||||
await pool.query("TRUNCATE pgboss.job");
|
||||
}
|
||||
const server = BASE_URL ? null : startServer("web");
|
||||
try {
|
||||
await waitForHealth(180_000, server);
|
||||
|
||||
const idleBefore = await settledCount();
|
||||
await sleep(5_000);
|
||||
const idleStatementsPerSecond =
|
||||
Math.round(((await statementCount()) - idleBefore) / 0.5) / 10;
|
||||
|
||||
const yesterday = new Date(Date.now() - 86_400_000)
|
||||
.toISOString()
|
||||
.slice(0, 10);
|
||||
const reads: Array<[string, string]> = [
|
||||
["/api/auth/me", "/api/auth/me"],
|
||||
["dashboard summary", "/api/dashboard/summary"],
|
||||
["dashboard snapshot", "/api/dashboard/snapshot"],
|
||||
["series pulse 30 d", "/api/measurements/series?kind=pulse&days=30"],
|
||||
["series pulse 365 d", "/api/measurements/series?kind=pulse&days=365"],
|
||||
["series pulse 3650 d", "/api/measurements/series?kind=pulse&days=3650"],
|
||||
["day view", `/api/day/${yesterday}`],
|
||||
["insights health status", "/api/insights/health-status"],
|
||||
["insights correlations", "/api/insights/correlations"],
|
||||
];
|
||||
|
||||
const results: ScenarioResult[] = [];
|
||||
for (const [name, path] of reads) {
|
||||
await resetRateLimits();
|
||||
for (let i = 0; i < WARMUP; i++) await call("GET", path);
|
||||
const cold: Sample[] = [];
|
||||
for (let i = 0; i < ITERATIONS; i++) {
|
||||
await bust(yesterday);
|
||||
cold.push(await call("GET", path));
|
||||
}
|
||||
results.push(summarise(name, "cold", `GET ${path}`, cold));
|
||||
await resetRateLimits();
|
||||
const warm: Sample[] = [];
|
||||
for (let i = 0; i < ITERATIONS; i++) warm.push(await call("GET", path));
|
||||
results.push(summarise(name, "warm", `GET ${path}`, warm));
|
||||
console.log(
|
||||
`${name.padEnd(26)} cold p95 ${results.at(-2)!.p95Ms} ms / ${results.at(-2)!.statementsMedian} st, warm p95 ${results.at(-1)!.p95Ms} ms / ${results.at(-1)!.statementsMedian} st`,
|
||||
);
|
||||
}
|
||||
|
||||
// Rows an earlier run wrote would turn this run's inserts into
|
||||
// reconciles of the same natural key, so they go first, and again at
|
||||
// the end, leaving the seeded fixture as it was.
|
||||
const clean = () =>
|
||||
pool.query(
|
||||
`DELETE FROM measurements WHERE user_id = $1 AND external_id LIKE 'bench-%'`,
|
||||
[USER_ID],
|
||||
);
|
||||
await clean();
|
||||
await resetRateLimits();
|
||||
const run = Date.now().toString(36);
|
||||
const writes: Sample[] = [];
|
||||
for (let chunk = 0; chunk < 10; chunk++) {
|
||||
writes.push(
|
||||
await call("POST", "/api/measurements/batch", {
|
||||
entries: freshBatch(run, chunk),
|
||||
}),
|
||||
);
|
||||
}
|
||||
const batch = summarise(
|
||||
"batch ingest 5k (per 500)",
|
||||
"write",
|
||||
"POST /api/measurements/batch ×10",
|
||||
writes,
|
||||
);
|
||||
results.push(batch);
|
||||
await clean();
|
||||
const totalMs = writes.reduce((a, s) => a + s.ms, 0);
|
||||
const totalStatements = writes.reduce((a, s) => a + s.statements, 0);
|
||||
console.log(
|
||||
`batch ingest 5k total ${Math.round(totalMs)} ms / ${totalStatements} st`,
|
||||
);
|
||||
|
||||
let commit = "unknown";
|
||||
try {
|
||||
commit = execSync("git rev-parse --short HEAD").toString().trim();
|
||||
} catch {
|
||||
// not a checkout
|
||||
}
|
||||
const meta = {
|
||||
label: OUT.split("/").pop(),
|
||||
commit,
|
||||
date: new Date().toISOString(),
|
||||
node: process.version,
|
||||
postgres: (
|
||||
await pool.query<{ v: string }>(
|
||||
"SELECT current_setting('server_version') AS v",
|
||||
)
|
||||
).rows[0]?.v,
|
||||
cpus: cpus().length,
|
||||
loadAverage: loadavg().map((n) => Math.round(n * 10) / 10),
|
||||
measurements,
|
||||
iterations: ITERATIONS,
|
||||
warmup: WARMUP,
|
||||
heapMb: HEAP_MB,
|
||||
idleStatementsPerSecond,
|
||||
batch5k: { totalMs: Math.round(totalMs), totalStatements },
|
||||
};
|
||||
mkdirSync(dirname(OUT), { recursive: true });
|
||||
writeFileSync(
|
||||
`${OUT}.json`,
|
||||
`${JSON.stringify({ meta, results }, null, 2)}\n`,
|
||||
);
|
||||
writeFileSync(`${OUT}.md`, markdown(meta, results));
|
||||
console.log(`wrote ${OUT}.json and ${OUT}.md`);
|
||||
} finally {
|
||||
if (server) await stopServer(server);
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(`non-2xx responses:\n ${failures.join("\n ")}`);
|
||||
process.exitCode = 2;
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((err) => {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
})
|
||||
.finally(async () => {
|
||||
await prisma.$disconnect();
|
||||
await pool.end();
|
||||
});
|
||||
@@ -0,0 +1,659 @@
|
||||
/**
|
||||
* Seed one synthetic account the size of a heavy real one, for the benchmark
|
||||
* harness (`scripts/bench/run.ts`) and for reproducing the maintenance jobs:
|
||||
* about 1.26 million measurement rows over ten years, plus workouts, mood
|
||||
* entries and medications with their intake history.
|
||||
*
|
||||
* DATABASE_URL=postgresql://… pnpm dlx tsx scripts/bench/seed-large-account.ts \
|
||||
* [--user bench-large] [--years 10] [--anchor 2026-10-22] [--reset]
|
||||
* [--no-rollups]
|
||||
*
|
||||
* Never point this at a database that holds real data: it inserts rows under
|
||||
* the given user id and, with `--reset`, deletes that user (and, through the
|
||||
* cascades, everything it owns) first.
|
||||
*
|
||||
* Deterministic: every value and timestamp is a function of the row index and
|
||||
* the anchor day (`--anchor`, UTC midnight; default today), and ids are
|
||||
* derived from the user id, the series and the index. Two runs with the same
|
||||
* arguments write the same rows.
|
||||
*
|
||||
* The shape follows what the integrations write, with overlapping sources so
|
||||
* the canonical-source pick has work to do:
|
||||
*
|
||||
* - Apple Health dense spot rows (heart rate about every ten minutes, HRV,
|
||||
* SpO2, respiratory rate, gait metrics, audio exposure) keyed by a hashed
|
||||
* sample id;
|
||||
* - Withings morning and evening blood pressure with its pulse, and daily
|
||||
* weight, body fat and muscle mass; a weekly manual weight beside it;
|
||||
* - Oura nightly resting heart rate and RMSSD for the last three years,
|
||||
* beside Apple Health's resting heart rate;
|
||||
* - Nightscout CGM glucose every five minutes for the last 90 days;
|
||||
* - one `stats:` daily row per cumulative type and day (the export import)
|
||||
* and per-sample cumulative rows with opaque ids for the last 90 days
|
||||
* (the iOS sync);
|
||||
* - sleep-stage segments every night;
|
||||
* - about 3,000 workouts, a mood entry a day for three years, and three
|
||||
* daily medications with their intake events for three years.
|
||||
*
|
||||
* Everything goes in through set-based `INSERT … SELECT … generate_series`,
|
||||
* so a million rows take seconds rather than a million round trips. The
|
||||
* rollup tier is then folded with `recomputeUserRollups`, the same path the
|
||||
* boot backfill takes for an imported account, so reads see the state a real
|
||||
* account is in. `ANALYZE` runs at the end: a planner reading pre-seed
|
||||
* statistics picks plans no production table of this size would get.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
import {
|
||||
ALL_GRANULARITIES,
|
||||
recomputeUserRollups,
|
||||
} from "@/lib/rollups/measurement-rollups";
|
||||
|
||||
function arg(name: string, fallback: string): string {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
return i >= 0 && process.argv[i + 1] ? process.argv[i + 1]! : fallback;
|
||||
}
|
||||
|
||||
const USER_ID = arg("user", "bench-large");
|
||||
const YEARS = Number(arg("years", "10"));
|
||||
const ANCHOR = arg("anchor", new Date().toISOString().slice(0, 10));
|
||||
const RESET = process.argv.includes("--reset");
|
||||
const ROLLUPS = !process.argv.includes("--no-rollups");
|
||||
const ZONE = "Europe/Berlin";
|
||||
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(ANCHOR)) {
|
||||
throw new Error(`--anchor must be YYYY-MM-DD, got ${ANCHOR}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* A spot series: rows evenly spaced over the last `days` days before the
|
||||
* anchor. `perDay` above one spaces rows through the day; `everyDays` above
|
||||
* one writes one row every that many days at `offsetSec` past midnight UTC.
|
||||
*/
|
||||
interface SpotSeries {
|
||||
type: string;
|
||||
unit: string;
|
||||
source: string;
|
||||
base: number;
|
||||
spread: number;
|
||||
days: number;
|
||||
perDay?: number;
|
||||
everyDays?: number;
|
||||
offsetSec?: number;
|
||||
/** externalId prefix; `stats:` rows are generated separately. */
|
||||
idPrefix: string;
|
||||
}
|
||||
|
||||
async function insertSpot(s: SpotSeries): Promise<number> {
|
||||
const step = s.everyDays
|
||||
? s.everyDays * 86_400
|
||||
: Math.floor(86_400 / (s.perDay ?? 1));
|
||||
const count = Math.floor((s.days * 86_400) / step);
|
||||
// Jitter stays below the step so every measured_at is distinct per type
|
||||
// and source (the (user, type, measured_at, source, sleep_stage) unique
|
||||
// index). Series that share a step and offset (BP systolic, diastolic and
|
||||
// the Withings pulse) share their timestamps, as a cuff reading does.
|
||||
const jitter = Math.max(1, Math.min(59, step - 1));
|
||||
return prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO measurements
|
||||
(id, user_id, type, value, unit, source, measured_at, external_id, created_at, updated_at)
|
||||
SELECT
|
||||
'seed-' || $1 || '-' || $2 || '-' || $10 || '-' || g,
|
||||
$1,
|
||||
$2::measurement_type,
|
||||
round(($3 + $4 * sin(g / 37.0) + ($4 / 2) * ((g * 7919) % 100) / 100.0)::numeric, 2),
|
||||
$5,
|
||||
$10::measurement_source,
|
||||
$11::date::timestamp - make_interval(secs => $6 * $7::bigint)
|
||||
+ make_interval(secs => g * $6 + $12 + (g * 31) % $8),
|
||||
$9 || md5($2 || $10 || g::text),
|
||||
now(), now()
|
||||
FROM generate_series(0, $7::bigint - 1) AS g
|
||||
`,
|
||||
USER_ID,
|
||||
s.type,
|
||||
s.base,
|
||||
s.spread,
|
||||
s.unit,
|
||||
step,
|
||||
count,
|
||||
jitter,
|
||||
s.idPrefix,
|
||||
s.source,
|
||||
ANCHOR,
|
||||
s.offsetSec ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
function log(label: string, n: number): void {
|
||||
console.log(`${label.padEnd(40)} ${String(n).padStart(9)}`);
|
||||
}
|
||||
|
||||
async function seedMeasurements(days: number): Promise<number> {
|
||||
const recent = Math.min(days, 90);
|
||||
const threeYears = Math.min(days, 3 * 365);
|
||||
const spot: SpotSeries[] = [
|
||||
// Apple Health, the dense tier.
|
||||
{
|
||||
type: "PULSE",
|
||||
unit: "bpm",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 147,
|
||||
base: 62,
|
||||
spread: 25,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "HEART_RATE_VARIABILITY",
|
||||
unit: "ms",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 12,
|
||||
base: 45,
|
||||
spread: 20,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "OXYGEN_SATURATION",
|
||||
unit: "%",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 24,
|
||||
base: 96,
|
||||
spread: 2,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "RESPIRATORY_RATE",
|
||||
unit: "breaths/min",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 24,
|
||||
base: 14,
|
||||
spread: 3,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "WALKING_SPEED",
|
||||
unit: "m/s",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 12,
|
||||
base: 1.3,
|
||||
spread: 0.2,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "WALKING_STEP_LENGTH",
|
||||
unit: "m",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 12,
|
||||
base: 0.72,
|
||||
spread: 0.06,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "WALKING_ASYMMETRY",
|
||||
unit: "%",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 12,
|
||||
base: 3,
|
||||
spread: 2,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "WALKING_DOUBLE_SUPPORT",
|
||||
unit: "%",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 12,
|
||||
base: 28,
|
||||
spread: 3,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "AUDIO_EXPOSURE_ENV",
|
||||
unit: "dBA",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 24,
|
||||
base: 62,
|
||||
spread: 12,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "RESTING_HEART_RATE",
|
||||
unit: "bpm",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 1,
|
||||
offsetSec: 6 * 3600,
|
||||
base: 56,
|
||||
spread: 4,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "TIME_IN_DAYLIGHT",
|
||||
unit: "minutes",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 1,
|
||||
offsetSec: 18 * 3600,
|
||||
base: 60,
|
||||
spread: 40,
|
||||
days,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
// The iOS sync's per-sample cumulative rows: dense, recent, opaque ids.
|
||||
{
|
||||
type: "ACTIVITY_STEPS",
|
||||
unit: "steps",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 100,
|
||||
base: 90,
|
||||
spread: 60,
|
||||
days: recent,
|
||||
idPrefix: "uuid-",
|
||||
},
|
||||
{
|
||||
type: "ACTIVE_ENERGY_BURNED",
|
||||
unit: "kcal",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 400,
|
||||
base: 1.2,
|
||||
spread: 0.9,
|
||||
days: recent,
|
||||
idPrefix: "uuid-",
|
||||
},
|
||||
{
|
||||
type: "WALKING_RUNNING_DISTANCE",
|
||||
unit: "m",
|
||||
source: "APPLE_HEALTH",
|
||||
perDay: 100,
|
||||
base: 70,
|
||||
spread: 50,
|
||||
days: recent,
|
||||
idPrefix: "uuid-",
|
||||
},
|
||||
// Withings: a cuff reading morning and evening, and the scale.
|
||||
{
|
||||
type: "BLOOD_PRESSURE_SYS",
|
||||
unit: "mmHg",
|
||||
source: "WITHINGS",
|
||||
perDay: 2,
|
||||
offsetSec: 6 * 3600,
|
||||
base: 126,
|
||||
spread: 10,
|
||||
days,
|
||||
idPrefix: "wi-",
|
||||
},
|
||||
{
|
||||
type: "BLOOD_PRESSURE_DIA",
|
||||
unit: "mmHg",
|
||||
source: "WITHINGS",
|
||||
perDay: 2,
|
||||
offsetSec: 6 * 3600,
|
||||
base: 81,
|
||||
spread: 7,
|
||||
days,
|
||||
idPrefix: "wi-",
|
||||
},
|
||||
{
|
||||
type: "PULSE",
|
||||
unit: "bpm",
|
||||
source: "WITHINGS",
|
||||
perDay: 2,
|
||||
offsetSec: 6 * 3600,
|
||||
base: 66,
|
||||
spread: 8,
|
||||
days,
|
||||
idPrefix: "wi-",
|
||||
},
|
||||
{
|
||||
type: "WEIGHT",
|
||||
unit: "kg",
|
||||
source: "WITHINGS",
|
||||
perDay: 1,
|
||||
offsetSec: 5 * 3600,
|
||||
base: 82,
|
||||
spread: 1.5,
|
||||
days,
|
||||
idPrefix: "wi-",
|
||||
},
|
||||
{
|
||||
type: "BODY_FAT",
|
||||
unit: "%",
|
||||
source: "WITHINGS",
|
||||
perDay: 1,
|
||||
offsetSec: 5 * 3600,
|
||||
base: 22,
|
||||
spread: 2,
|
||||
days,
|
||||
idPrefix: "wi-",
|
||||
},
|
||||
{
|
||||
type: "MUSCLE_MASS",
|
||||
unit: "kg",
|
||||
source: "WITHINGS",
|
||||
perDay: 1,
|
||||
offsetSec: 5 * 3600,
|
||||
base: 61,
|
||||
spread: 1,
|
||||
days,
|
||||
idPrefix: "wi-",
|
||||
},
|
||||
// A weekly manual weight beside the scale.
|
||||
{
|
||||
type: "WEIGHT",
|
||||
unit: "kg",
|
||||
source: "MANUAL",
|
||||
everyDays: 7,
|
||||
offsetSec: 19 * 3600,
|
||||
base: 82.4,
|
||||
spread: 1.5,
|
||||
days,
|
||||
idPrefix: "man-",
|
||||
},
|
||||
// Oura, nightly, the last three years.
|
||||
{
|
||||
type: "RESTING_HEART_RATE",
|
||||
unit: "bpm",
|
||||
source: "OURA",
|
||||
perDay: 1,
|
||||
offsetSec: 4 * 3600,
|
||||
base: 54,
|
||||
spread: 4,
|
||||
days: threeYears,
|
||||
idPrefix: "oura-",
|
||||
},
|
||||
{
|
||||
type: "HRV_RMSSD",
|
||||
unit: "ms",
|
||||
source: "OURA",
|
||||
perDay: 1,
|
||||
offsetSec: 4 * 3600,
|
||||
base: 48,
|
||||
spread: 15,
|
||||
days: threeYears,
|
||||
idPrefix: "oura-",
|
||||
},
|
||||
// Nightscout CGM, every five minutes, the last 90 days.
|
||||
{
|
||||
type: "BLOOD_GLUCOSE",
|
||||
unit: "mg/dL",
|
||||
source: "NIGHTSCOUT",
|
||||
perDay: 288,
|
||||
base: 105,
|
||||
spread: 30,
|
||||
days: recent,
|
||||
idPrefix: "ns-",
|
||||
},
|
||||
];
|
||||
|
||||
let total = 0;
|
||||
for (const s of spot) {
|
||||
const n = await insertSpot(s);
|
||||
total += n;
|
||||
log(`${s.type} (${s.source})`, n);
|
||||
}
|
||||
|
||||
// One export-estimate `stats:` row per cumulative type and local day, at
|
||||
// local noon — what `flushCumulativeBuckets` in the export import writes —
|
||||
// for every day older than the per-sample window.
|
||||
for (const [type, hk, unit, base] of [
|
||||
["ACTIVITY_STEPS", "HKQuantityTypeIdentifierStepCount", "steps", 8000],
|
||||
[
|
||||
"ACTIVE_ENERGY_BURNED",
|
||||
"HKQuantityTypeIdentifierActiveEnergyBurned",
|
||||
"kcal",
|
||||
550,
|
||||
],
|
||||
[
|
||||
"WALKING_RUNNING_DISTANCE",
|
||||
"HKQuantityTypeIdentifierDistanceWalkingRunning",
|
||||
"m",
|
||||
6000,
|
||||
],
|
||||
["FLIGHTS_CLIMBED", "HKQuantityTypeIdentifierFlightsClimbed", "flights", 9],
|
||||
] as const) {
|
||||
const n = await prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO measurements
|
||||
(id, user_id, type, value, unit, source, measured_at, external_id,
|
||||
aggregation_provenance, created_at, updated_at)
|
||||
SELECT
|
||||
'seed-' || $1 || '-' || $2 || '-stats-' || d,
|
||||
$1, $2::measurement_type, $5 + (d * 7919) % ($5 / 2), $4, 'APPLE_HEALTH',
|
||||
(($7::date - d)::timestamp + interval '12 hours')
|
||||
AT TIME ZONE $8 AT TIME ZONE 'UTC',
|
||||
'stats:' || $3 || ':' || to_char($7::date - d, 'YYYY-MM-DD'),
|
||||
'EXPORT_XML_SOURCE_MAX', now(), now()
|
||||
FROM generate_series(91, $6::int) AS d
|
||||
`,
|
||||
USER_ID,
|
||||
type,
|
||||
hk,
|
||||
unit,
|
||||
base,
|
||||
days,
|
||||
ANCHOR,
|
||||
ZONE,
|
||||
);
|
||||
total += n;
|
||||
log(`${type} (stats:)`, n);
|
||||
}
|
||||
|
||||
// Sleep: 30 segments a night from 23:00 local, 16 minutes apart, rotating
|
||||
// through the stages. Distinct measured_at per segment.
|
||||
const sleep = await prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO measurements
|
||||
(id, user_id, type, value, unit, source, measured_at, external_id,
|
||||
sleep_stage, created_at, updated_at)
|
||||
SELECT
|
||||
'seed-' || $1 || '-sleep-' || d || '-' || s,
|
||||
$1, 'SLEEP_DURATION', 16, 'minutes', 'APPLE_HEALTH',
|
||||
(($3::date - d)::timestamp - interval '1 hour' + s * interval '16 minutes')
|
||||
AT TIME ZONE $4 AT TIME ZONE 'UTC',
|
||||
'hk-' || md5('sleep' || d || '-' || s),
|
||||
(ARRAY['CORE','DEEP','CORE','REM','AWAKE','CORE']::sleep_stage[])[1 + s % 6],
|
||||
now(), now()
|
||||
FROM generate_series(1, $2::int) AS d, generate_series(0, 29) AS s
|
||||
`,
|
||||
USER_ID,
|
||||
days,
|
||||
ANCHOR,
|
||||
ZONE,
|
||||
);
|
||||
total += sleep;
|
||||
log("SLEEP_DURATION (APPLE_HEALTH)", sleep);
|
||||
return total;
|
||||
}
|
||||
|
||||
async function seedWorkouts(days: number): Promise<number> {
|
||||
// About one workout every 29 hours, rotating through five sports.
|
||||
const count = Math.floor((days * 24) / 29);
|
||||
return prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO workouts
|
||||
(id, user_id, sport_type, started_at, ended_at, duration_sec,
|
||||
total_energy_kcal, total_distance_m, avg_heart_rate, max_heart_rate,
|
||||
source, external_id, created_at, updated_at)
|
||||
SELECT
|
||||
'seed-' || $1 || '-workout-' || g,
|
||||
$1,
|
||||
(ARRAY['walking','running','cycling','hiking','swimming'])[1 + g % 5],
|
||||
$3::date::timestamp - make_interval(hours => (g * 29)::int) + interval '17 hours',
|
||||
$3::date::timestamp - make_interval(hours => (g * 29)::int) + interval '17 hours'
|
||||
+ make_interval(secs => (1800 + (g * 97) % 3600)::int),
|
||||
(1800 + (g * 97) % 3600)::int,
|
||||
round((250 + (g * 13) % 400)::numeric, 1),
|
||||
round((3000 + (g * 211) % 9000)::numeric, 1),
|
||||
(115 + (g * 7) % 40)::int,
|
||||
(150 + (g * 5) % 35)::int,
|
||||
'APPLE_HEALTH',
|
||||
'hk-workout-' || md5($1 || g::text),
|
||||
now(), now()
|
||||
FROM generate_series(0, $2::int - 1) AS g
|
||||
`,
|
||||
USER_ID,
|
||||
count,
|
||||
ANCHOR,
|
||||
);
|
||||
}
|
||||
|
||||
async function seedMood(days: number): Promise<number> {
|
||||
const span = Math.min(days, 3 * 365);
|
||||
return prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO mood_entries
|
||||
(id, user_id, date, mood, score, mood_a1, stress_a2, energy_a3,
|
||||
source, mood_logged_at, tz, synced_at, created_at, updated_at)
|
||||
SELECT
|
||||
'seed-' || $1 || '-mood-' || d,
|
||||
$1,
|
||||
to_char($3::date - d, 'YYYY-MM-DD'),
|
||||
(ARRAY['LAUSIG','SCHLECHT','OKAY','GUT','SUPER_GUT'])[1 + (d * 7) % 5],
|
||||
1 + (d * 7) % 5,
|
||||
2 * ((d * 7) % 5) + 1,
|
||||
(d * 3) % 11,
|
||||
(d * 5) % 11,
|
||||
'MANUAL',
|
||||
(($3::date - d)::timestamp + interval '20 hours') AT TIME ZONE $4 AT TIME ZONE 'UTC',
|
||||
$4,
|
||||
now(), now(), now()
|
||||
FROM generate_series(1, $2::int) AS d
|
||||
`,
|
||||
USER_ID,
|
||||
span,
|
||||
ANCHOR,
|
||||
ZONE,
|
||||
);
|
||||
}
|
||||
|
||||
async function seedMedications(days: number): Promise<number> {
|
||||
const span = Math.min(days, 3 * 365);
|
||||
const meds = [
|
||||
{ key: "a", name: "Synthetic A", dose: "5 mg", window: ["08:00", "09:00"] },
|
||||
{
|
||||
key: "b",
|
||||
name: "Synthetic B",
|
||||
dose: "100 mg",
|
||||
window: ["08:00", "09:00"],
|
||||
},
|
||||
{
|
||||
key: "c",
|
||||
name: "Synthetic C",
|
||||
dose: "20 mg",
|
||||
window: ["20:00", "21:00"],
|
||||
},
|
||||
];
|
||||
let events = 0;
|
||||
for (const m of meds) {
|
||||
const medId = `seed-${USER_ID}-med-${m.key}`;
|
||||
await prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO medications
|
||||
(id, user_id, name, dose, starts_on, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, ($5::date - $6::int)::timestamp, now(), now())
|
||||
`,
|
||||
medId,
|
||||
USER_ID,
|
||||
m.name,
|
||||
m.dose,
|
||||
ANCHOR,
|
||||
span,
|
||||
);
|
||||
await prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO medication_schedules
|
||||
(id, medication_id, window_start, window_end, schedule_type)
|
||||
VALUES ($1, $2, $3, $4, 'SCHEDULED')
|
||||
`,
|
||||
`${medId}-schedule`,
|
||||
medId,
|
||||
m.window[0],
|
||||
m.window[1],
|
||||
);
|
||||
// One intake event per day; about one in twelve skipped, the rest taken
|
||||
// a few minutes into the window.
|
||||
events += await prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO medication_intake_events
|
||||
(id, user_id, medication_id, scheduled_for, taken_at, skipped,
|
||||
attribution_source, source, created_at, updated_at)
|
||||
SELECT
|
||||
$1 || '-intake-' || d,
|
||||
$2,
|
||||
$1,
|
||||
(($3::date - d)::timestamp + $5::time) AT TIME ZONE $6 AT TIME ZONE 'UTC',
|
||||
CASE WHEN d % 12 = 0 THEN NULL
|
||||
ELSE (($3::date - d)::timestamp + $5::time
|
||||
+ make_interval(mins => (d * 7) % 40)) AT TIME ZONE $6 AT TIME ZONE 'UTC'
|
||||
END,
|
||||
d % 12 = 0,
|
||||
'AUTO',
|
||||
'WEB',
|
||||
now(), now()
|
||||
FROM generate_series(1, $4::int) AS d
|
||||
`,
|
||||
medId,
|
||||
USER_ID,
|
||||
ANCHOR,
|
||||
span,
|
||||
m.window[0],
|
||||
ZONE,
|
||||
);
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const days = Math.round(YEARS * 365);
|
||||
const started = Date.now();
|
||||
|
||||
if (RESET) {
|
||||
await prisma.$executeRawUnsafe(`DELETE FROM users WHERE id = $1`, USER_ID);
|
||||
}
|
||||
await prisma.$executeRawUnsafe(
|
||||
`INSERT INTO users (id, username, timezone, onboarding_completed_at, updated_at)
|
||||
VALUES ($1, $1, $2, now(), now())
|
||||
ON CONFLICT (id) DO NOTHING`,
|
||||
USER_ID,
|
||||
ZONE,
|
||||
);
|
||||
|
||||
const measurements = await seedMeasurements(days);
|
||||
log("measurements total", measurements);
|
||||
log("workouts", await seedWorkouts(days));
|
||||
log("mood entries", await seedMood(days));
|
||||
log("medication intake events", await seedMedications(days));
|
||||
|
||||
if (ROLLUPS) {
|
||||
const folded = await recomputeUserRollups(USER_ID, {
|
||||
granularities: ALL_GRANULARITIES,
|
||||
statementTimeoutMs: 600_000,
|
||||
});
|
||||
log("rollup rows", folded.rowsUpserted);
|
||||
}
|
||||
|
||||
for (const table of [
|
||||
"measurements",
|
||||
"measurement_rollups",
|
||||
"workouts",
|
||||
"mood_entries",
|
||||
"medication_intake_events",
|
||||
]) {
|
||||
await prisma.$executeRawUnsafe(`ANALYZE ${table}`);
|
||||
}
|
||||
console.log(`done in ${((Date.now() - started) / 1000).toFixed(1)} s`);
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((err) => {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
})
|
||||
.finally(() => prisma.$disconnect());
|
||||
@@ -1,296 +0,0 @@
|
||||
/**
|
||||
* Seed one account shaped like a multi-year Apple Health `export.xml` import,
|
||||
* for reproducing the maintenance jobs at the size a real heavy account
|
||||
* reaches (issue #1031: about 1.26 million measurement rows).
|
||||
*
|
||||
* DATABASE_URL=postgresql://… pnpm dlx tsx scripts/seed-large-account.ts \
|
||||
* [--user bench-large] [--years 4] [--pulse-seconds 165]
|
||||
*
|
||||
* Never point this at a database that holds real data: it inserts rows under
|
||||
* the given user id and, with `--reset`, deletes that user first.
|
||||
*
|
||||
* The shape follows what the import and the iOS sync actually write:
|
||||
*
|
||||
* - dense spot rows (heart rate every `--pulse-seconds`, HRV, SpO2) keyed by
|
||||
* a hashed sample id, the tier the dense-intraday retention folds;
|
||||
* - high-frequency mean rows (respiratory rate, gait metrics, audio
|
||||
* exposure), the tier the daily-mean consolidation folds;
|
||||
* - one `stats:` daily row per cumulative type and day (what the export
|
||||
* import writes for steps, energy, distance and flights);
|
||||
* - per-sample cumulative rows with opaque sample ids for the most recent
|
||||
* 90 days (what the iOS sync writes, and what the cumulative drain folds);
|
||||
* - sleep-stage segments every night.
|
||||
*
|
||||
* Everything is generated inside Postgres with `generate_series`, so a million
|
||||
* rows take seconds rather than a million round trips. `ANALYZE` runs at the
|
||||
* end: a planner reading pre-seed statistics picks plans no production table
|
||||
* of this size would get.
|
||||
*/
|
||||
import { prisma } from "@/lib/db";
|
||||
|
||||
function arg(name: string, fallback: string): string {
|
||||
const i = process.argv.indexOf(`--${name}`);
|
||||
return i >= 0 && process.argv[i + 1] ? process.argv[i + 1]! : fallback;
|
||||
}
|
||||
|
||||
const USER_ID = arg("user", "bench-large");
|
||||
const YEARS = Number(arg("years", "4"));
|
||||
const PULSE_SECONDS = Number(arg("pulse-seconds", "165"));
|
||||
const RESET = process.argv.includes("--reset");
|
||||
|
||||
/** A spot series: `perDay` evenly spaced rows per day over the whole span. */
|
||||
interface SpotSeries {
|
||||
type: string;
|
||||
unit: string;
|
||||
perDay: number;
|
||||
base: number;
|
||||
spread: number;
|
||||
/** Days back from now the series covers. Defaults to the whole span. */
|
||||
days?: number;
|
||||
/** externalId prefix; `stats:` rows are generated separately. */
|
||||
idPrefix: string;
|
||||
}
|
||||
|
||||
async function insertSpot(s: SpotSeries, days: number): Promise<number> {
|
||||
const step = Math.floor(86_400 / s.perDay);
|
||||
const count = Math.floor((days * 86_400) / step);
|
||||
// Jitter stays below the step so every measured_at is distinct per type
|
||||
// (the (user, type, measured_at, source, sleep_stage) unique index).
|
||||
const jitter = Math.max(1, Math.min(59, step - 1));
|
||||
return prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO measurements
|
||||
(id, user_id, type, value, unit, source, measured_at, external_id, created_at, updated_at)
|
||||
SELECT
|
||||
'seed-' || $1 || '-' || $2 || '-' || g,
|
||||
$1,
|
||||
$2::measurement_type,
|
||||
round(($3 + $4 * sin(g / 37.0) + ($4 / 2) * ((g * 7919) % 100) / 100.0)::numeric, 2),
|
||||
$5,
|
||||
'APPLE_HEALTH',
|
||||
date_trunc('minute', now()) - make_interval(secs => $6 * $7::bigint)
|
||||
+ make_interval(secs => g * $6 + (g * 31) % $8),
|
||||
$9 || md5($2 || g::text),
|
||||
now(), now()
|
||||
FROM generate_series(0, $7::bigint - 1) AS g
|
||||
`,
|
||||
USER_ID,
|
||||
s.type,
|
||||
s.base,
|
||||
s.spread,
|
||||
s.unit,
|
||||
step,
|
||||
count,
|
||||
jitter,
|
||||
s.idPrefix,
|
||||
);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const days = Math.round(YEARS * 365);
|
||||
|
||||
if (RESET) {
|
||||
await prisma.$executeRawUnsafe(`DELETE FROM users WHERE id = $1`, USER_ID);
|
||||
}
|
||||
await prisma.$executeRawUnsafe(
|
||||
`INSERT INTO users (id, username, timezone, updated_at)
|
||||
VALUES ($1, $1, 'Europe/Berlin', now())
|
||||
ON CONFLICT (id) DO NOTHING`,
|
||||
USER_ID,
|
||||
);
|
||||
|
||||
const pulsePerDay = Math.floor(86_400 / PULSE_SECONDS);
|
||||
const spot: SpotSeries[] = [
|
||||
{
|
||||
type: "PULSE",
|
||||
unit: "bpm",
|
||||
perDay: pulsePerDay,
|
||||
base: 62,
|
||||
spread: 25,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "HEART_RATE_VARIABILITY",
|
||||
unit: "ms",
|
||||
perDay: 12,
|
||||
base: 45,
|
||||
spread: 20,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "OXYGEN_SATURATION",
|
||||
unit: "%",
|
||||
perDay: 24,
|
||||
base: 96,
|
||||
spread: 2,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "RESPIRATORY_RATE",
|
||||
unit: "breaths/min",
|
||||
perDay: 48,
|
||||
base: 14,
|
||||
spread: 3,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "WALKING_SPEED",
|
||||
unit: "km/h",
|
||||
perDay: 30,
|
||||
base: 4.8,
|
||||
spread: 0.8,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "WALKING_STEP_LENGTH",
|
||||
unit: "cm",
|
||||
perDay: 30,
|
||||
base: 72,
|
||||
spread: 6,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "WALKING_ASYMMETRY",
|
||||
unit: "%",
|
||||
perDay: 30,
|
||||
base: 3,
|
||||
spread: 2,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "WALKING_DOUBLE_SUPPORT",
|
||||
unit: "%",
|
||||
perDay: 30,
|
||||
base: 28,
|
||||
spread: 3,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
{
|
||||
type: "AUDIO_EXPOSURE_ENV",
|
||||
unit: "dB",
|
||||
perDay: 60,
|
||||
base: 62,
|
||||
spread: 12,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
// The iOS sync's per-sample cumulative rows: dense, recent, opaque ids.
|
||||
{
|
||||
type: "ACTIVITY_STEPS",
|
||||
unit: "steps",
|
||||
perDay: 100,
|
||||
base: 90,
|
||||
spread: 60,
|
||||
days: 90,
|
||||
idPrefix: "uuid-",
|
||||
},
|
||||
{
|
||||
type: "ACTIVE_ENERGY_BURNED",
|
||||
unit: "kcal",
|
||||
perDay: 400,
|
||||
base: 1.2,
|
||||
spread: 0.9,
|
||||
days: 90,
|
||||
idPrefix: "uuid-",
|
||||
},
|
||||
{
|
||||
type: "WALKING_RUNNING_DISTANCE",
|
||||
unit: "m",
|
||||
perDay: 100,
|
||||
base: 70,
|
||||
spread: 50,
|
||||
days: 90,
|
||||
idPrefix: "uuid-",
|
||||
},
|
||||
{
|
||||
type: "WEIGHT",
|
||||
unit: "kg",
|
||||
perDay: 1,
|
||||
base: 82,
|
||||
spread: 1.5,
|
||||
idPrefix: "hk-",
|
||||
},
|
||||
];
|
||||
|
||||
let total = 0;
|
||||
for (const s of spot) {
|
||||
const n = await insertSpot(s, s.days ?? days);
|
||||
total += n;
|
||||
console.log(`${s.type.padEnd(28)} ${String(n).padStart(9)}`);
|
||||
}
|
||||
|
||||
// One export-estimate `stats:` row per cumulative type and local day, at
|
||||
// local noon — what `flushCumulativeBuckets` in the export import writes.
|
||||
for (const [type, hk, unit, base] of [
|
||||
["ACTIVITY_STEPS", "HKQuantityTypeIdentifierStepCount", "steps", 8000],
|
||||
[
|
||||
"ACTIVE_ENERGY_BURNED",
|
||||
"HKQuantityTypeIdentifierActiveEnergyBurned",
|
||||
"kcal",
|
||||
550,
|
||||
],
|
||||
[
|
||||
"WALKING_RUNNING_DISTANCE",
|
||||
"HKQuantityTypeIdentifierDistanceWalkingRunning",
|
||||
"m",
|
||||
6000,
|
||||
],
|
||||
["FLIGHTS_CLIMBED", "HKQuantityTypeIdentifierFlightsClimbed", "count", 9],
|
||||
] as const) {
|
||||
const n = await prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO measurements
|
||||
(id, user_id, type, value, unit, source, measured_at, external_id,
|
||||
aggregation_provenance, created_at, updated_at)
|
||||
SELECT
|
||||
'seed-' || $1 || '-' || $2 || '-stats-' || d,
|
||||
$1, $2::measurement_type, $5 + (d * 7919) % ($5 / 2), $4, 'APPLE_HEALTH',
|
||||
((current_date - d)::timestamp + interval '12 hours')
|
||||
AT TIME ZONE 'Europe/Berlin' AT TIME ZONE 'UTC',
|
||||
'stats:' || $3 || ':' || to_char(current_date - d, 'YYYY-MM-DD'),
|
||||
'EXPORT_XML_SOURCE_MAX', now(), now()
|
||||
FROM generate_series(91, $6::int) AS d
|
||||
`,
|
||||
USER_ID,
|
||||
type,
|
||||
hk,
|
||||
unit,
|
||||
base,
|
||||
days,
|
||||
);
|
||||
total += n;
|
||||
console.log(`${`${type} (stats:)`.padEnd(28)} ${String(n).padStart(9)}`);
|
||||
}
|
||||
|
||||
// Sleep: 30 segments a night from 23:00 local, 16 minutes apart, rotating
|
||||
// through the stages. Distinct measured_at per segment.
|
||||
const sleep = await prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO measurements
|
||||
(id, user_id, type, value, unit, source, measured_at, external_id,
|
||||
sleep_stage, created_at, updated_at)
|
||||
SELECT
|
||||
'seed-' || $1 || '-sleep-' || d || '-' || s,
|
||||
$1, 'SLEEP_DURATION', 16, 'minutes', 'APPLE_HEALTH',
|
||||
((current_date - d)::timestamp - interval '1 hour' + s * interval '16 minutes')
|
||||
AT TIME ZONE 'Europe/Berlin' AT TIME ZONE 'UTC',
|
||||
'hk-' || md5('sleep' || d || '-' || s),
|
||||
(ARRAY['CORE','DEEP','CORE','REM','AWAKE','CORE']::sleep_stage[])[1 + s % 6],
|
||||
now(), now()
|
||||
FROM generate_series(1, $2::int) AS d, generate_series(0, 29) AS s
|
||||
`,
|
||||
USER_ID,
|
||||
days,
|
||||
);
|
||||
total += sleep;
|
||||
console.log(`${"SLEEP_DURATION".padEnd(28)} ${String(sleep).padStart(9)}`);
|
||||
|
||||
await prisma.$executeRawUnsafe(`ANALYZE measurements`);
|
||||
console.log(`${"total".padEnd(28)} ${String(total).padStart(9)}`);
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((err) => {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
})
|
||||
.finally(() => prisma.$disconnect());
|
||||
Reference in new issue
Block a user