perf(bench): profile a production boot and the idle heap

scripts/bench/boot-profile.ts starts the production build with every role
under the production heap ceiling against the seeded benchmark database,
and records the time to a healthy probe and to a ready worker, every
event-loop stall the process logs, the heap lines, the full collections
from --trace-gc, the main thread's busy stretches during the boot with
their self time by source module (resolved through the chunk source maps),
and one heap snapshot after an idle wait, summarised by node type,
constructor, largest strings and largest retained subtrees.

The baseline on the current runtime: a 753 ms busy stretch at 0.7 s, most
of it the worker graph, and a live heap of 164 to 172 MB after five idle
minutes, of which the query compiler holds about 50 MB and the eight
locale catalogs about 33 MB (32 parsed copies and two 9.7 MB chunk
sources).
This commit is contained in:
Marc-André Bombeck committed 2026-10-22 06:41:15 +02:00
1 parent 0f6d668df1
commit 63e8630fcb
4 files changed
+2569

No files matched your search

+398
View File
@@ -0,0 +1,398 @@
/**
* Boot and idle-memory profile of the production build.
*
* DATABASE_URL=postgresql://… API_TOKEN_HMAC_KEY=… ENCRYPTION_KEYS=… \
* ENCRYPTION_ACTIVE_KEY_ID=… pnpm dlx tsx scripts/bench/boot-profile.ts \
* [--idle-seconds 300] [--heap-mb 792] [--port 3101] \
* [--no-snapshot] [--no-cpu-prof] [--out scripts/bench/results/<label>-boot]
*
* Starts `next start` with every role (web and worker, as production runs it)
* under the production heap ceiling, against the seeded benchmark database
* (see docs/contributing/benchmarks.md; never a database with real data: a
* heap snapshot holds whatever the process decrypted). It records:
*
* - the time to the first healthy `/api/health` and to `[boot] background
* role ready`;
* - every `event_loop_stall` the process logs, with the task it names;
* - every `runtime.heap.sampled` line;
* - with `--cpu-prof` (default on), the busy stretches of the main thread
* over 100 ms during the boot, each with its self time by source module;
* - with a snapshot (default on), one heap snapshot after the idle wait
* (taken with `--heapsnapshot-signal`, so after a full GC: the live heap),
* summarised by node type, constructor, largest strings and the largest
* retained subtrees with a retaining path.
*
* The raw `.cpuprofile` and `.heapsnapshot` are written to a temporary
* directory and deleted after the summary unless `--keep` is given.
*
* Output: `<out>.json` and `<out>.md`.
*/
import { execSync, spawn } from "node:child_process";
import {
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { cpus, loadavg, tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { longTasks, selfTimeByModule, summariseHeap } from "./profile-analyze";
function arg(name: string, fallback: string): string {
const i = process.argv.indexOf(`--${name}`);
return i >= 0 && process.argv[i + 1] ? process.argv[i + 1]! : fallback;
}
const flag = (name: string) => process.argv.includes(`--${name}`);
const IDLE_SECONDS = Number(arg("idle-seconds", "300"));
const HEAP_MB = Number(arg("heap-mb", "792"));
const PORT = Number(arg("port", "3101"));
const OUT = arg("out", join("scripts", "bench", "results", "latest-boot"));
const SNAPSHOT = !flag("no-snapshot");
const CPU_PROF = !flag("no-cpu-prof");
const KEEP = flag("keep");
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
function find(obj: unknown, key: string): unknown {
if (!obj || typeof obj !== "object") return undefined;
const rec = obj as Record<string, unknown>;
if (key in rec) return rec[key];
for (const v of Object.values(rec)) {
const hit = find(v, key);
if (hit !== undefined) return hit;
}
return undefined;
}
async function main(): Promise<void> {
if (!process.env.DATABASE_URL) throw new Error("DATABASE_URL is required");
const work = mkdtempSync(join(tmpdir(), "hl-boot-profile-"));
const nodeOptions = [
`--max-old-space-size=${HEAP_MB}`,
...(CPU_PROF ? ["--cpu-prof", `--cpu-prof-dir=${work}`] : []),
...(SNAPSHOT ? ["--heapsnapshot-signal=SIGUSR2"] : []),
].join(" ");
const started = Date.now();
const child = spawn(
process.execPath,
[
// One line per collection; the mark-compacts say how much of the used
// heap was live and how much was garbage waiting for a full GC. Not
// allowed in NODE_OPTIONS, and `next start` serves in this process.
"--trace-gc",
"node_modules/next/dist/bin/next",
"start",
"-p",
String(PORT),
"-H",
"127.0.0.1",
],
{
cwd: process.cwd(),
env: {
...process.env,
NODE_ENV: "production",
PORT: String(PORT),
HOSTNAME: "127.0.0.1",
HEALTHLOG_PROCESS_TYPE: "all",
SESSION_COOKIE_SECURE: "false",
NODE_OPTIONS: nodeOptions,
},
stdio: ["ignore", "pipe", "pipe"],
},
);
// Node writes the snapshot into the process's cwd.
const snapshotDir = process.cwd();
const stalls: Array<Record<string, unknown>> = [];
const heapLines: Array<Record<string, unknown>> = [];
const bootLines: string[] = [];
const markCompacts: Array<{
atMs: number;
beforeMb: number;
afterMb: number;
}> = [];
let readyAtMs: number | null = null;
let snapshotTaken = false;
let buffer = "";
const onLine = (line: string) => {
const gc =
/Mark-Compact(?: \(reduce\))? ([\d.]+) \([\d.]+\) -> ([\d.]+) \(/.exec(
line,
);
if (gc) {
markCompacts.push({
atMs: Date.now() - started,
beforeMb: Number(gc[1]),
afterMb: Number(gc[2]),
});
return;
}
if (line.includes("[boot]")) {
bootLines.push(`${Date.now() - started} ms: ${line.slice(0, 200)}`);
if (line.includes("background role ready"))
readyAtMs = Date.now() - started;
}
if (!line.startsWith("{")) return;
try {
const evt = JSON.parse(line) as Record<string, unknown>;
const task = find(evt, "task_name");
if (task === "startup" && readyAtMs === null) {
readyAtMs = Date.now() - started;
}
if (task === "event_loop_stall") {
// The snapshot itself blocks the process for seconds.
if (snapshotTaken) return;
stalls.push({
atMs: Date.now() - started,
...(find(evt, "result") as object),
});
} else if (task === "runtime.heap.sampled") {
heapLines.push({
atMs: Date.now() - started,
...((find(evt, "meta") as object) ?? {}),
});
} else if (typeof task === "string" && task.startsWith("boot.")) {
bootLines.push(
`${Date.now() - started} ms: ${task} ${JSON.stringify(find(evt, "meta") ?? {})}`.slice(
0,
400,
),
);
}
} catch {
// not a wide event
}
};
child.stdout!.on("data", (d: Buffer) => {
buffer += d.toString();
let nl: number;
while ((nl = buffer.indexOf("\n")) >= 0) {
onLine(buffer.slice(0, nl));
buffer = buffer.slice(nl + 1);
}
});
child.stderr!.on("data", (d: Buffer) => {
for (const line of d.toString().split("\n"))
if (line.includes("[boot]")) onLine(line);
});
let healthyAtMs: number | null = null;
const deadline = Date.now() + 180_000;
while (Date.now() < deadline && healthyAtMs === null) {
if (child.exitCode !== null)
throw new Error(`server exited ${child.exitCode}`);
try {
const res = await fetch(`http://127.0.0.1:${PORT}/api/health`);
if (res.status < 500 || res.status === 503) {
const body = await res.text();
if (res.ok || body.includes("starting"))
healthyAtMs = Date.now() - started;
}
} catch {
// not listening yet
}
await sleep(100);
}
while (readyAtMs === null && Date.now() < deadline) await sleep(200);
console.log(
`health ${healthyAtMs} ms, worker ready ${readyAtMs} ms; idling ${IDLE_SECONDS} s`,
);
await sleep(IDLE_SECONDS * 1000);
let heap: ReturnType<typeof summariseHeap> | null = null;
if (SNAPSHOT) {
const before = new Set(readdirSync(snapshotDir));
snapshotTaken = true;
child.kill("SIGUSR2");
let file = "";
let lastSize = -1;
for (let i = 0; i < 600; i++) {
await sleep(1000);
const f = readdirSync(snapshotDir).find(
(n) => n.endsWith(".heapsnapshot") && !before.has(n),
);
if (!f) continue;
const size = statSync(join(snapshotDir, f)).size;
if (size === lastSize && size > 0) {
file = join(snapshotDir, f);
break;
}
lastSize = size;
}
if (file) {
// A snapshot is finished when the process answers again.
await fetch(`http://127.0.0.1:${PORT}/api/health`).catch(() => undefined);
const moved = join(work, "idle.heapsnapshot");
execSync(`mv ${JSON.stringify(file)} ${JSON.stringify(moved)}`);
console.log("summarising heap snapshot");
heap = summariseHeap(moved, {
"parsed locale catalogs": ["common", "apiErrors", "doctorReport"],
});
}
}
const exited = new Promise((r) => child.once("exit", r));
child.kill("SIGTERM");
await Promise.race([exited, sleep(60_000)]);
if (child.exitCode === null) child.kill("SIGKILL");
let tasks: ReturnType<typeof longTasks> = [];
let bootSelf: Array<[string, number]> = [];
if (CPU_PROF) {
await sleep(1000);
const profiles = readdirSync(work).filter((n) => n.endsWith(".cpuprofile"));
// The largest profile is the server process (`next start` may run a
// small parent).
const main = profiles
.map((n) => join(work, n))
.sort((a, b) => statSync(b).size - statSync(a).size)[0];
if (main) {
const profile = JSON.parse(readFileSync(main, "utf8"));
tasks = longTasks(profile, 50, 0.5).filter((t) => t.startMs < 120_000);
bootSelf = selfTimeByModule(
profile,
0,
(readyAtMs ?? 30_000) + 5_000,
).slice(0, 40);
}
}
if (!KEEP) rmSync(work, { recursive: true, force: true });
else console.log(`raw files kept in ${work}`);
let commit = "unknown";
try {
commit = execSync("git rev-parse --short HEAD").toString().trim();
} catch {
// not a checkout
}
const meta = {
label: OUT.split("/").pop(),
commit,
date: new Date().toISOString(),
node: process.version,
cpus: cpus().length,
loadAverage: loadavg().map((n) => Math.round(n * 10) / 10),
heapMb: HEAP_MB,
idleSeconds: IDLE_SECONDS,
healthyAtMs,
readyAtMs,
stallMaxMs: Math.max(0, ...stalls.map((s) => Number(s.lag_ms ?? 0))),
};
mkdirSync(dirname(OUT), { recursive: true });
// Paths in frame and retainer names are made relative to the checkout.
const relative = (text: string) => text.split(`${process.cwd()}/`).join("");
writeFileSync(
`${OUT}.json`,
relative(
`${JSON.stringify({ meta, bootLines, stalls, heapLines, markCompacts, tasks, bootSelf, heap }, null, 2)}\n`,
),
);
const md: string[] = [
`# Boot profile ${String(meta.label)}`,
"",
`Commit ${commit}, ${meta.date}. Production build, every role, heap ceiling ${HEAP_MB} MB, ${IDLE_SECONDS} s idle. ${meta.cpus} cores, load average ${meta.loadAverage.join(",")} at the end.`,
"",
`- first answer from /api/health: ${healthyAtMs} ms; background role ready: ${readyAtMs} ms`,
`- event-loop stalls logged: ${stalls.length}, max ${meta.stallMaxMs} ms`,
"",
"## Stalls",
"",
...stalls.map((s) => `- ${JSON.stringify(s)}`),
"",
"## Heap samples",
"",
"| at s | used MB | old | large obj | code | external | array buffers | rss |",
"| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
...heapLines.map(
(h) =>
`| ${Math.round(Number(h.atMs) / 1000)} | ${h.heap_used_mb} | ${h.old_space_mb} | ${h.large_object_space_mb} | ${h.code_space_mb} | ${h.external_mb} | ${h.array_buffers_mb} | ${h.rss_mb} |`,
),
"",
"## Full collections (mark-compact)",
"",
`${markCompacts.length} mark-compacts during the run. Used heap before the collection, and what survived it (the live heap):`,
"",
"| at s | before MB | after MB |",
"| ---: | ---: | ---: |",
...markCompacts
.filter(
(m, i, all) =>
i === all.length - 1 || i % Math.ceil(all.length / 15) === 0,
)
.map(
(m) =>
`| ${Math.round(m.atMs / 1000)} | ${m.beforeMb} | ${m.afterMb} |`,
),
"",
"## Main-thread busy stretches over 50 ms without an idle sample, first two minutes",
"",
...tasks.flatMap((t) => [
`### at ${t.startMs} ms, ${t.durationMs} ms`,
"",
"| self time by module | ms |",
"| --- | ---: |",
...t.byModule.slice(0, 10).map(([m, v]) => `| ${m} | ${v} |`),
"",
"| inclusive, application modules | ms |",
"| --- | ---: |",
...t.byEntry.slice(0, 10).map(([m, v]) => `| ${m} | ${v} |`),
"",
]),
"## Self time by module until the worker is ready",
"",
"| module | ms |",
"| --- | ---: |",
...bootSelf.slice(0, 25).map(([m, v]) => `| ${m} | ${v} |`),
"",
];
if (heap) {
md.push(
`## Heap snapshot after ${IDLE_SECONDS} s idle (live heap after a full GC)`,
"",
`${heap.totalMb} MB in ${heap.nodeCount} nodes; module source strings over 50 KB: ${heap.sourceStringsMb} MB.`,
"",
...Object.entries(heap.probes).map(
([k, v]) => `- ${k}: ${v.count} objects, ${v.retainedMb} MB retained`,
),
"",
"| node type | MB |",
"| --- | ---: |",
...heap.byType.map(([k, v]) => `| ${k} | ${v} |`),
"",
"| constructor | MB | count |",
"| --- | ---: | ---: |",
...heap.byConstructor
.slice(0, 20)
.map(([k, v, c]) => `| ${k.replace(/\|/g, "/")} | ${v} | ${c} |`),
"",
"| largest retained subtrees | MB | retaining path |",
"| --- | ---: | --- |",
...heap.topRetainers
.slice(0, 30)
.map(
(r) =>
`| ${r.name.replace(/\|/g, "/")} | ${r.mb} | ${r.path.replace(/\|/g, "/").slice(0, 160)} |`,
),
"",
"| largest strings | MB |",
"| --- | ---: |",
...heap.largestStrings
.slice(0, 15)
.map(([s, t]) => `| ${t.replace(/\|/g, "/").slice(0, 90)} | ${s} |`),
"",
);
}
writeFileSync(`${OUT}.md`, relative(md.join("\n")));
console.log(`wrote ${OUT}.json and ${OUT}.md`);
}
main().catch((err) => {
console.error(err);
process.exitCode = 1;
});
+599
View File
@@ -0,0 +1,599 @@
/**
* Readers for the two files `boot-profile.ts` collects: a V8 CPU profile
* (`.cpuprofile`) and a heap snapshot (`.heapsnapshot`). Both are plain
* JSON; these turn them into the few tables a boot investigation needs, so a
* before/after comparison does not depend on opening DevTools by hand.
*
* Frames are attributed to source modules through the module ids Turbopack
* writes into every server chunk (`"[project]/src/lib/x.ts [app-route] …"`):
* a sample at chunk line N belongs to the last module id that starts at or
* before N.
*/
import { existsSync, readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
// ---------------------------------------------------------------- modules
const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
const B64_INDEX = new Map([...B64].map((c, i) => [c, i]));
interface ChunkMap {
sources: string[];
/** Per generated line: [column, source index] pairs, ascending. */
lines: Array<Array<[number, number]>>;
}
const chunkMapCache = new Map<string, ChunkMap | null>();
/** Decode a chunk's source map down to (line, column) → source. */
function chunkMap(url: string): ChunkMap | null {
if (chunkMapCache.has(url)) return chunkMapCache.get(url)!;
let path = url;
try {
if (url.startsWith("file://")) path = fileURLToPath(url);
} catch {
// not a file URL
}
let result: ChunkMap | null = null;
if (existsSync(`${path}.map`)) {
type RawMap = {
sources?: string[];
mappings?: string;
sections?: Array<{ offset: { line: number }; map: RawMap }>;
};
const raw = JSON.parse(readFileSync(`${path}.map`, "utf8")) as RawMap;
const sources: string[] = [];
const lines: Array<Array<[number, number]>> = [];
// An index map (`sections`) holds one ordinary map per offset.
const flat: Array<{ line: number; map: RawMap }> = raw.sections
? raw.sections.map((s) => ({ line: s.offset.line, map: s.map }))
: [{ line: 0, map: raw }];
for (const { line: lineOffset, map } of flat) {
const base = sources.length;
sources.push(...(map.sources ?? []));
let src = 0;
let lineNo = lineOffset;
for (const line of (map.mappings ?? "").split(";")) {
const segs: Array<[number, number]> = lines[lineNo] ?? [];
let col = 0;
for (const seg of line.split(",")) {
if (!seg) continue;
const vals: number[] = [];
let shift = 0;
let value = 0;
for (const ch of seg) {
const d = B64_INDEX.get(ch)!;
value += (d & 31) << shift;
if (d & 32) shift += 5;
else {
const neg = value & 1;
value >>= 1;
vals.push(neg ? -value : value);
value = 0;
shift = 0;
}
}
col += vals[0] ?? 0;
if (vals.length > 1) {
src += vals[1]!;
segs.push([col, base + src]);
}
}
lines[lineNo] = segs;
lineNo++;
}
}
for (const segs of lines) segs?.sort((x, y) => x[0] - y[0]);
result = { sources, lines };
}
chunkMapCache.set(url, result);
return result;
}
function packageOf(path: string): string {
const parts = path.split("node_modules/");
const rest = parts[parts.length - 1]!.split("/");
return `npm:${rest[0]!.startsWith("@") ? `${rest[0]}/${rest[1]}` : rest[0]}`;
}
/** The source module a frame at (url, 0-based line, column) belongs to. */
export function moduleForFrame(url: string, line: number, column = 0): string {
if (!url) return "(native)";
if (url.startsWith("node:")) return url;
if (url.includes("/node_modules/") && !url.includes("/.next/")) {
return packageOf(url);
}
const map = chunkMap(url);
const segs = map?.lines[line];
if (map && segs && segs.length > 0) {
let lo = 0;
let hi = segs.length - 1;
let found = segs[0]![1];
while (lo <= hi) {
const mid = (lo + hi) >> 1;
if (segs[mid]![0] <= column) {
found = segs[mid]![1];
lo = mid + 1;
} else hi = mid - 1;
}
const source = (map.sources[found] ?? "")
.replace(/^turbopack:\/\/\/\[project\]\//, "")
.replace(/^(\.\.\/)+/, "");
if (source.includes("node_modules/")) return packageOf(source);
return source || url;
}
return url.replace(/^.*\/\.next\//, ".next/");
}
// ------------------------------------------------------------ CPU profile
interface CpuNode {
id: number;
callFrame: {
functionName: string;
url: string;
lineNumber: number;
columnNumber: number;
};
children?: number[];
}
interface CpuProfile {
nodes: CpuNode[];
startTime: number;
endTime: number;
samples: number[];
timeDeltas: number[];
}
export interface LongTask {
startMs: number;
durationMs: number;
/** Self time by source module, descending. */
byModule: Array<[string, number]>;
/** Self time by function, descending. */
byFunction: Array<[string, number]>;
/** Inclusive time of the outermost application frames. */
byEntry: Array<[string, number]>;
}
const IDLE = new Set(["(idle)", "(root)"]);
/**
* Busy stretches of the main thread: consecutive non-idle samples whose gaps
* stay under `gapMs`, longer than `minMs`. Each comes with its self time by
* module and by function.
*/
export function longTasks(
profile: CpuProfile,
minMs = 100,
gapMs = 5,
): LongTask[] {
const byId = new Map<number, CpuNode>();
const parent = new Map<number, number>();
for (const n of profile.nodes) {
byId.set(n.id, n);
for (const c of n.children ?? []) parent.set(c, n.id);
}
const tasks: LongTask[] = [];
let current: {
start: number;
end: number;
samples: Array<[number, number]>;
} | null = null;
const flush = () => {
if (!current) return;
const dur = (current.end - current.start) / 1000;
if (dur >= minMs) {
const mods = new Map<string, number>();
const fns = new Map<string, number>();
const entries = new Map<string, number>();
for (const [id, dt] of current.samples) {
const n = byId.get(id)!;
const cf = n.callFrame;
const mod = moduleForFrame(cf.url, cf.lineNumber, cf.columnNumber);
mods.set(mod, (mods.get(mod) ?? 0) + dt);
const fn = `${cf.functionName || "(anonymous)"} ${mod}`;
fns.set(fn, (fns.get(fn) ?? 0) + dt);
// Inclusive: every distinct module on the stack gets the sample once.
const seen = new Set<string>();
let cur: number | undefined = id;
while (cur !== undefined) {
const c = byId.get(cur)!.callFrame;
if (c.url) {
const m = moduleForFrame(c.url, c.lineNumber, c.columnNumber);
if (m.startsWith("src/") && !seen.has(m)) {
seen.add(m);
entries.set(m, (entries.get(m) ?? 0) + dt);
}
}
cur = parent.get(cur);
}
}
const top = (m: Map<string, number>) =>
[...m.entries()]
.map(([k, v]) => [k, Math.round(v / 100) / 10] as [string, number])
.sort((a, b) => b[1] - a[1])
.slice(0, 15);
tasks.push({
startMs: Math.round((current.start - profile.startTime) / 1000),
durationMs: Math.round(dur),
byModule: top(mods),
byFunction: top(fns),
byEntry: top(entries),
});
}
current = null;
};
let at = profile.startTime;
for (let i = 0; i < profile.samples.length; i++) {
at += profile.timeDeltas[i] ?? 0;
const id = profile.samples[i]!;
const name = byId.get(id)?.callFrame.functionName ?? "";
const dt = profile.timeDeltas[i + 1] ?? 0;
if (IDLE.has(name)) {
if (current && at - current.end > gapMs * 1000) flush();
continue;
}
if (!current) current = { start: at, end: at, samples: [] };
else if (at - current.end > gapMs * 1000) {
flush();
current = { start: at, end: at, samples: [] };
}
current.end = at + dt;
current.samples.push([id, dt]);
}
flush();
return tasks;
}
/** Self time per module over the whole profile, or a window of it. */
export function selfTimeByModule(
profile: CpuProfile,
fromMs = 0,
toMs = Infinity,
): Array<[string, number]> {
const byId = new Map(profile.nodes.map((n) => [n.id, n]));
const mods = new Map<string, number>();
let at = profile.startTime;
for (let i = 0; i < profile.samples.length; i++) {
at += profile.timeDeltas[i] ?? 0;
const rel = (at - profile.startTime) / 1000;
if (rel < fromMs || rel > toMs) continue;
const n = byId.get(profile.samples[i]!)!;
if (IDLE.has(n.callFrame.functionName)) continue;
const mod = moduleForFrame(
n.callFrame.url,
n.callFrame.lineNumber,
n.callFrame.columnNumber,
);
const dt = profile.timeDeltas[i + 1] ?? 0;
mods.set(mod, (mods.get(mod) ?? 0) + dt);
}
return [...mods.entries()]
.map(([k, v]) => [k, Math.round(v / 100) / 10] as [string, number])
.sort((a, b) => b[1] - a[1]);
}
// ---------------------------------------------------------- heap snapshot
interface HeapSnapshot {
snapshot: {
meta: {
node_fields: string[];
node_types: [string[], ...unknown[]];
edge_fields: string[];
edge_types: [string[], ...unknown[]];
};
node_count: number;
};
nodes: number[];
edges: number[];
strings: string[];
}
export interface HeapSummary {
totalMb: number;
nodeCount: number;
byType: Array<[string, number]>;
byConstructor: Array<[string, number, number]>;
largestStrings: Array<[number, string]>;
/** Retained size of the largest dominator subtrees below the GC roots. */
topRetainers: Array<{ mb: number; name: string; path: string }>;
/** Module source strings (whole chunk scripts) over 50 KB. */
sourceStringsMb: number;
/**
* For each probe, the objects that carry every listed property: how many,
* and their summed retained size (the outermost ones only).
*/
probes: Record<string, { count: number; retainedMb: number }>;
}
/**
* Self size by node type and constructor, the largest strings, and retained
* sizes from the dominator tree (Cooper, Harvey, Kennedy), with a retaining
* path for each of the largest subtrees.
*/
export function summariseHeap(
file: string,
probes: Record<string, string[]> = {},
): HeapSummary {
const snap = JSON.parse(readFileSync(file, "utf8")) as HeapSnapshot;
const nf = snap.snapshot.meta.node_fields;
const ef = snap.snapshot.meta.edge_fields;
const nodeTypes = snap.snapshot.meta.node_types[0];
const edgeTypes = snap.snapshot.meta.edge_types[0];
const NF = nf.length;
const EF = ef.length;
const iType = nf.indexOf("type");
const iName = nf.indexOf("name");
const iSelf = nf.indexOf("self_size");
const iEdges = nf.indexOf("edge_count");
const eType = ef.indexOf("type");
const eName = ef.indexOf("name_or_index");
const eTo = ef.indexOf("to_node");
const weak = edgeTypes.indexOf("weak");
const shortcut = edgeTypes.indexOf("shortcut");
const elementT = edgeTypes.indexOf("element");
const hiddenT = edgeTypes.indexOf("hidden");
const N = snap.nodes.length / NF;
const nodes = snap.nodes;
const edges = snap.edges;
const strings = snap.strings;
const firstEdge = new Uint32Array(N + 1);
for (let i = 0, e = 0; i < N; i++) {
firstEdge[i] = e;
e += nodes[i * NF + iEdges]! * EF;
}
firstEdge[N] = edges.length;
const self = (i: number) => nodes[i * NF + iSelf]!;
const typeOf = (i: number) => nodeTypes[nodes[i * NF + iType]!]!;
const nameOf = (i: number) => strings[nodes[i * NF + iName]!]!;
let total = 0;
const byType = new Map<string, number>();
const byCtor = new Map<string, [number, number]>();
const big: Array<[number, string]> = [];
let sourceStrings = 0;
for (let i = 0; i < N; i++) {
const s = self(i);
total += s;
const t = typeOf(i);
byType.set(t, (byType.get(t) ?? 0) + s);
if (t === "object" || t === "closure" || t === "array" || t === "native") {
const k = `${t}:${nameOf(i)}`.slice(0, 80);
const cur = byCtor.get(k) ?? [0, 0];
cur[0] += s;
cur[1] += 1;
byCtor.set(k, cur);
}
if (
t === "string" ||
t === "concatenated string" ||
t === "sliced string"
) {
if (s > 200_000)
big.push([s, nameOf(i).slice(0, 100).replace(/\s+/g, " ")]);
const nm = nameOf(i);
if (
s > 50_000 &&
/__turbopack_context__|"use strict";|module\.exports/.test(
nm.slice(0, 2000),
)
) {
sourceStrings += s;
}
}
}
// Graph for the dominator tree: strong edges only.
const isStrong = (et: number) => et !== weak && et !== shortcut;
// Post-order DFS from node 0.
const order = new Int32Array(N).fill(-1);
const post: number[] = [];
const stackN: number[] = [0];
const stackE: number[] = [firstEdge[0]!];
order[0] = -2;
while (stackN.length) {
const top = stackN.length - 1;
const n = stackN[top]!;
const e = stackE[top]!;
if (e < firstEdge[n + 1]!) {
stackE[top] = e + EF;
const et = edges[e + eType]!;
if (!isStrong(et)) continue;
const to = edges[e + eTo]! / NF;
if (order[to] === -1) {
order[to] = -2;
stackN.push(to);
stackE.push(firstEdge[to]!);
}
} else {
order[n] = post.length;
post.push(n);
stackN.pop();
stackE.pop();
}
}
const R = post.length;
// Predecessors in post-order numbering (CSR).
const predCount = new Uint32Array(R + 1);
for (let n = 0; n < N; n++) {
if (order[n]! < 0) continue;
for (let e = firstEdge[n]!; e < firstEdge[n + 1]!; e += EF) {
if (!isStrong(edges[e + eType]!)) continue;
const to = edges[e + eTo]! / NF;
if (order[to]! >= 0) predCount[order[to]! + 1]! += 1;
}
}
for (let i = 0; i < R; i++) predCount[i + 1]! += predCount[i]!;
const preds = new Uint32Array(predCount[R]!);
const fill = predCount.slice(0, R);
for (let n = 0; n < N; n++) {
if (order[n]! < 0) continue;
for (let e = firstEdge[n]!; e < firstEdge[n + 1]!; e += EF) {
if (!isStrong(edges[e + eType]!)) continue;
const to = edges[e + eTo]! / NF;
if (order[to]! >= 0) preds[fill[order[to]!]!++] = order[n]!;
}
}
const rootPo = order[0]!;
const idom = new Int32Array(R).fill(-1);
idom[rootPo] = rootPo;
let changed = true;
while (changed) {
changed = false;
for (let b = R - 1; b >= 0; b--) {
if (b === rootPo) continue;
let ni = -1;
for (let p = predCount[b]!; p < predCount[b + 1]!; p++) {
let q = preds[p]!;
if (idom[q] === -1) continue;
if (ni === -1) {
ni = q;
continue;
}
let a = ni;
while (a !== q) {
while (a < q) a = idom[a]!;
while (q < a) q = idom[q]!;
}
ni = a;
}
if (ni !== -1 && idom[b] !== ni) {
idom[b] = ni;
changed = true;
}
}
}
const retained = new Float64Array(R);
for (let b = 0; b < R; b++) retained[b] += self(post[b]!);
for (let b = 0; b < R; b++) {
if (b !== rootPo && idom[b]! >= 0) retained[idom[b]!] += retained[b]!;
}
// A retaining path: the first strong, named predecessor edge, walked up.
const retainerEdge = new Map<number, [number, string]>();
const wanted = new Set<number>();
const candidates: number[] = [];
for (let b = 0; b < R; b++) {
const n = post[b]!;
const t = typeOf(n);
if (b === rootPo || t === "synthetic" || t === "hidden") continue;
// Skip nodes whose immediate dominator is itself a big application
// object: report the outermost holders only.
if (retained[b]! < 2_000_000) continue;
candidates.push(b);
}
candidates.sort((a, b) => retained[b]! - retained[a]!);
const picked: number[] = [];
for (const b of candidates) {
// keep the outermost: skip if an ancestor in the dominator tree is
// already picked and within 1.5× (it would be a duplicate line)
let a = idom[b]!;
let dup = false;
while (a !== rootPo && a >= 0) {
if (picked.includes(a) && retained[a]! < retained[b]! * 1.5) {
dup = true;
break;
}
a = idom[a]!;
}
if (!dup) picked.push(b);
if (picked.length >= 40) break;
}
for (const b of picked) wanted.add(post[b]!);
// Build one retainer per node in a BFS from the root (shortest path).
const dist = new Int32Array(N).fill(-1);
const queue = new Int32Array(N);
let qh = 0;
let qt = 0;
queue[qt++] = 0;
dist[0] = 0;
while (qh < qt) {
const n = queue[qh++]!;
for (let e = firstEdge[n]!; e < firstEdge[n + 1]!; e += EF) {
const et = edges[e + eType]!;
if (!isStrong(et)) continue;
const to = edges[e + eTo]! / NF;
if (dist[to] !== -1) continue;
dist[to] = dist[n]! + 1;
const label =
et === elementT || et === hiddenT
? `[${edges[e + eName]}]`
: String(strings[edges[e + eName]!] ?? "");
retainerEdge.set(to, [n, label]);
queue[qt++] = to;
}
}
const pathOf = (n: number) => {
const parts: string[] = [];
let cur = n;
for (let depth = 0; depth < 12 && cur !== 0; depth++) {
const r = retainerEdge.get(cur);
if (!r) break;
parts.unshift(`${r[1]}`);
cur = r[0];
const t = typeOf(cur);
if (t !== "hidden" && t !== "synthetic" && depth > 0) {
parts.unshift(`${nameOf(cur).slice(0, 40)}`);
if (depth > 4) break;
}
}
return parts.join(" > ");
};
const mb = (n: number) => Math.round((n / 1048576) * 10) / 10;
const probeResults: HeapSummary["probes"] = {};
const propertyT = edgeTypes.indexOf("property");
for (const [label, props] of Object.entries(probes)) {
const hits = new Set<number>();
for (let n = 0; n < N; n++) {
if (typeOf(n) !== "object" || order[n]! < 0) continue;
const seen = new Set<string>();
for (let e = firstEdge[n]!; e < firstEdge[n + 1]!; e += EF) {
if (edges[e + eType] !== propertyT) continue;
seen.add(strings[edges[e + eName]!]!);
}
if (props.every((p) => seen.has(p))) hits.add(order[n]!);
}
let retainedSum = 0;
for (const b of hits) {
// Count a hit only when no dominator above it is a hit too.
let a = idom[b]!;
let nested = false;
while (a !== rootPo && a >= 0) {
if (hits.has(a)) {
nested = true;
break;
}
a = idom[a]!;
}
if (!nested) retainedSum += retained[b]!;
}
probeResults[label] = { count: hits.size, retainedMb: mb(retainedSum) };
}
return {
probes: probeResults,
totalMb: mb(total),
nodeCount: N,
byType: [...byType.entries()]
.map(([k, v]) => [k, mb(v)] as [string, number])
.sort((a, b) => b[1] - a[1]),
byConstructor: [...byCtor.entries()]
.map(([k, [s, c]]) => [k, mb(s), c] as [string, number, number])
.sort((a, b) => b[1] - a[1])
.slice(0, 30),
largestStrings: big
.sort((a, b) => b[0] - a[0])
.slice(0, 25)
.map(([s, t]) => [mb(s), t]),
topRetainers: picked.map((b) => ({
mb: mb(retained[b]!),
name: `${typeOf(post[b]!)}:${nameOf(post[b]!).slice(0, 60).replace(/\s+/g, " ")}`,
path: pathOf(post[b]!),
})),
sourceStringsMb: mb(sourceStrings),
};
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,221 @@
# Boot profile v1.45.0-before-boot
Commit 0f6d668df, 2026-10-22T04:36:55.826Z. Production build, every role, heap ceiling 792 MB, 300 s idle. 14 cores, load average 9.3,9.4,11.2 at the end.
- first answer from /api/health: 1590 ms; background role ready: 5432 ms
- event-loop stalls logged: 0, max 0 ms
## Stalls
## Heap samples
| at s | used MB | old | large obj | code | external | array buffers | rss |
| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| 6 | 184 | 105 | 68 | 4 | 11 | 0 | 267 |
| 308 | 164 | 99 | 58 | 5 | 11 | 0 | 623 |
## Full collections (mark-compact)
47 mark-compacts during the run. Used heap before the collection, and what survived it (the live heap):
| at s | before MB | after MB |
| ---: | ---: | ---: |
| 0 | 21.4 | 17.4 |
| 9 | 189.6 | 172.9 |
| 154 | 193.8 | 171.6 |
| 170 | 187.3 | 161.7 |
| 188 | 181.6 | 163.6 |
| 205 | 182.3 | 163.6 |
| 222 | 190.6 | 165 |
| 239 | 182.9 | 163.4 |
| 258 | 190.5 | 163.7 |
| 274 | 187.7 | 164.2 |
| 290 | 193.9 | 164.2 |
| 306 | 175.3 | 163.9 |
| 309 | 174 | 163.7 |
## Main-thread busy stretches over 50 ms without an idle sample, first two minutes
### at 5 ms, 279 ms
| self time by module | ms |
| --- | ---: |
| (native) | 122.3 |
| npm:next | 57.1 |
| node:internal/modules/cjs/loader | 41.7 |
| node:internal/bootstrap/realm | 24.7 |
| node:fs | 5.3 |
| node:internal/deps/undici/undici | 5.3 |
| node:internal/modules/package_json_reader | 4.5 |
| node:path | 4 |
| node:internal/streams/destroy | 1.5 |
| node:internal/deps/cjs-module-lexer/lexer | 1.5 |
| inclusive, application modules | ms |
| --- | ---: |
### at 339 ms, 324 ms
| self time by module | ms |
| --- | ---: |
| (native) | 87.6 |
| node:internal/modules/cjs/loader | 54.2 |
| npm:@prisma/client | 38 |
| wasm://wasm/00d1cb12 | 36 |
| node:internal/modules/esm/utils | 26.7 |
| turbopack:///[turbopack]/nodejs/runtime/build-base.ts | 24.9 |
| npm:next | 22.1 |
| npm:zod | 6 |
| turbopack:///[turbopack]/shared/runtime/runtime-utils.ts | 3.1 |
| npm:@prisma/client-runtime-utils | 3 |
| inclusive, application modules | ms |
| --- | ---: |
| src/instrumentation.ts | 80.6 |
| src/lib/db-session.ts | 60.7 |
| src/lib/notifications/types.ts | 13.2 |
| src/lib/boot/readiness-summary.ts | 13.2 |
| src/lib/ai/local-host-allowlist.ts | 13.2 |
| src/generated/prisma/internal/class.ts | 7.3 |
| src/lib/logging/context.ts | 6 |
| src/lib/crypto.ts | 6 |
| src/lib/boot/key-check.ts | 6 |
| src/lib/validations/measurement.ts | 3 |
### at 698 ms, 753 ms
| self time by module | ms |
| --- | ---: |
| node:internal/modules/cjs/loader | 178.2 |
| turbopack:///[turbopack]/nodejs/runtime/build-base.ts | 177.4 |
| (native) | 83.8 |
| npm:zod | 39.5 |
| npm:next | 34.6 |
| turbopack:///[turbopack]/shared/runtime/async-module.ts | 33.9 |
| npm:mmdb-lib | 26.1 |
| node:fs | 23.5 |
| npm:yaml | 22.4 |
| src/lib/activity/goal-bounds.ts | 18.5 |
| inclusive, application modules | ms |
| --- | ---: |
| src/instrumentation.ts | 410 |
| src/lib/jobs/reminder/register-maintenance.ts | 279.5 |
| src/lib/sharing/acting-session.ts | 138.5 |
| src/lib/jobs/offhost-backup.ts | 137.1 |
| src/lib/integrations/oauth-refresh.ts | 136.3 |
| src/lib/jobs/morning-digest-refresh-shared.ts | 136.3 |
| src/lib/export/environment-backup.ts | 134.1 |
| src/lib/export/full-backup-payload.ts | 134.1 |
| src/app/api/health/route.ts | 127.5 |
| src/lib/insights/snapshot-hash.ts | 124.9 |
## Self time by module until the worker is ready
| module | ms |
| --- | ---: |
| (native) | 524.6 |
| node:internal/modules/cjs/loader | 274 |
| turbopack:///[turbopack]/nodejs/runtime/build-base.ts | 202.3 |
| npm:next | 136.8 |
| wasm://wasm/00d1cb12 | 108.1 |
| npm:@prisma/client | 76.1 |
| npm:pg | 56.5 |
| npm:zod | 45.5 |
| node:internal/modules/esm/utils | 41.4 |
| npm:luxon | 38.5 |
| turbopack:///[turbopack]/shared/runtime/async-module.ts | 35.3 |
| node:fs | 31.7 |
| node:internal/bootstrap/realm | 28.9 |
| npm:pg-protocol | 27.7 |
| npm:mmdb-lib | 26.1 |
| npm:yaml | 22.4 |
| npm:pg-boss | 19.3 |
| src/lib/activity/goal-bounds.ts | 18.5 |
| node:internal/process/task_queues | 17.7 |
| node:internal/streams/readable | 15.5 |
| node:internal/async_hooks | 15.1 |
| npm:cron-parser | 14.6 |
| node:internal/timers | 12.8 |
| turbopack:///[turbopack]/shared/runtime/runtime-utils.ts | 12.4 |
| node:buffer | 11.6 |
## Heap snapshot after 300 s idle (live heap after a full GC)
172.3 MB in 1412114 nodes; module source strings over 50 KB: 44.2 MB.
- parsed locale catalogs: 32 objects, 14.1 MB retained
| node type | MB |
| --- | ---: |
| string | 79.1 |
| array | 27.6 |
| code | 18.6 |
| object | 16.7 |
| hidden | 9.3 |
| native | 8 |
| object shape | 5.8 |
| closure | 5 |
| concatenated string | 1.3 |
| number | 0.4 |
| regexp | 0.2 |
| sliced string | 0.2 |
| synthetic | 0 |
| symbol | 0 |
| bigint | 0 |
| constructor | MB | count |
| --- | ---: | ---: |
| array:(object elements) | 20.6 | 24726 |
| object:Object | 11.9 | 300844 |
| native:system / JSArrayBufferData | 7.5 | 169 |
| array:(object properties) | 4.4 | 6540 |
| array: | 2.7 | 9578 |
| object:system / Context | 1.6 | 28887 |
| closure: | 1.5 | 27081 |
| object:Date1 | 1 | 10632 |
| closure:native_bind | 0.9 | 20134 |
| object:Array | 0.6 | 19550 |
| object:a | 0.4 | 3880 |
| native:Node / histogram | 0.4 | 1 |
| closure:get | 0.3 | 4717 |
| object:Set | 0.2 | 5102 |
| closure:set | 0.2 | 3414 |
| object:d | 0.2 | 3874 |
| object:Module | 0.1 | 1907 |
| object:Promise | 0.1 | 1590 |
| closure:n | 0.1 | 1400 |
| closure:i | 0.1 | 1390 |
| largest retained subtrees | MB | retaining path |
| --- | ---: | --- |
| object:yr | 33.9 | [1] > [10] > 15 > extension > global > prisma > [1] > _engine |
| object:Object | 10 | tX > onClose > onClose > context > system / Context > previous > system / Context > e > Context > __proto__ > c |
| code:.next/server/chu | 9.8 | system / Context > timerListMap > Object > [120000] > TimersList > _idleNext > Timeout > _onTimeout > > shared > script |
| code:.next/server/chu | 9.7 | Module > _cache > Object > .next/server/chunks/_09yxiz_._.js > Module > exports > Array > [1] > > shared > script |
| object:system / Context | 7.4 | Node / PrincipalRealm > base_object_list > Node / BaseObjectList > [25] > Node / ModuleWrap > native_to_javascript > ModuleWrap > 2 > [4] > context |
| code:.next/server/chu | 5.6 | global > prisma > [1] > t > _engineConfig > Object > compilerWasm > Object > getRuntime > getRuntime > shared > script |
| code:.next/server/chu | 5.6 | Module > _cache > Object > .next/server/chunks/_0hfea7u._.js > Module > exports > Array > [1] > > shared > script |
| object:Generator | 4.4 | [6] > principal_realm > Node / PrincipalRealm > base_object_list > Node / BaseObjectList > [26] > Node / ModuleWrap > native_to_javascript > ModuleWrap > 2 > [4 |
| string:AGFzbQEAAAABtgRLYAN/f38Bf2ACf38Bf2ACf38AYAF/AGADf39/AGAEf39/ | 4.4 | [1] > [21] > 59867 |
| code:.next/server/chu | 3.7 | [1] > [10] > 15 > extension > global > __healthlog_pgboss__ > PgBoss > _events > Object > error > > shared > script |
| object:Client | 3.6 | Object > close > > context > system / Context > self > Connection > _events > Object > parseComplete > bound_this |
| largest strings | MB |
| --- | ---: |
| module.exports=[425781,(e,a,i)=>{a.exports=JSON.parse('{"common":{"listCapped":"Hier stehe | 9.7 |
| module.exports=[157893,(e,a,i)=>{a.exports=JSON.parse('{"common":{"listCapped":"Hier stehe | 9.7 |
| module.exports=[17856,A=>{"use strict";var e,C,g,I,Q=Object.defineProperty,B={},s={$:()=>i | 5.6 |
| module.exports=[234494,A=>{"use strict";var e,C,g,I,Q=Object.defineProperty,B={},s={$:()=> | 5.6 |
| const wasm = "AGFzbQEAAAABtgRLYAN/f38Bf2ACf38Bf2ACf38AYAF/AGADf39/AGAEf39/fwBgBX9/f39/AGAB | 4.4 |
| AGFzbQEAAAABtgRLYAN/f38Bf2ACf38Bf2ACf38AYAF/AGADf39/AGAEf39/fwBgBX9/f39/AGABfwF/YAZ/f39/f3 | 4.4 |
| module.exports=[799092,(e,t,n)=>{"use strict";Object.defineProperty(n,"__esModule",{value: | 3.7 |
| lliADOAUngIEAACgKwAgBgAAoSsAIAcAAKErACAIAACiKwAgCQAAoysAIAoAAKQrACALAAClKwAgDAAApisAIA0AAK | 1.9 |
| (()=>{var e,t,r={"./dist/build/webpack/alias/react-dom-server.js"(e,t,r){"use strict";var | 1.3 |
| module.exports=[842865,(e,t,r)=>{let i=e.r(522734);t.exports=function(e){return e?/-----BE | 1.2 |
| module.exports=[834886,(e,t,n)=>{"use strict";Object.defineProperty(n,"__esModule",{value: | 1.1 |
| // v1.4.34 web-freeze. Schema is locked additive-only until iOS launch // + v1.5.0 marker. | 1.1 |
| module.exports=[971743,(A,e,t)=>{"use strict";let r=A.r(57328),{addAbortListener:s}=A.r(68 | 1.1 |
| module.exports=[835717,(A,e,t)=>{"use strict";let r=A.r(57328),{addAbortListener:s}=A.r(68 | 1 |
| module.exports=[594108,806176,674750,98746,t=>{"use strict";var i;let r;function n(t){retu | 0.7 |