From 73f2004c54af428103e826bd77acd76e185562b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Tue, 20 Oct 2026 20:27:24 +0200 Subject: [PATCH] feat(observability): break the heap sample down by V8 space The heap sample named old_space alone, so a heap that stood well above it could not be explained from the log: a buffer past V8's large-object threshold lives in large_object_space, compiled code and bytecode in the code and trusted spaces. The sampled and pressure lines now carry the used size of every space, with any space the runtime reports that is not named summed into other_spaces_mb, so the fields account for the whole used heap whatever the Node version. --- docs/ops/heap-diagnostics.md | 2 +- .../__tests__/heap-monitor.test.ts | 54 ++++++++++++++++ src/lib/observability/heap-monitor.ts | 63 +++++++++++++++++-- 3 files changed, 112 insertions(+), 7 deletions(-) diff --git a/docs/ops/heap-diagnostics.md b/docs/ops/heap-diagnostics.md index 87b12052f..7652079a3 100644 --- a/docs/ops/heap-diagnostics.md +++ b/docs/ops/heap-diagnostics.md @@ -6,7 +6,7 @@ A Node process that reaches its heap limit stops with `JavaScript heap out of me Two background wide events, no configuration needed: -- `runtime.heap.sampled` (info), every five minutes, and every minute while the heap is at or above 70 % of its limit. It goes to stdout and Loki but not to Admin > App logs, so it cannot push other events out of that page's 500-entry buffer. It carries `heap_used_mb`, `heap_total_mb`, `heap_limit_mb`, `heap_used_ratio`, `old_space_mb`, `rss_mb`, `external_mb`, `array_buffers_mb`, `active_requests`, `active_jobs`, `in_flight_dropped` and `uptime_s`. +- `runtime.heap.sampled` (info), every five minutes, and every minute while the heap is at or above 70 % of its limit. It goes to stdout and Loki but not to Admin > App logs, so it cannot push other events out of that page's 500-entry buffer. It carries `heap_used_mb`, `heap_total_mb`, `heap_limit_mb`, `heap_used_ratio`, the used size of each heap space (`new_space_mb`, `old_space_mb`, `code_space_mb`, `large_object_space_mb`, `new_large_object_space_mb`, `code_large_object_space_mb`, `trusted_space_mb`, `read_only_space_mb`, and `other_spaces_mb` for any space not named), `rss_mb`, `external_mb`, `array_buffers_mb`, `active_requests`, `active_jobs`, `in_flight_dropped` and `uptime_s`. The spaces add up to `heap_used_mb`: a large buffer or string (a profile picture held in memory, a long response body) shows in `large_object_space_mb`, not in `old_space_mb`, and memory outside the V8 heap shows in `external_mb` and `array_buffers_mb`. - `runtime.heap.pressure` (warn), the first time the heap reaches 85 % of its limit. Besides the figures above it lists the 50 oldest requests and jobs in flight (`in_flight_oldest`: kind, method, route, age in seconds, whether it was a server action) and the routes most of them share (`in_flight_by_route`). Routes are reduced to their shape: no query string, secret path segments replaced by `[REDACTED]`, ids by `:id`, anything unusual by `:x`. It writes again only after the heap has dropped below 70 % and thirty minutes have passed. One exception: when the heap reaches 95 %, the same report is written again at once, with `pressure_ratio: 0.95`, whatever the cooldown; that tier re-arms as soon as the heap is back under 85 %. A heap that climbs to its limit shortly after an 85 % report therefore still leaves a report naming the work that pushed it over. The heap is read every five seconds, a single call that takes microseconds. Tracking requests costs one map insert and one delete per request. diff --git a/src/lib/observability/__tests__/heap-monitor.test.ts b/src/lib/observability/__tests__/heap-monitor.test.ts index 9e686a52d..9df0ae3bb 100644 --- a/src/lib/observability/__tests__/heap-monitor.test.ts +++ b/src/lib/observability/__tests__/heap-monitor.test.ts @@ -10,6 +10,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { getHeapSpaceStatistics } from "node:v8"; import { describe, expect, it, vi } from "vitest"; @@ -34,6 +35,7 @@ import { type HeapFigures, type HeapMonitorDeps, emitHeapLine, + heapSpaceFigures, isPrivateDirectory, writeDiagnosticHeapSnapshot, } from "../heap-monitor"; @@ -46,7 +48,15 @@ const FIGURE_KEYS: Array = [ "heap_total_mb", "heap_limit_mb", "heap_used_ratio", + "new_space_mb", "old_space_mb", + "code_space_mb", + "large_object_space_mb", + "new_large_object_space_mb", + "code_large_object_space_mb", + "trusted_space_mb", + "read_only_space_mb", + "other_spaces_mb", "rss_mb", "external_mb", "array_buffers_mb", @@ -125,6 +135,50 @@ describe("heap monitor sampling", () => { }); }); +describe("heap space breakdown", () => { + const MB = 1024 * 1024; + + it("reports each named space and sums the rest into other_spaces_mb", () => { + const figures = heapSpaceFigures([ + { space_name: "old_space", space_used_size: 300 * MB }, + { space_name: "large_object_space", space_used_size: 120 * MB }, + { space_name: "trusted_space", space_used_size: 40 * MB }, + { space_name: "shared_space", space_used_size: 2 * MB }, + { space_name: "a_space_v8_adds_later", space_used_size: 3 * MB }, + ]); + expect(figures).toEqual({ + new_space_mb: 0, + old_space_mb: 300, + code_space_mb: 0, + large_object_space_mb: 120, + new_large_object_space_mb: 0, + code_large_object_space_mb: 0, + trusted_space_mb: 40, + read_only_space_mb: 0, + other_spaces_mb: 5, + }); + }); + + it("accounts for every space this runtime reports", () => { + const spaces = getHeapSpaceStatistics(); + expect(spaces.length).toBeGreaterThan(0); + const figures = heapSpaceFigures(spaces); + expect(Object.keys(figures).sort()).toEqual( + FIGURE_KEYS.filter( + (k) => k.endsWith("space_mb") || k === "other_spaces_mb", + ) + .map(String) + .sort(), + ); + const total = spaces.reduce((sum, s) => sum + s.space_used_size, 0); + const reported = Object.values(figures).reduce((a, b) => a + b, 0); + // Each space is rounded to the megabyte on its own. + expect(Math.abs(reported - total / MB)).toBeLessThanOrEqual( + Object.keys(figures).length, + ); + }); +}); + describe("heap pressure report", () => { it("fires once at 85 % with the work in flight, at warn", () => { const h = harness(); diff --git a/src/lib/observability/heap-monitor.ts b/src/lib/observability/heap-monitor.ts index 362a9f7ae..5e049763d 100644 --- a/src/lib/observability/heap-monitor.ts +++ b/src/lib/observability/heap-monitor.ts @@ -72,12 +72,66 @@ export const HEAP_CRITICAL_RATIO = 0.95; export const HEAP_PRESSURE_COOLDOWN_MS = 30 * 60_000; export const HEAP_PRESSURE_LIST_MAX = 50; -export interface HeapFigures { +/** + * The used size of each V8 heap space, by the field it is reported under. + * + * `old_space` alone left most of a large heap unexplained: a buffer or string + * past V8's large-object threshold (a profile picture read into memory, a + * long JSON payload) lives in `large_object_space`, compiled code in the code + * spaces, and since V8 12 bytecode and other sandboxed objects in + * `trusted_space`. Every space a runtime reports that is not named here + * (the shared spaces, a space a later V8 adds) is summed into + * `other_spaces_mb`, so the fields account for the whole used heap and the + * line keeps one shape whatever the Node version. + */ +const HEAP_SPACE_FIELDS = { + new_space: "new_space_mb", + old_space: "old_space_mb", + code_space: "code_space_mb", + large_object_space: "large_object_space_mb", + new_large_object_space: "new_large_object_space_mb", + code_large_object_space: "code_large_object_space_mb", + trusted_space: "trusted_space_mb", + read_only_space: "read_only_space_mb", +} as const; + +type HeapSpaceField = + | (typeof HEAP_SPACE_FIELDS)[keyof typeof HEAP_SPACE_FIELDS] + | "other_spaces_mb"; + +export type HeapSpaceFigures = Record; + +/** The per-space breakdown of `getHeapSpaceStatistics()`, in megabytes. */ +export function heapSpaceFigures( + spaces: ReadonlyArray<{ space_name: string; space_used_size: number }>, +): HeapSpaceFigures { + const bytes: Record = { + new_space_mb: 0, + old_space_mb: 0, + code_space_mb: 0, + large_object_space_mb: 0, + new_large_object_space_mb: 0, + code_large_object_space_mb: 0, + trusted_space_mb: 0, + read_only_space_mb: 0, + other_spaces_mb: 0, + }; + for (const space of spaces) { + const field = + HEAP_SPACE_FIELDS[space.space_name as keyof typeof HEAP_SPACE_FIELDS] ?? + "other_spaces_mb"; + bytes[field] += space.space_used_size; + } + return Object.fromEntries( + Object.entries(bytes).map(([field, size]) => [field, megabytes(size)]), + ) as HeapSpaceFigures; +} + +export interface HeapFigures extends HeapSpaceFigures { heap_used_mb: number; heap_total_mb: number; heap_limit_mb: number; heap_used_ratio: number; - old_space_mb: number; rss_mb: number; external_mb: number; array_buffers_mb: number; @@ -115,9 +169,6 @@ function defaultReadHeap(): { used: number; limit: number } { function defaultReadFigures(registry: InFlightRegistry): HeapFigures { const memory = process.memoryUsage(); const limit = getHeapStatistics().heap_size_limit; - const oldSpace = getHeapSpaceStatistics().find( - (space) => space.space_name === "old_space", - ); const counts = registry.counts(); return { heap_used_mb: megabytes(memory.heapUsed), @@ -125,7 +176,7 @@ function defaultReadFigures(registry: InFlightRegistry): HeapFigures { heap_limit_mb: megabytes(limit), heap_used_ratio: limit > 0 ? Math.round((memory.heapUsed / limit) * 1000) / 1000 : 0, - old_space_mb: megabytes(oldSpace?.space_used_size ?? 0), + ...heapSpaceFigures(getHeapSpaceStatistics()), rss_mb: megabytes(memory.rss), external_mb: megabytes(memory.external), array_buffers_mb: megabytes(memory.arrayBuffers),