diff --git a/docs/ops/heap-diagnostics.md b/docs/ops/heap-diagnostics.md index 87b12052f..7652079a3 100644 --- a/docs/ops/heap-diagnostics.md +++ b/docs/ops/heap-diagnostics.md @@ -6,7 +6,7 @@ A Node process that reaches its heap limit stops with `JavaScript heap out of me Two background wide events, no configuration needed: -- `runtime.heap.sampled` (info), every five minutes, and every minute while the heap is at or above 70 % of its limit. It goes to stdout and Loki but not to Admin > App logs, so it cannot push other events out of that page's 500-entry buffer. It carries `heap_used_mb`, `heap_total_mb`, `heap_limit_mb`, `heap_used_ratio`, `old_space_mb`, `rss_mb`, `external_mb`, `array_buffers_mb`, `active_requests`, `active_jobs`, `in_flight_dropped` and `uptime_s`. +- `runtime.heap.sampled` (info), every five minutes, and every minute while the heap is at or above 70 % of its limit. It goes to stdout and Loki but not to Admin > App logs, so it cannot push other events out of that page's 500-entry buffer. It carries `heap_used_mb`, `heap_total_mb`, `heap_limit_mb`, `heap_used_ratio`, the used size of each heap space (`new_space_mb`, `old_space_mb`, `code_space_mb`, `large_object_space_mb`, `new_large_object_space_mb`, `code_large_object_space_mb`, `trusted_space_mb`, `read_only_space_mb`, and `other_spaces_mb` for any space not named), `rss_mb`, `external_mb`, `array_buffers_mb`, `active_requests`, `active_jobs`, `in_flight_dropped` and `uptime_s`. The spaces add up to `heap_used_mb`: a large buffer or string (a profile picture held in memory, a long response body) shows in `large_object_space_mb`, not in `old_space_mb`, and memory outside the V8 heap shows in `external_mb` and `array_buffers_mb`. - `runtime.heap.pressure` (warn), the first time the heap reaches 85 % of its limit. Besides the figures above it lists the 50 oldest requests and jobs in flight (`in_flight_oldest`: kind, method, route, age in seconds, whether it was a server action) and the routes most of them share (`in_flight_by_route`). Routes are reduced to their shape: no query string, secret path segments replaced by `[REDACTED]`, ids by `:id`, anything unusual by `:x`. It writes again only after the heap has dropped below 70 % and thirty minutes have passed. One exception: when the heap reaches 95 %, the same report is written again at once, with `pressure_ratio: 0.95`, whatever the cooldown; that tier re-arms as soon as the heap is back under 85 %. A heap that climbs to its limit shortly after an 85 % report therefore still leaves a report naming the work that pushed it over. The heap is read every five seconds, a single call that takes microseconds. Tracking requests costs one map insert and one delete per request. diff --git a/src/lib/observability/__tests__/heap-monitor.test.ts b/src/lib/observability/__tests__/heap-monitor.test.ts index 9e686a52d..9df0ae3bb 100644 --- a/src/lib/observability/__tests__/heap-monitor.test.ts +++ b/src/lib/observability/__tests__/heap-monitor.test.ts @@ -10,6 +10,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { getHeapSpaceStatistics } from "node:v8"; import { describe, expect, it, vi } from "vitest"; @@ -34,6 +35,7 @@ import { type HeapFigures, type HeapMonitorDeps, emitHeapLine, + heapSpaceFigures, isPrivateDirectory, writeDiagnosticHeapSnapshot, } from "../heap-monitor"; @@ -46,7 +48,15 @@ const FIGURE_KEYS: Array = [ "heap_total_mb", "heap_limit_mb", "heap_used_ratio", + "new_space_mb", "old_space_mb", + "code_space_mb", + "large_object_space_mb", + "new_large_object_space_mb", + "code_large_object_space_mb", + "trusted_space_mb", + "read_only_space_mb", + "other_spaces_mb", "rss_mb", "external_mb", "array_buffers_mb", @@ -125,6 +135,50 @@ describe("heap monitor sampling", () => { }); }); +describe("heap space breakdown", () => { + const MB = 1024 * 1024; + + it("reports each named space and sums the rest into other_spaces_mb", () => { + const figures = heapSpaceFigures([ + { space_name: "old_space", space_used_size: 300 * MB }, + { space_name: "large_object_space", space_used_size: 120 * MB }, + { space_name: "trusted_space", space_used_size: 40 * MB }, + { space_name: "shared_space", space_used_size: 2 * MB }, + { space_name: "a_space_v8_adds_later", space_used_size: 3 * MB }, + ]); + expect(figures).toEqual({ + new_space_mb: 0, + old_space_mb: 300, + code_space_mb: 0, + large_object_space_mb: 120, + new_large_object_space_mb: 0, + code_large_object_space_mb: 0, + trusted_space_mb: 40, + read_only_space_mb: 0, + other_spaces_mb: 5, + }); + }); + + it("accounts for every space this runtime reports", () => { + const spaces = getHeapSpaceStatistics(); + expect(spaces.length).toBeGreaterThan(0); + const figures = heapSpaceFigures(spaces); + expect(Object.keys(figures).sort()).toEqual( + FIGURE_KEYS.filter( + (k) => k.endsWith("space_mb") || k === "other_spaces_mb", + ) + .map(String) + .sort(), + ); + const total = spaces.reduce((sum, s) => sum + s.space_used_size, 0); + const reported = Object.values(figures).reduce((a, b) => a + b, 0); + // Each space is rounded to the megabyte on its own. + expect(Math.abs(reported - total / MB)).toBeLessThanOrEqual( + Object.keys(figures).length, + ); + }); +}); + describe("heap pressure report", () => { it("fires once at 85 % with the work in flight, at warn", () => { const h = harness(); diff --git a/src/lib/observability/heap-monitor.ts b/src/lib/observability/heap-monitor.ts index 362a9f7ae..5e049763d 100644 --- a/src/lib/observability/heap-monitor.ts +++ b/src/lib/observability/heap-monitor.ts @@ -72,12 +72,66 @@ export const HEAP_CRITICAL_RATIO = 0.95; export const HEAP_PRESSURE_COOLDOWN_MS = 30 * 60_000; export const HEAP_PRESSURE_LIST_MAX = 50; -export interface HeapFigures { +/** + * The used size of each V8 heap space, by the field it is reported under. + * + * `old_space` alone left most of a large heap unexplained: a buffer or string + * past V8's large-object threshold (a profile picture read into memory, a + * long JSON payload) lives in `large_object_space`, compiled code in the code + * spaces, and since V8 12 bytecode and other sandboxed objects in + * `trusted_space`. Every space a runtime reports that is not named here + * (the shared spaces, a space a later V8 adds) is summed into + * `other_spaces_mb`, so the fields account for the whole used heap and the + * line keeps one shape whatever the Node version. + */ +const HEAP_SPACE_FIELDS = { + new_space: "new_space_mb", + old_space: "old_space_mb", + code_space: "code_space_mb", + large_object_space: "large_object_space_mb", + new_large_object_space: "new_large_object_space_mb", + code_large_object_space: "code_large_object_space_mb", + trusted_space: "trusted_space_mb", + read_only_space: "read_only_space_mb", +} as const; + +type HeapSpaceField = + | (typeof HEAP_SPACE_FIELDS)[keyof typeof HEAP_SPACE_FIELDS] + | "other_spaces_mb"; + +export type HeapSpaceFigures = Record; + +/** The per-space breakdown of `getHeapSpaceStatistics()`, in megabytes. */ +export function heapSpaceFigures( + spaces: ReadonlyArray<{ space_name: string; space_used_size: number }>, +): HeapSpaceFigures { + const bytes: Record = { + new_space_mb: 0, + old_space_mb: 0, + code_space_mb: 0, + large_object_space_mb: 0, + new_large_object_space_mb: 0, + code_large_object_space_mb: 0, + trusted_space_mb: 0, + read_only_space_mb: 0, + other_spaces_mb: 0, + }; + for (const space of spaces) { + const field = + HEAP_SPACE_FIELDS[space.space_name as keyof typeof HEAP_SPACE_FIELDS] ?? + "other_spaces_mb"; + bytes[field] += space.space_used_size; + } + return Object.fromEntries( + Object.entries(bytes).map(([field, size]) => [field, megabytes(size)]), + ) as HeapSpaceFigures; +} + +export interface HeapFigures extends HeapSpaceFigures { heap_used_mb: number; heap_total_mb: number; heap_limit_mb: number; heap_used_ratio: number; - old_space_mb: number; rss_mb: number; external_mb: number; array_buffers_mb: number; @@ -115,9 +169,6 @@ function defaultReadHeap(): { used: number; limit: number } { function defaultReadFigures(registry: InFlightRegistry): HeapFigures { const memory = process.memoryUsage(); const limit = getHeapStatistics().heap_size_limit; - const oldSpace = getHeapSpaceStatistics().find( - (space) => space.space_name === "old_space", - ); const counts = registry.counts(); return { heap_used_mb: megabytes(memory.heapUsed), @@ -125,7 +176,7 @@ function defaultReadFigures(registry: InFlightRegistry): HeapFigures { heap_limit_mb: megabytes(limit), heap_used_ratio: limit > 0 ? Math.round((memory.heapUsed / limit) * 1000) / 1000 : 0, - old_space_mb: megabytes(oldSpace?.space_used_size ?? 0), + ...heapSpaceFigures(getHeapSpaceStatistics()), rss_mb: megabytes(memory.rss), external_mb: megabytes(memory.external), array_buffers_mb: megabytes(memory.arrayBuffers),