fix(heap): report again at 95 % and leave a report on an out-of-memory exit

The pressure report re-armed only after the heap fell under 70 % and
half an hour passed, so a heap that climbed to its limit soon after an
85 % report died with only that older report on record. A 95 % tier
now writes the same report at once, whatever the cooldown, and re-arms
under 85 %.

The image sets NODE_OPTIONS so Node writes its diagnostic report to
stderr when the heap runs out, with environment variables and network
interfaces excluded. The heap-diagnostics and scaling docs say how to
keep the flags when overriding NODE_OPTIONS.

Heap figures share one megabytes helper with the support report.

src/lib/observability/__tests__/heap-monitor.test.ts

@@ -23,7 +23,9 @@ import { walkSourceFiles } from "@/__tests__/helpers/source-files";
 
 import {
   HEAP_CHECK_INTERVAL_MS,
+  HEAP_CRITICAL_RATIO,
   HEAP_PRESSURE_COOLDOWN_MS,
+  HEAP_PRESSURE_RATIO,
   HEAP_PRESSURE_LIST_MAX,
   HEAP_SAMPLE_INTERVAL_ELEVATED_MS,
   HEAP_SAMPLE_INTERVAL_MS,
@@ -175,6 +177,41 @@ describe("heap pressure report", () => {
     expect(h.named(HEAP_TASK_NAMES.pressure)).toHaveLength(2);
   });
 
+  it("reports again at 95 % inside the cooldown, once per climb", () => {
+    const h = harness();
+    h.state.used = 900;
+    h.run(60_000);
+    // Ten minutes later, well inside the cooldown, the heap nears its limit.
+    h.state.used = 800;
+    h.run(10 * 60_000);
+    h.state.used = 960;
+    h.run(60_000);
+    const pressure = h.named(HEAP_TASK_NAMES.pressure);
+    expect(pressure.map((p) => p.meta.pressure_ratio)).toEqual([
+      HEAP_PRESSURE_RATIO,
+      HEAP_CRITICAL_RATIO,
+    ]);
+    expect(pressure[1].meta).toHaveProperty("in_flight_oldest");
+
+    // Hovering at 95 % writes nothing more; back under 85 % re-arms it.
+    h.run(5 * 60_000);
+    expect(h.named(HEAP_TASK_NAMES.pressure)).toHaveLength(2);
+    h.state.used = 840;
+    h.run(HEAP_CHECK_INTERVAL_MS);
+    h.state.used = 970;
+    h.run(HEAP_CHECK_INTERVAL_MS);
+    expect(h.named(HEAP_TASK_NAMES.pressure)).toHaveLength(3);
+  });
+
+  it("writes one report at the tier a sudden jump reaches", () => {
+    const h = harness();
+    h.state.used = 990;
+    h.run(60_000);
+    const pressure = h.named(HEAP_TASK_NAMES.pressure);
+    expect(pressure).toHaveLength(1);
+    expect(pressure[0].meta.pressure_ratio).toBe(HEAP_CRITICAL_RATIO);
+  });
+
   it("writes no snapshot unless opted in, and at most one when it is", () => {
     const off = harness();
     off.state.used = 900;

src/lib/observability/heap-monitor.ts

@@ -9,6 +9,7 @@ import {
 import { envFlag, envOr } from "@/lib/env";
 import { WideEventBuilder } from "@/lib/logging/event-builder";
 import { emitIfSampled } from "@/lib/logging/transports";
+import { megabytes } from "@/lib/support/heap-limit";
 
 import { inFlightRegistry, type InFlightRegistry } from "./in-flight";
 
@@ -32,7 +33,11 @@ import { inFlightRegistry, type InFlightRegistry } from "./in-flight";
  *     flight and the routes most of them share (see `in-flight.ts`). It
  *     re-arms only after the heap has fallen back under 70 % AND thirty minutes
  *     have passed, so a heap hovering at the line writes one report, not one
- *     every check.
+ *     every check. A second tier at 95 % writes the same report with no
+ *     cooldown at all, and re-arms as soon as the heap is back under 85 %:
+ *     a heap that climbs to its limit within the half hour after an 85 %
+ *     report would otherwise die with only that older report on record, the
+ *     requests that pushed it over missing from it.
  *   - `runtime.heap.snapshot`, at `warn`, only with the opt-in below: where
  *     the snapshot was written, or that writing it failed.
  *
@@ -63,6 +68,7 @@ export const HEAP_SAMPLE_INTERVAL_MS = 5 * 60_000;
 export const HEAP_SAMPLE_INTERVAL_ELEVATED_MS = 60_000;
 export const HEAP_ELEVATED_RATIO = 0.7;
 export const HEAP_PRESSURE_RATIO = 0.85;
+export const HEAP_CRITICAL_RATIO = 0.95;
 export const HEAP_PRESSURE_COOLDOWN_MS = 30 * 60_000;
 export const HEAP_PRESSURE_LIST_MAX = 50;
 
@@ -101,10 +107,6 @@ export interface HeapMonitorDeps {
   writeSnapshot: () => SnapshotOutcome;
 }
 
-function megabytes(bytes: number): number {
-  return Math.round(bytes / 1024 / 1024);
-}
-
 function defaultReadHeap(): { used: number; limit: number } {
   const stats = getHeapStatistics();
   return { used: stats.used_heap_size, limit: stats.heap_size_limit };
@@ -205,6 +207,7 @@ export function writeDiagnosticHeapSnapshot(): SnapshotOutcome {
 export class HeapMonitor {
   private lastSampleAt: number | null = null;
   private pressureArmed = true;
+  private criticalArmed = true;
   private lastPressureAt: number | null = null;
   private snapshotTaken = false;
 
@@ -225,11 +228,29 @@ export class HeapMonitor {
       this.pressureArmed = true;
     }
 
+    if (!this.criticalArmed && ratio < HEAP_PRESSURE_RATIO) {
+      this.criticalArmed = true;
+    }
+
     if (this.pressureArmed && ratio >= HEAP_PRESSURE_RATIO) {
       this.pressureArmed = false;
       this.lastPressureAt = at;
       this.lastSampleAt = at;
-      this.reportPressure(at);
+      // A heap that jumps straight past 95 % gets one report, at the tier
+      // it reached.
+      const critical = ratio >= HEAP_CRITICAL_RATIO;
+      if (critical) this.criticalArmed = false;
+      this.reportPressure(
+        at,
+        critical ? HEAP_CRITICAL_RATIO : HEAP_PRESSURE_RATIO,
+      );
+      return;
+    }
+
+    if (this.criticalArmed && ratio >= HEAP_CRITICAL_RATIO) {
+      this.criticalArmed = false;
+      this.lastSampleAt = at;
+      this.reportPressure(at, HEAP_CRITICAL_RATIO);
       return;
     }
 
@@ -245,7 +266,7 @@ export class HeapMonitor {
     }
   }
 
-  private reportPressure(at: number): void {
+  private reportPressure(at: number, tier: number): void {
     const { registry, emit, readFigures } = this.deps;
     const inFlight = registry.report(at, HEAP_PRESSURE_LIST_MAX);
     const willSnapshot = this.deps.snapshotEnabled && !this.snapshotTaken;
@@ -253,7 +274,7 @@ export class HeapMonitor {
     // process over, the report is already on stdout.
     emit(HEAP_TASK_NAMES.pressure, "warn", {
       ...readFigures(),
-      pressure_ratio: HEAP_PRESSURE_RATIO,
+      pressure_ratio: tier,
       in_flight_oldest: inFlight.oldest,
       in_flight_by_route: inFlight.by_route,
       snapshot: willSnapshot ? "pending" : "disabled",

src/lib/support/heap-limit.ts

@@ -14,9 +14,14 @@ import { getHeapStatistics } from "node:v8";
 /** Below this, the heap is flagged. Between the 1g (524) and 1536m (792) heaps. */
 export const LOW_HEAP_LIMIT_MB = 700;
 
+/** Bytes as whole megabytes (MiB), the unit every heap figure is shown in. */
+export function megabytes(bytes: number): number {
+  return Math.round(bytes / 1024 / 1024);
+}
+
 /** V8's heap limit for this process, in megabytes. */
 export function heapLimitMb(): number {
-  return Math.round(getHeapStatistics().heap_size_limit / 1024 / 1024);
+  return megabytes(getHeapStatistics().heap_size_limit);
 }
 
 export function isHeapLimitLow(limitMb: number): boolean {

src/lib/support/report.ts

@@ -25,6 +25,7 @@ import {
   heapLimitMb,
   isHeapLimitLow,
   lowHeapLimitAdvice,
+  megabytes,
 } from "@/lib/support/heap-limit";
 import { shouldEmitSecureCookie } from "@/lib/auth/secure-cookie";
 import { isOidcConfigured, isOidcOnly } from "@/lib/auth/oidc";
@@ -692,8 +693,8 @@ function or(value: string | number | null | undefined, fallback = "unknown") {
     : String(value);
 }
 
-function megabytes(bytes: number | null): string {
-  return bytes === null ? "unknown" : `${Math.round(bytes / 1024 / 1024)} MB`;
+function sizeLine(bytes: number | null): string {
+  return bytes === null ? "unknown" : `${megabytes(bytes)} MB`;
 }
 
 function runLine(run: QueueRunFact | null): string {
@@ -741,7 +742,7 @@ export function renderSupportReport(facts: SupportReportFacts): string {
   section("Database");
   row("PostgreSQL", facts.database.version);
   row("Server time zone", facts.database.timezone);
-  row("Size", megabytes(facts.database.sizeBytes));
+  row("Size", sizeLine(facts.database.sizeBytes));
   row(
     "Last migration",
     facts.database.lastMigration