diff --git a/docker-compose.yml b/docker-compose.yml index 80d8c9015..e03a72d4e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -492,8 +492,8 @@ services: # workload can grow large enough to starve Postgres on a shared host. # Node sizes its heap from this limit: 1g gives a 524 MB heap, which a # long-used account fills in normal operation, so garbage collection - # stalls requests for seconds and a weekly backup can crash the process. - # 1536m gives 792 MB. On a host with 2 GB or less, lower it with + # stalls requests for seconds. 1536m gives 792 MB. The support report and + # Admin > System status flag a heap below 700 MB. On a host with 2 GB or less, lower it with # APP_MEMORY_LIMIT in .env; on a busy instance, raise it. mem_limit: "${APP_MEMORY_LIMIT:-1536m}" healthcheck: diff --git a/docs/self-hosting/getting-started.md b/docs/self-hosting/getting-started.md index 58e7f5856..e9ee7aa1f 100644 --- a/docs/self-hosting/getting-started.md +++ b/docs/self-hosting/getting-started.md @@ -18,7 +18,7 @@ already installed, longer if a TLS-fronted public hostname is in scope heap at about 792 MB). On a 2 GB host that leaves too little for Postgres and the system, so set `APP_MEMORY_LIMIT=1g` in `.env` (a heap of about 524 MB); a long-used account then runs closer to - that heap, and the weekly backup with it. + that heap, and the support report and Admin > System status say so. - **Outbound HTTPS** to `ghcr.io` (image pulls) plus whichever integration endpoints you plan to enable (Withings, OpenAI, etc.). The opt-in environmental-context module additionally reaches diff --git a/messages/de.json b/messages/de.json index 9e8123961..1e7d0ea10 100644 --- a/messages/de.json +++ b/messages/de.json @@ -7571,6 +7571,9 @@ "offlineGeoLabel": "Geodaten", "offlineGeoEnabled": "Offline (GeoLite2)", "offlineGeoFallback": "Online ({host}). GeoLite2-Datenbanken einbinden und GEOLITE2_DIR setzen", + "heapLimitLabel": "Heap-Grenze", + "heapLimitValue": "{mb} MB", + "heapLimitLow": "{mb} MB, zu knapp. APP_MEMORY_LIMIT=1536m in .env setzen", "offlineGeoOff": "Aus", "offlineGeoChosen": "Online ({host})", "apiGlobal": "API-Endpunkte und API-Tokens erlauben.", diff --git a/messages/en.json b/messages/en.json index e4ca118bc..5e59955ce 100644 --- a/messages/en.json +++ b/messages/en.json @@ -7571,6 +7571,9 @@ "offlineGeoLabel": "Geo data", "offlineGeoEnabled": "Offline (GeoLite2)", "offlineGeoFallback": "Online ({host}). Mount GeoLite2 databases and set GEOLITE2_DIR", + "heapLimitLabel": "Heap limit", + "heapLimitValue": "{mb} MB", + "heapLimitLow": "{mb} MB, low. Set APP_MEMORY_LIMIT=1536m in .env", "offlineGeoOff": "Off", "offlineGeoChosen": "Online ({host})", "apiGlobal": "Allow API endpoints and API tokens.", diff --git a/messages/es.json b/messages/es.json index 8a08be38e..1c799ce67 100644 --- a/messages/es.json +++ b/messages/es.json @@ -7571,6 +7571,9 @@ "offlineGeoLabel": "Datos geográficos", "offlineGeoEnabled": "Sin conexión (GeoLite2)", "offlineGeoFallback": "En línea ({host}): monta las bases GeoLite2 y define GEOLITE2_DIR", + "heapLimitLabel": "Límite de heap", + "heapLimitValue": "{mb} MB", + "heapLimitLow": "{mb} MB, escaso: define APP_MEMORY_LIMIT=1536m en .env", "offlineGeoOff": "Desactivado", "offlineGeoChosen": "En línea ({host})", "apiGlobal": "Permitir endpoints de API y tokens de API.", diff --git a/messages/fr.json b/messages/fr.json index 8aff76af4..230f4a60a 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -7571,6 +7571,9 @@ "offlineGeoLabel": "Données géographiques", "offlineGeoEnabled": "Hors ligne (GeoLite2)", "offlineGeoFallback": "En ligne ({host}) : montez les bases GeoLite2 et définissez GEOLITE2_DIR", + "heapLimitLabel": "Limite du tas", + "heapLimitValue": "{mb} Mo", + "heapLimitLow": "{mb} Mo, trop juste : définissez APP_MEMORY_LIMIT=1536m dans .env", "offlineGeoOff": "Désactivé", "offlineGeoChosen": "En ligne ({host})", "apiGlobal": "Autoriser les points de terminaison d'API et les jetons d'API.", diff --git a/messages/it.json b/messages/it.json index 576240f9b..5446a3205 100644 --- a/messages/it.json +++ b/messages/it.json @@ -7571,6 +7571,9 @@ "offlineGeoLabel": "Dati geografici", "offlineGeoEnabled": "Offline (GeoLite2)", "offlineGeoFallback": "Online ({host}): monta i database GeoLite2 e imposta GEOLITE2_DIR", + "heapLimitLabel": "Limite heap", + "heapLimitValue": "{mb} MB", + "heapLimitLow": "{mb} MB, poco: imposta APP_MEMORY_LIMIT=1536m in .env", "offlineGeoOff": "Disattivato", "offlineGeoChosen": "Online ({host})", "apiGlobal": "Consentire endpoint API e token API.", diff --git a/messages/ko.json b/messages/ko.json index 51f44d7a8..753810a79 100644 --- a/messages/ko.json +++ b/messages/ko.json @@ -7546,6 +7546,9 @@ "offlineGeoLabel": "위치 데이터", "offlineGeoEnabled": "오프라인 (GeoLite2)", "offlineGeoFallback": "온라인 ({host}). GeoLite2 데이터베이스를 마운트하고 GEOLITE2_DIR을 설정하세요", + "heapLimitLabel": "힙 한도", + "heapLimitValue": "{mb} MB", + "heapLimitLow": "{mb} MB, 부족합니다. .env에 APP_MEMORY_LIMIT=1536m을 설정하세요", "offlineGeoOff": "꺼짐", "offlineGeoChosen": "온라인 ({host})", "apiGlobal": "API 엔드포인트와 API 토큰을 허용해요.", diff --git a/messages/pl.json b/messages/pl.json index eea453854..3046dc530 100644 --- a/messages/pl.json +++ b/messages/pl.json @@ -7571,6 +7571,9 @@ "offlineGeoLabel": "Dane geograficzne", "offlineGeoEnabled": "Offline (GeoLite2)", "offlineGeoFallback": "Online ({host}): podłącz bazy GeoLite2 i ustaw GEOLITE2_DIR", + "heapLimitLabel": "Limit sterty", + "heapLimitValue": "{mb} MB", + "heapLimitLow": "{mb} MB, za mało: ustaw APP_MEMORY_LIMIT=1536m w .env", "offlineGeoOff": "Wyłączone", "offlineGeoChosen": "Online ({host})", "apiGlobal": "Zezwól na punkty końcowe API i tokeny API.", diff --git a/src/app/api/admin/status/__tests__/route.test.ts b/src/app/api/admin/status/__tests__/route.test.ts index deef1897b..fc4f84c46 100644 --- a/src/app/api/admin/status/__tests__/route.test.ts +++ b/src/app/api/admin/status/__tests__/route.test.ts @@ -44,6 +44,14 @@ vi.mock("@/lib/jobs/worker-status", () => ({ })), })); +const heapMock = vi.hoisted(() => ({ mb: 792 })); +vi.mock("@/lib/support/heap-limit", async () => { + const actual = await vi.importActual< + typeof import("@/lib/support/heap-limit") + >("@/lib/support/heap-limit"); + return { ...actual, heapLimitMb: () => heapMock.mb }; +}); + const geoMock = vi.hoisted(() => ({ lookup: "online" as "offline" | "online" | "off", chosen: false, @@ -187,6 +195,19 @@ describe("GET /api/admin/status (integration health summary)", () => { expect(tokenCallArgs).toEqual({ where: { revoked: false } }); }); + it("flags a heap limit too small for a long-used account", async () => { + heapMock.mb = 524; + const low = (await (await GET()).json()) as { + data: { memory: { heapLimitMb: number; low: boolean } }; + }; + expect(low.data.memory).toEqual({ heapLimitMb: 524, low: true }); + heapMock.mb = 792; + const enough = (await (await GET()).json()) as { + data: { memory: { heapLimitMb: number; low: boolean } }; + }; + expect(enough.data.memory).toEqual({ heapLimitMb: 792, low: false }); + }); + it("reports named failing queues through failingJobs only", async () => { vi.mocked(readFailingQueues).mockResolvedValueOnce([ { diff --git a/src/app/api/admin/status/route.ts b/src/app/api/admin/status/route.ts index 5e7f4b9ab..b016702f3 100644 --- a/src/app/api/admin/status/route.ts +++ b/src/app/api/admin/status/route.ts @@ -9,6 +9,7 @@ import { readFailingQueues, } from "@/lib/jobs/job-failures"; import { readFileSync } from "node:fs"; +import { heapLimitMb, isHeapLimitLow } from "@/lib/support/heap-limit"; import { geoLookupMode, geoProviderChosen, @@ -73,6 +74,9 @@ export const GET = apiHandler(async () => { // reports. `null` = no queue schema to ask (web-only deployment); an empty // array = asked, nothing is failing. const failingJobs = await readFailingQueues(); + // The process's heap ceiling, and whether it is the 1 GB-container size + // that a long-used account can fill (`heap-limit.ts`). + const heapLimit = heapLimitMb(); return apiSuccess({ version: process.env.npm_package_version || "0.1.0", @@ -81,6 +85,7 @@ export const GET = apiHandler(async () => { buildTime: BUILD_TIME, startTime: START_TIME, database: "connected", + memory: { heapLimitMb: heapLimit, low: isHeapLimitLow(heapLimit) }, worker: { running: workerStatus.running, startedAt: workerStatus.startedAt, diff --git a/src/components/admin/_shared.tsx b/src/components/admin/_shared.tsx index 6d4545810..7abbebe4b 100644 --- a/src/components/admin/_shared.tsx +++ b/src/components/admin/_shared.tsx @@ -61,6 +61,8 @@ export interface SystemStatus { buildTime: string; startTime: string; database: string; + /** The process's heap ceiling; `low` when it is below what a long-used account needs. */ + memory: { heapLimitMb: number; low: boolean }; worker: WorkerStatus; /** * Queues whose jobs exhausted their retries recently. `null` means the diff --git a/src/components/admin/system-status-section.tsx b/src/components/admin/system-status-section.tsx index a3cf1f4c4..b2811261e 100644 --- a/src/components/admin/system-status-section.tsx +++ b/src/components/admin/system-status-section.tsx @@ -13,6 +13,7 @@ import { Key, Loader2, Map, + MemoryStick, Server, Users, } from "lucide-react"; @@ -156,6 +157,16 @@ export function SystemStatusSection() { } tone={status.database === "connected" ? "success" : "destructive"} /> + { expect(text).toContain("environment-fetch"); }); + it("says what to raise when the heap limit is low, and keeps it through redaction", () => { + const facts = seededFacts(); + const low = redactSupportText( + renderSupportReport({ + ...facts, + build: { ...facts.build, heapLimitMb: 524 }, + }), + CONTEXT, + ); + expect(low).toContain("- Heap limit: 524 MB"); + expect(low).toContain( + "- Heap limit note: The heap limit of 524 MB is below 700 MB.", + ); + expect(low).toContain("APP_MEMORY_LIMIT=1536m in .env"); + expect(low).toContain("docker compose up -d."); + const enough = redactSupportText(renderSupportReport(facts), CONTEXT); + expect(enough).not.toContain("Heap limit note"); + }); + it("the seeds really are in the unredacted text", () => { const raw = renderSupportReport(seededFacts()); for (const value of FORBIDDEN) expect(raw).toContain(value); @@ -256,6 +275,21 @@ describe("error classes never carry the message", () => { expect(errorClassOf("job timed out: did not finish within 1 h")).toBe( "timeout", ); + // A statement past its timeout, as Prisma words it: the SQLSTATE is the + // class, not the bare "Error" it used to read as. + expect( + errorClassOf( + "\nInvalid `prisma.measurement.findMany()` invocation:\n\n\nDatabase error. Code: `57014`. Message: `canceling statement due to statement timeout`", + ), + ).toBe("statement_timeout"); + expect( + errorClassOf( + "compaction-tombstone-purge: stopped while deleting a batch (lock_timeout)", + ), + ).toBe("lock_timeout"); + expect(errorClassOf("timeout exceeded when trying to connect")).toBe( + "pool_timeout", + ); expect(errorClassOf("anything at all, an email a@b.co")).toBe("Error"); expect(errorClassOf(null)).toBe("unknown"); }); diff --git a/src/lib/support/heap-limit.ts b/src/lib/support/heap-limit.ts new file mode 100644 index 000000000..e11c9656e --- /dev/null +++ b/src/lib/support/heap-limit.ts @@ -0,0 +1,35 @@ +/** + * Whether this process's heap is below what a long-used account needs. + * + * Node sizes its heap from the container's memory limit: the 1 GB limit older + * compose files set gives 524 MB, the current default of 1536m gives 792 MB. + * At 524 MB a long-used account fills the heap in normal operation, garbage + * collection stalls requests, and a large import can run the process out of + * memory, which restarts the whole instance. The support report and the admin + * system status say so in one sentence, because the number alone ("Heap + * limit: 524 MB") reads as a fact rather than as the cause (#1239). + */ +import { getHeapStatistics } from "node:v8"; + +/** Below this, the heap is flagged. Between the 1g (524) and 1536m (792) heaps. */ +export const LOW_HEAP_LIMIT_MB = 700; + +/** V8's heap limit for this process, in megabytes. */ +export function heapLimitMb(): number { + return Math.round(getHeapStatistics().heap_size_limit / 1024 / 1024); +} + +export function isHeapLimitLow(limitMb: number): boolean { + return limitMb < LOW_HEAP_LIMIT_MB; +} + +/** The support report's sentence for a low heap limit. */ +export function lowHeapLimitAdvice(limitMb: number): string { + return ( + `The heap limit of ${limitMb} MB is below ${LOW_HEAP_LIMIT_MB} MB. The ` + + "usual cause is a 1 GB container memory limit. A long-used account can " + + "fill this heap, which slows requests and can restart the app. Set " + + "APP_MEMORY_LIMIT=1536m in .env (about 792 MB of heap) and run " + + "docker compose up -d." + ); +} diff --git a/src/lib/support/report.ts b/src/lib/support/report.ts index 8dae07ce3..15ac11743 100644 --- a/src/lib/support/report.ts +++ b/src/lib/support/report.ts @@ -18,9 +18,14 @@ * * The report is English on purpose: it is read by whoever answers the issue. */ -import { getHeapStatistics } from "node:v8"; import { prisma } from "@/lib/db"; +import { failureCondition } from "@/lib/jobs/failure-condition"; +import { + heapLimitMb, + isHeapLimitLow, + lowHeapLimitAdvice, +} from "@/lib/support/heap-limit"; import { shouldEmitSecureCookie } from "@/lib/auth/secure-cookie"; import { isOidcConfigured, isOidcOnly } from "@/lib/auth/oidc"; import { getLoggingConfig } from "@/lib/logging/config"; @@ -183,6 +188,11 @@ export function errorClassOf(text: string | null | undefined): string { const parts: string[] = []; const named = /^([A-Z][A-Za-z0-9]{1,60}(?:Error|Exception))\b/.exec(trimmed); if (named) parts.push(named[1]); + // The database condition (`statement_timeout`, `lock_timeout`, ...): a + // Prisma error carries it only as a SQLSTATE inside the message, and + // without it a statement timeout read as the bare class "Error" (#1239). + const condition = failureCondition(trimmed); + if (condition) return [...parts, condition].join(" "); const code = /\b(P\d{4})\b/.exec(trimmed) ?? /\b(E[A-Z][A-Z_]{2,30})\b/.exec(trimmed) ?? @@ -605,9 +615,7 @@ export async function collectSupportReportFacts( builtAt: process.env.NEXT_PUBLIC_APP_BUILT_AT?.trim() || null, node: process.version, processType: getProcessType(), - heapLimitMb: Math.round( - getHeapStatistics().heap_size_limit / 1024 / 1024, - ), + heapLimitMb: heapLimitMb(), uptimeSeconds: Math.round(process.uptime()), }, database, @@ -722,6 +730,9 @@ export function renderSupportReport(facts: SupportReportFacts): string { row("Node", facts.build.node); row("Process type", facts.build.processType); row("Heap limit", `${facts.build.heapLimitMb} MB`); + if (isHeapLimitLow(facts.build.heapLimitMb)) { + row("Heap limit note", lowHeapLimitAdvice(facts.build.heapLimitMb)); + } row( "Uptime", `${Math.floor(facts.build.uptimeSeconds / 3600)} h ${Math.floor((facts.build.uptimeSeconds % 3600) / 60)} min`,