diff --git a/src/lib/jobs/__tests__/host-metric-sampler.test.ts b/src/lib/jobs/__tests__/host-metric-sampler.test.ts new file mode 100644 index 000000000..73d30c396 --- /dev/null +++ b/src/lib/jobs/__tests__/host-metric-sampler.test.ts @@ -0,0 +1,162 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import type { PrismaClient } from "@/generated/prisma/client"; +import { + captureHostMetric, + getHostMetricRetentionDays, + runHostMetricTick, + DEFAULT_HOST_METRIC_RETENTION_DAYS, +} from "../host-metric-sampler"; + +function makePrismaMock() { + return { + hostMetric: { + create: vi.fn().mockResolvedValue({}), + deleteMany: vi.fn().mockResolvedValue({ count: 3 }), + }, + } as unknown as PrismaClient; +} + +const ORIGINAL_ENV = { ...process.env }; + +beforeEach(() => { + process.env = { ...ORIGINAL_ENV }; + delete process.env.HOST_METRIC_RETENTION_DAYS; +}); + +afterEach(() => { + process.env = ORIGINAL_ENV; + vi.restoreAllMocks(); +}); + +describe("getHostMetricRetentionDays", () => { + it("returns 7 days when env is unset", () => { + expect(getHostMetricRetentionDays()).toBe( + DEFAULT_HOST_METRIC_RETENTION_DAYS, + ); + }); + + it("respects a valid override", () => { + process.env.HOST_METRIC_RETENTION_DAYS = "14"; + expect(getHostMetricRetentionDays()).toBe(14); + }); + + it("ignores nonsensical values (NaN, 0, negative)", () => { + process.env.HOST_METRIC_RETENTION_DAYS = "0"; + expect(getHostMetricRetentionDays()).toBe( + DEFAULT_HOST_METRIC_RETENTION_DAYS, + ); + process.env.HOST_METRIC_RETENTION_DAYS = "-100"; + expect(getHostMetricRetentionDays()).toBe( + DEFAULT_HOST_METRIC_RETENTION_DAYS, + ); + process.env.HOST_METRIC_RETENTION_DAYS = "not a number"; + expect(getHostMetricRetentionDays()).toBe( + DEFAULT_HOST_METRIC_RETENTION_DAYS, + ); + }); +}); + +describe("captureHostMetric", () => { + it("returns load + memory fields and forwards disk-stats output", async () => { + const sample = await captureHostMetric(async () => ({ + readBytes: BigInt(12345), + writeBytes: BigInt(67890), + })); + + expect(typeof sample.loadAvg1).toBe("number"); + expect(typeof sample.loadAvg5).toBe("number"); + expect(typeof sample.loadAvg15).toBe("number"); + // Two-decimal precision keeps Postgres rows compact. + expect(sample.loadAvg1).toBe(Math.round(sample.loadAvg1 * 100) / 100); + + expect(typeof sample.memUsedBytes).toBe("bigint"); + expect(typeof sample.memTotalBytes).toBe("bigint"); + expect(sample.memTotalBytes).toBeGreaterThan(BigInt(0)); + expect(sample.memUsedBytes).toBeLessThanOrEqual(sample.memTotalBytes); + + expect(sample.diskReadBytes).toBe(BigInt(12345)); + expect(sample.diskWriteBytes).toBe(BigInt(67890)); + }); + + it("returns null disk fields when the diskstats reader returns null", async () => { + const sample = await captureHostMetric(async () => null); + + expect(sample.diskReadBytes).toBeNull(); + expect(sample.diskWriteBytes).toBeNull(); + }); + + it("returns null disk fields when the diskstats reader throws", async () => { + const sample = await captureHostMetric(async () => { + throw new Error("EACCES /proc/diskstats"); + }); + + expect(sample.diskReadBytes).toBeNull(); + expect(sample.diskWriteBytes).toBeNull(); + }); +}); + +describe("runHostMetricTick", () => { + it("inserts a sample row and prunes expired rows", async () => { + const prisma = makePrismaMock(); + const now = new Date("2026-05-09T12:00:00Z"); + + const result = await runHostMetricTick(prisma, { + now, + captureFn: async () => ({ + loadAvg1: 0.42, + loadAvg5: 0.55, + loadAvg15: 0.61, + memUsedBytes: BigInt(4_000_000_000), + memTotalBytes: BigInt(8_000_000_000), + diskReadBytes: BigInt(1_000), + diskWriteBytes: BigInt(2_000), + }), + }); + + expect(result).toEqual({ inserted: 1, pruned: 3 }); + + expect(prisma.hostMetric.create).toHaveBeenCalledTimes(1); + expect(prisma.hostMetric.create).toHaveBeenCalledWith({ + data: { + capturedAt: now, + loadAvg1: 0.42, + loadAvg5: 0.55, + loadAvg15: 0.61, + memUsedBytes: BigInt(4_000_000_000), + memTotalBytes: BigInt(8_000_000_000), + diskReadBytes: BigInt(1_000), + diskWriteBytes: BigInt(2_000), + }, + }); + + // 7-day cutoff by default. + const expectedCutoff = new Date(now.getTime() - 7 * 86_400_000); + expect(prisma.hostMetric.deleteMany).toHaveBeenCalledWith({ + where: { capturedAt: { lt: expectedCutoff } }, + }); + }); + + it("respects HOST_METRIC_RETENTION_DAYS for the prune cutoff", async () => { + process.env.HOST_METRIC_RETENTION_DAYS = "14"; + const prisma = makePrismaMock(); + const now = new Date("2026-05-09T12:00:00Z"); + + await runHostMetricTick(prisma, { + now, + captureFn: async () => ({ + loadAvg1: 0, + loadAvg5: 0, + loadAvg15: 0, + memUsedBytes: BigInt(1), + memTotalBytes: BigInt(2), + diskReadBytes: null, + diskWriteBytes: null, + }), + }); + + const expectedCutoff = new Date(now.getTime() - 14 * 86_400_000); + expect(prisma.hostMetric.deleteMany).toHaveBeenCalledWith({ + where: { capturedAt: { lt: expectedCutoff } }, + }); + }); +}); diff --git a/src/lib/jobs/host-metric-sampler.ts b/src/lib/jobs/host-metric-sampler.ts new file mode 100644 index 000000000..6eede1ba7 --- /dev/null +++ b/src/lib/jobs/host-metric-sampler.ts @@ -0,0 +1,174 @@ +/** + * Per-minute host-load sampler that backs the admin /admin/system-status + * "last 2 hours" chart (v1.4.16 phase B3). + * + * Marc asked for a small at-a-glance graph above the system-status facts + * grid showing CPU + memory + disk-IO so a glance at /admin tells him + * whether the apps-01 host is healthy. We considered scraping Coolify's + * Sentinel collector, but the Coolify HTTP API does not expose a clean + * read endpoint for per-server metrics — every approach ended up either + * scraping the Coolify dashboard HTML or hitting a private RPC. An + * in-process sampler costs us a single 7-column row per minute, never + * leaves the app's own DB, and works identically on dev (macOS) and + * production (Linux). + * + * The sampler uses `os.loadavg()` (1/5/15-minute averages) and + * `os.totalmem() - os.freemem()` for memory. On Linux it also reads the + * cumulative /proc/diskstats counters; non-Linux hosts return null disk + * fields, which the API endpoint then surfaces as "no disk data" instead + * of failing the whole chart. + * + * Retention: rows older than 7 days are deleted at the end of every + * tick. With one insert per minute that's 10,080 rows steady-state, + * indexed by `captured_at` so the 2h query stays under 200 rows. + */ +import { loadavg, totalmem, freemem } from "node:os"; +import { readFile } from "node:fs/promises"; +import type { PrismaClient } from "@/generated/prisma/client"; + +export const DEFAULT_HOST_METRIC_RETENTION_DAYS = 7; + +export interface HostMetricSample { + loadAvg1: number; + loadAvg5: number; + loadAvg15: number; + memUsedBytes: bigint; + memTotalBytes: bigint; + diskReadBytes: bigint | null; + diskWriteBytes: bigint | null; +} + +/** + * Capture a single host-load snapshot. Pure-ish — only `os.*` and + * `/proc/diskstats` are touched, so the unit test can stub + * `readDiskStats` to assert the disk path without mocking `node:os`. + */ +export async function captureHostMetric( + readDiskStats: () => Promise<{ + readBytes: bigint; + writeBytes: bigint; + } | null> = readLinuxDiskStats, +): Promise { + const [load1, load5, load15] = loadavg(); + const total = totalmem(); + const free = freemem(); + const used = total - free; + const disk = await readDiskStats().catch(() => null); + + return { + loadAvg1: round(load1), + loadAvg5: round(load5), + loadAvg15: round(load15), + memUsedBytes: BigInt(used), + memTotalBytes: BigInt(total), + diskReadBytes: disk?.readBytes ?? null, + diskWriteBytes: disk?.writeBytes ?? null, + }; +} + +/** + * Linux /proc/diskstats reader. Returns the sum of read + write byte + * counters across all non-virtual block devices (loop/ram/dm-* are + * excluded — they double-count or report meaningless numbers). + * + * Each /proc/diskstats line is whitespace-separated: + * 3:major 4:minor ... 6:sectors_read ... 10:sectors_written ... + * Multiplying sectors by the canonical 512 B sector size gives the + * cumulative byte counters the chart uses to derive bytes-per-second + * between samples. + */ +async function readLinuxDiskStats(): Promise<{ + readBytes: bigint; + writeBytes: bigint; +} | null> { + let raw: string; + try { + raw = await readFile("/proc/diskstats", "utf-8"); + } catch { + return null; + } + + let totalRead = BigInt(0); + let totalWrite = BigInt(0); + const SECTOR_BYTES = BigInt(512); + + for (const line of raw.split("\n")) { + const parts = line.trim().split(/\s+/); + if (parts.length < 14) continue; + const device = parts[2]; + // Skip virtual / synthetic devices that double-count or report + // bogus numbers. ram*/loop* are pseudo-devices; dm-* and md* are + // device-mapper / RAID overlays whose underlying disks are already + // counted by the physical device line. + if ( + !device || + /^(loop|ram|dm-|md|sr|fd)/.test(device) || + /^(nbd|zram)/.test(device) + ) { + continue; + } + + const sectorsRead = BigInt(parts[5] ?? "0"); + const sectorsWritten = BigInt(parts[9] ?? "0"); + totalRead += sectorsRead * SECTOR_BYTES; + totalWrite += sectorsWritten * SECTOR_BYTES; + } + + return { readBytes: totalRead, writeBytes: totalWrite }; +} + +export function getHostMetricRetentionDays(): number { + const raw = process.env.HOST_METRIC_RETENTION_DAYS; + if (raw === undefined) return DEFAULT_HOST_METRIC_RETENTION_DAYS; + const parsed = parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + return DEFAULT_HOST_METRIC_RETENTION_DAYS; + } + // Refuse a misconfigured value below 1 day — the chart needs at least + // 2h of data, but cron drift + missed ticks make 1d the safe floor. + if (parsed < 1) return DEFAULT_HOST_METRIC_RETENTION_DAYS; + return parsed; +} + +/** + * Insert a single sample row + delete expired rows. Returns the number + * of rows pruned so the worker can annotate its Wide Event. + */ +export async function runHostMetricTick( + prisma: PrismaClient, + options: { + now?: Date; + captureFn?: () => Promise; + } = {}, +): Promise<{ inserted: 1; pruned: number }> { + const now = options.now ?? new Date(); + const sample = await (options.captureFn ?? (() => captureHostMetric()))(); + + await prisma.hostMetric.create({ + data: { + capturedAt: now, + loadAvg1: sample.loadAvg1, + loadAvg5: sample.loadAvg5, + loadAvg15: sample.loadAvg15, + memUsedBytes: sample.memUsedBytes, + memTotalBytes: sample.memTotalBytes, + diskReadBytes: sample.diskReadBytes, + diskWriteBytes: sample.diskWriteBytes, + }, + }); + + const cutoff = new Date( + now.getTime() - getHostMetricRetentionDays() * 86_400_000, + ); + const { count } = await prisma.hostMetric.deleteMany({ + where: { capturedAt: { lt: cutoff } }, + }); + + return { inserted: 1, pruned: count }; +} + +function round(value: number): number { + // 2-decimal precision is enough for a chart whose y-axis spans 0–4 + // load — and it keeps Postgres `DOUBLE PRECISION` rows compact. + return Math.round(value * 100) / 100; +} diff --git a/src/lib/jobs/reminder-worker.ts b/src/lib/jobs/reminder-worker.ts index 74391dca9..01a6455a3 100644 --- a/src/lib/jobs/reminder-worker.ts +++ b/src/lib/jobs/reminder-worker.ts @@ -29,6 +29,7 @@ import { import { setGlobalBoss } from "@/lib/jobs/boss-instance"; import { cleanupExpiredIdempotencyKeys } from "@/lib/jobs/idempotency-cleanup"; import { cleanupOldAuditLogs } from "@/lib/jobs/audit-log-cleanup"; +import { runHostMetricTick } from "@/lib/jobs/host-metric-sampler"; import { rotateLegacyMoodLogSecrets } from "@/lib/moodlog-secret"; import { deleteMessage } from "@/lib/telegram"; import { decrypt, encrypt } from "@/lib/crypto"; @@ -105,6 +106,9 @@ const OFFHOST_BACKUP_QUEUE = "data-backup-offhost"; // in-DB DATA_BACKUP at 03:00 (Sundays only) so the off-host copy is // always at-or-ahead of the local one. const OFFHOST_BACKUP_CRON = "30 2 * * *"; +const HOST_METRIC_QUEUE = "host-metric-sample"; +// Per-minute cadence — matches the chart's 60s polling refetchInterval. +const HOST_METRIC_CRON = "* * * * *"; interface ReminderCheckPayload { triggeredAt: string; @@ -168,6 +172,10 @@ interface OffhostBackupPayload { triggeredAt: string; } +interface HostMetricSamplePayload { + triggeredAt: string; +} + // Re-export timezone utilities under local names for backward compatibility const getUserTodayBounds = getUserTodayBoundsUtil; const getDayOfWeekInTz = getDayOfWeekInTzUtil; @@ -915,6 +923,21 @@ async function handleAuditLogCleanup(jobs: Job[]) { }); } +async function handleHostMetricSample(jobs: Job[]) { + void jobs; + await withBackgroundEvent("job.host_metric_sample", async (evt) => { + const p = getWorkerPrisma(); + try { + const { pruned } = await runHostMetricTick(p); + evt.addMeta("host_metric_pruned", pruned); + } catch (err) { + // The chart degrades gracefully when samples are missing — log + // and move on rather than poisoning the boss queue with retries. + evt.addWarning(`host-metric-sample failed: ${err}`); + } + }); +} + async function handleOffhostBackup(jobs: Job[]) { void jobs; await withBackgroundEvent("job.offhost_backup", async (evt) => { @@ -1160,6 +1183,7 @@ export async function startReminderWorker() { IDEMPOTENCY_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_QUEUE, OFFHOST_BACKUP_QUEUE, + HOST_METRIC_QUEUE, ]; for (const q of allQueues) { @@ -1182,6 +1206,7 @@ export async function startReminderWorker() { [IDEMPOTENCY_CLEANUP_QUEUE, IDEMPOTENCY_CLEANUP_CRON], [AUDIT_LOG_CLEANUP_QUEUE, AUDIT_LOG_CLEANUP_CRON], [OFFHOST_BACKUP_QUEUE, OFFHOST_BACKUP_CRON], + [HOST_METRIC_QUEUE, HOST_METRIC_CRON], ]; for (const [name, cron] of schedules) { @@ -1264,6 +1289,11 @@ export async function startReminderWorker() { { localConcurrency: 1 }, handleOffhostBackup, ); + await boss.work( + HOST_METRIC_QUEUE, + { localConcurrency: 1 }, + handleHostMetricSample, + ); return boss; }