mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-10 21:32:18 +00:00
Marc-André Bombeck
b79bd8370d
The pressure report re-armed only after the heap fell under 70 % and half an hour passed, so a heap that climbed to its limit soon after an 85 % report died with only that older report on record. A 95 % tier now writes the same report at once, whatever the cooldown, and re-arms under 85 %. The image sets NODE_OPTIONS so Node writes its diagnostic report to stderr when the heap runs out, with environment variables and network interfaces excluded. The heap-diagnostics and scaling docs say how to keep the flags when overriding NODE_OPTIONS. Heap figures share one megabytes helper with the support report.
888 lines
28 KiB
TypeScript
888 lines
28 KiB
TypeScript
/**
|
|
* The operator's support report: a redacted plain-text summary of what this
|
|
* instance looks like, for pasting into a bug report.
|
|
*
|
|
* Three stages, kept apart so each can be tested on its own:
|
|
*
|
|
* 1. `collectSupportReportFacts()` reads the database, the queue and this
|
|
* process. Accounts leave it as numbers ("Account 1..n", oldest first);
|
|
* no account id, username or email is part of the facts.
|
|
* 2. `renderSupportReport()` turns the facts into text. Pure.
|
|
* 3. `redactSupportText()` runs every redaction step over that text, with
|
|
* the instance's own emails, usernames, names and ids as known values.
|
|
*
|
|
* What it never contains: an email address, an account id, a value anyone
|
|
* recorded, a token, the host name of a foreign service. Configuration is yes
|
|
* or no, never the setting itself. Errors appear as an action name and an
|
|
* error class, never as their message.
|
|
*
|
|
* The report is English on purpose: it is read by whoever answers the issue.
|
|
*/
|
|
|
|
import { prisma } from "@/lib/db";
|
|
import { failureCondition } from "@/lib/jobs/failure-condition";
|
|
import {
|
|
heapLimitMb,
|
|
isHeapLimitLow,
|
|
lowHeapLimitAdvice,
|
|
megabytes,
|
|
} from "@/lib/support/heap-limit";
|
|
import { shouldEmitSecureCookie } from "@/lib/auth/secure-cookie";
|
|
import { isOidcConfigured, isOidcOnly } from "@/lib/auth/oidc";
|
|
import { getLoggingConfig } from "@/lib/logging/config";
|
|
import {
|
|
readLogBuffer,
|
|
PROBLEMS_BUFFER_MAX,
|
|
} from "@/lib/logging/in-memory-buffer";
|
|
import type { WideEvent } from "@/lib/logging/types";
|
|
import {
|
|
offhostBackupConfigured,
|
|
OFFHOST_BACKUP_QUEUE,
|
|
} from "@/lib/jobs/offhost-backup";
|
|
import { DATA_BACKUP_QUEUE } from "@/lib/jobs/data-backup-policy";
|
|
import { RESTORE_DRILL_QUEUE } from "@/lib/jobs/restore-drill";
|
|
import {
|
|
JOB_FAILURE_WINDOW_HOURS,
|
|
readFailingQueues,
|
|
readLastQueueRun,
|
|
type LastQueueRun,
|
|
} from "@/lib/jobs/job-failures";
|
|
import { getWorkerStatus } from "@/lib/jobs/worker-status";
|
|
import { getProcessType, shouldRunWorker } from "@/lib/process-type";
|
|
import { offlineGeoReady } from "@/lib/geo";
|
|
import { configuredNotificationPrivateOrigins } from "@/lib/notifications/egress-policy";
|
|
import { isEmailConfigured } from "@/lib/notifications/senders/email-config";
|
|
import { loadConnectionHealth } from "@/lib/integrations/connection-health-load";
|
|
import { CONNECTION_KEYS } from "@/lib/integrations/connection-health";
|
|
import {
|
|
redactSupportText,
|
|
supportLine,
|
|
type SupportRedactionContext,
|
|
} from "./redact";
|
|
import packageJson from "../../../package.json";
|
|
|
|
/** Accounts whose connections the report reads; the rest are counted only. */
|
|
export const SUPPORT_REPORT_ACCOUNT_CAP = 200;
|
|
/** How far back the notification tallies reach. */
|
|
export const SUPPORT_REPORT_NOTIFICATION_DAYS = 7;
|
|
/** How many error actions the report lists. */
|
|
export const SUPPORT_REPORT_TOP_ERRORS = 20;
|
|
|
|
// ─── Facts ───────────────────────────────────────────────────────────────
|
|
|
|
export interface QueueRunFact {
|
|
at: string;
|
|
state: LastQueueRun["state"];
|
|
errorClass: string | null;
|
|
}
|
|
|
|
export interface SupportReportFacts {
|
|
generatedAt: string;
|
|
build: {
|
|
version: string;
|
|
buildSha: string | null;
|
|
builtAt: string | null;
|
|
node: string;
|
|
processType: string;
|
|
heapLimitMb: number;
|
|
uptimeSeconds: number;
|
|
};
|
|
database: {
|
|
version: string | null;
|
|
timezone: string | null;
|
|
sizeBytes: number | null;
|
|
lastMigration: string | null;
|
|
lastMigrationAt: string | null;
|
|
unfinishedMigrations: number | null;
|
|
};
|
|
/** Label and a yes / no / count answer. Never the setting itself. */
|
|
config: Array<[string, string]>;
|
|
jobs: {
|
|
workerInThisProcess: boolean;
|
|
workerRunning: boolean;
|
|
workerLastHeartbeat: string | null;
|
|
newestFinishedJobAt: string | null;
|
|
/** `null` when the queue tables could not be read. */
|
|
failing: Array<{
|
|
queue: string;
|
|
failures: number;
|
|
lastFailedAt: string;
|
|
errorClass: string;
|
|
}> | null;
|
|
};
|
|
accounts: { total: number; read: number };
|
|
connections: Array<{
|
|
provider: string;
|
|
state: string;
|
|
reason: string;
|
|
accounts: number[];
|
|
}>;
|
|
incidents: Array<{
|
|
provider: string;
|
|
reason: string;
|
|
openedAt: string | null;
|
|
announced: boolean;
|
|
account: number | null;
|
|
}>;
|
|
ai: {
|
|
defaultLevel: string | null;
|
|
maxLevel: string | null;
|
|
/** Level (or `instance default`) to number of accounts. */
|
|
levels: Array<[string, number]>;
|
|
providers: Array<{
|
|
provider: string;
|
|
ok: number;
|
|
failing: number;
|
|
authFailed: number;
|
|
paused: number;
|
|
lastFailureAt: string | null;
|
|
lastStatus: number | null;
|
|
}>;
|
|
};
|
|
notifications: Array<{
|
|
channel: string;
|
|
ok: number;
|
|
error: number;
|
|
skipped: number;
|
|
topReasons: Array<[string, number]>;
|
|
}>;
|
|
backups: {
|
|
newestLocalAt: string | null;
|
|
localCopies: number;
|
|
lastLocalRun: QueueRunFact | null;
|
|
offhostConfigured: boolean;
|
|
offhostNewestAt: string | null;
|
|
offhostOldestAt: string | null;
|
|
offhostAccounts: number;
|
|
offhostNeverStored: number;
|
|
lastOffhostRun: QueueRunFact | null;
|
|
lastRestoreDrill: QueueRunFact | null;
|
|
};
|
|
errors: {
|
|
ringSize: number;
|
|
ringMax: number;
|
|
top: Array<{
|
|
action: string;
|
|
level: string;
|
|
count: number;
|
|
lastSeen: string;
|
|
errorClass: string;
|
|
}>;
|
|
};
|
|
}
|
|
|
|
// ─── Error classes ───────────────────────────────────────────────────────
|
|
|
|
/**
|
|
* A short class for a failure text: never the message. Timeouts and lost
|
|
* heartbeats by name, otherwise the thrown class (`TypeError`), a Prisma code
|
|
* (`P2002`), a system code (`ECONNREFUSED`) or an HTTP status, whichever the
|
|
* text names; plain `Error` when it names none.
|
|
*/
|
|
export function errorClassOf(text: string | null | undefined): string {
|
|
if (!text) return "unknown";
|
|
const trimmed = text.trim();
|
|
if (/^(job timed out|handler execution exceeded)/i.test(trimmed)) {
|
|
return "timeout";
|
|
}
|
|
if (/^job heartbeat timeout/i.test(trimmed)) return "heartbeat_timeout";
|
|
const parts: string[] = [];
|
|
const named = /^([A-Z][A-Za-z0-9]{1,60}(?:Error|Exception))\b/.exec(trimmed);
|
|
if (named) parts.push(named[1]);
|
|
// The database condition (`statement_timeout`, `lock_timeout`, ...): a
|
|
// Prisma error carries it only as a SQLSTATE inside the message, and
|
|
// without it a statement timeout read as the bare class "Error" (#1239).
|
|
const condition = failureCondition(trimmed);
|
|
if (condition) return [...parts, condition].join(" ");
|
|
const code =
|
|
/\b(P\d{4})\b/.exec(trimmed) ??
|
|
/\b(E[A-Z][A-Z_]{2,30})\b/.exec(trimmed) ??
|
|
/\b(?:HTTP|status)\s*:?\s*([1-5]\d{2})\b/i.exec(trimmed);
|
|
if (code) parts.push(code[1]);
|
|
return parts.length > 0 ? parts.join(" ") : "Error";
|
|
}
|
|
|
|
/** The class of a wide event's failure: thrown class and code, else its level. */
|
|
export function eventErrorClass(event: WideEvent): string {
|
|
if (event.error) {
|
|
const type = supportLine(event.error.type, 60) || "Error";
|
|
const code =
|
|
event.error.code !== undefined && event.error.code !== null
|
|
? supportLine(String(event.error.code), 40)
|
|
: "";
|
|
return code ? `${type} ${code}` : type;
|
|
}
|
|
const status = event.http?.status;
|
|
return typeof status === "number" && status >= 400 ? `HTTP ${status}` : "-";
|
|
}
|
|
|
|
/** The name an event is grouped under: its action, else its kind and status. */
|
|
function eventActionName(event: WideEvent): string {
|
|
const name = supportLine(event.action?.name, 80);
|
|
if (name) return name;
|
|
if (event.kind === "background") {
|
|
return supportLine(event.background?.task_name, 80) || "background";
|
|
}
|
|
const status = event.http?.status;
|
|
return `http ${event.http?.method ?? ""} ${status ?? ""}`
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
/** The most frequent warn/error actions in the problems ring. */
|
|
export function summariseProblemEvents(
|
|
events: readonly WideEvent[],
|
|
limit: number = SUPPORT_REPORT_TOP_ERRORS,
|
|
): SupportReportFacts["errors"]["top"] {
|
|
const groups = new Map<
|
|
string,
|
|
{
|
|
action: string;
|
|
level: string;
|
|
count: number;
|
|
lastSeen: string;
|
|
errorClass: string;
|
|
}
|
|
>();
|
|
for (const event of events) {
|
|
const action = eventActionName(event);
|
|
const key = `${event.level}\u0000${action}`;
|
|
const existing = groups.get(key);
|
|
if (!existing) {
|
|
groups.set(key, {
|
|
action,
|
|
level: event.level,
|
|
count: 1,
|
|
lastSeen: event.timestamp,
|
|
errorClass: eventErrorClass(event),
|
|
});
|
|
continue;
|
|
}
|
|
existing.count += 1;
|
|
if (event.timestamp > existing.lastSeen) {
|
|
existing.lastSeen = event.timestamp;
|
|
existing.errorClass = eventErrorClass(event);
|
|
}
|
|
}
|
|
return [...groups.values()]
|
|
.sort((a, b) => b.count - a.count || b.lastSeen.localeCompare(a.lastSeen))
|
|
.slice(0, limit);
|
|
}
|
|
|
|
// ─── Collect ─────────────────────────────────────────────────────────────
|
|
|
|
function iso(value: Date | null | undefined): string | null {
|
|
return value ? value.toISOString() : null;
|
|
}
|
|
|
|
function runFact(run: LastQueueRun | null): QueueRunFact | null {
|
|
if (!run) return null;
|
|
return {
|
|
at: run.at,
|
|
state: run.state,
|
|
errorClass: run.error ? errorClassOf(run.error) : null,
|
|
};
|
|
}
|
|
|
|
function yesNo(value: boolean): string {
|
|
return value ? "yes" : "no";
|
|
}
|
|
|
|
function envSet(name: string): boolean {
|
|
return (process.env[name] ?? "").trim().length > 0;
|
|
}
|
|
|
|
async function readDatabaseFacts(): Promise<SupportReportFacts["database"]> {
|
|
const out: SupportReportFacts["database"] = {
|
|
version: null,
|
|
timezone: null,
|
|
sizeBytes: null,
|
|
lastMigration: null,
|
|
lastMigrationAt: null,
|
|
unfinishedMigrations: null,
|
|
};
|
|
try {
|
|
const [row] = await prisma.$queryRaw<
|
|
Array<{ version: string; timezone: string; size: bigint }>
|
|
>`
|
|
SELECT current_setting('server_version') AS version,
|
|
current_setting('TimeZone') AS timezone,
|
|
pg_database_size(current_database()) AS size
|
|
`;
|
|
if (row) {
|
|
out.version = row.version;
|
|
out.timezone = row.timezone;
|
|
out.sizeBytes = Number(row.size);
|
|
}
|
|
} catch {
|
|
/* reported as unknown */
|
|
}
|
|
try {
|
|
const [last] = await prisma.$queryRaw<
|
|
Array<{ migration_name: string; finished_at: Date | null }>
|
|
>`
|
|
SELECT migration_name, finished_at
|
|
FROM _prisma_migrations
|
|
WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL
|
|
ORDER BY finished_at DESC, migration_name DESC
|
|
LIMIT 1
|
|
`;
|
|
const [unfinished] = await prisma.$queryRaw<Array<{ count: number }>>`
|
|
SELECT count(*)::int AS count
|
|
FROM _prisma_migrations
|
|
WHERE finished_at IS NULL AND rolled_back_at IS NULL
|
|
`;
|
|
out.lastMigration = last?.migration_name ?? null;
|
|
out.lastMigrationAt = iso(last?.finished_at);
|
|
out.unfinishedMigrations = unfinished?.count ?? null;
|
|
} catch {
|
|
/* no migrations table: reported as unknown */
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function readNewestFinishedJobAt(): Promise<string | null> {
|
|
try {
|
|
const [row] = await prisma.$queryRaw<Array<{ at: Date | null }>>`
|
|
SELECT max(completed_on) AS at FROM pgboss.job WHERE state = 'completed'
|
|
`;
|
|
return iso(row?.at);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** Read everything the report shows. Accounts leave here as numbers only. */
|
|
export async function collectSupportReportFacts(
|
|
now: Date = new Date(),
|
|
): Promise<{ facts: SupportReportFacts; context: SupportRedactionContext }> {
|
|
const users = await prisma.user.findMany({
|
|
orderBy: [{ createdAt: "asc" }, { id: "asc" }],
|
|
select: {
|
|
id: true,
|
|
username: true,
|
|
email: true,
|
|
fullName: true,
|
|
aiUsageLevel: true,
|
|
},
|
|
});
|
|
const accountOf = new Map<string, number>();
|
|
users.forEach((u, i) => accountOf.set(u.id, i + 1));
|
|
const knownValues: string[] = [];
|
|
for (const u of users) {
|
|
knownValues.push(u.id, u.username);
|
|
if (u.email) knownValues.push(u.email);
|
|
if (u.fullName) knownValues.push(u.fullName);
|
|
}
|
|
|
|
const appSettings = await prisma.appSettings.findUnique({
|
|
where: { id: "singleton" },
|
|
select: {
|
|
glitchtipEnabled: true,
|
|
glitchtipDsn: true,
|
|
webPushVapidPublicKey: true,
|
|
webPushVapidPrivateKeyEncrypted: true,
|
|
adminAiKeyEncrypted: true,
|
|
adminCodexConnectionStatus: true,
|
|
aiUsageDefaultLevel: true,
|
|
aiUsageMaxLevel: true,
|
|
},
|
|
});
|
|
|
|
const providerTypes = await prisma.user.groupBy({
|
|
by: ["aiProvider"],
|
|
_count: { _all: true },
|
|
});
|
|
|
|
const config: Array<[string, string]> = [
|
|
["Secure cookie flag", yesNo(shouldEmitSecureCookie())],
|
|
["OIDC sign-in", yesNo(isOidcConfigured())],
|
|
["OIDC only", yesNo(isOidcOnly())],
|
|
["Loki", yesNo(Boolean(getLoggingConfig().lokiEndpoint))],
|
|
[
|
|
"GlitchTip",
|
|
yesNo(Boolean(appSettings?.glitchtipEnabled && appSettings.glitchtipDsn)),
|
|
],
|
|
["Off-host backup", yesNo(offhostBackupConfigured())],
|
|
[
|
|
"APNs",
|
|
yesNo(
|
|
envSet("APNS_KEY_ID") &&
|
|
envSet("APNS_TEAM_ID") &&
|
|
envSet("APNS_BUNDLE_ID") &&
|
|
(envSet("APNS_KEY_B64") ||
|
|
envSet("APNS_KEY") ||
|
|
envSet("APNS_KEY_FILE")),
|
|
),
|
|
],
|
|
[
|
|
"Web Push (VAPID)",
|
|
yesNo(
|
|
Boolean(
|
|
appSettings?.webPushVapidPublicKey &&
|
|
appSettings.webPushVapidPrivateKeyEncrypted,
|
|
) ||
|
|
(envSet("VAPID_PUBLIC_KEY") && envSet("VAPID_PRIVATE_KEY")),
|
|
),
|
|
],
|
|
["SMTP", yesNo(isEmailConfigured())],
|
|
["GeoLite2 databases", yesNo(offlineGeoReady())],
|
|
[
|
|
"NOTIFICATION_PRIVATE_ORIGINS entries",
|
|
String(configuredNotificationPrivateOrigins().size),
|
|
],
|
|
["Operator AI key", yesNo(Boolean(appSettings?.adminAiKeyEncrypted))],
|
|
[
|
|
"Central Codex connected",
|
|
yesNo(appSettings?.adminCodexConnectionStatus === "connected"),
|
|
],
|
|
[
|
|
"AI provider types chosen by accounts",
|
|
providerTypes
|
|
.filter((row) => row.aiProvider)
|
|
.map((row) => `${supportLine(row.aiProvider, 30)} ${row._count._all}`)
|
|
.sort()
|
|
.join(", ") || "none",
|
|
],
|
|
["Demo mode", yesNo(process.env.DEMO_MODE === "true")],
|
|
];
|
|
|
|
const workerStatus = getWorkerStatus();
|
|
const [database, failing, newestFinishedJobAt] = await Promise.all([
|
|
readDatabaseFacts(),
|
|
readFailingQueues(JOB_FAILURE_WINDOW_HOURS),
|
|
readNewestFinishedJobAt(),
|
|
]);
|
|
|
|
// Connections: one health read per account, capped.
|
|
const readUsers = users.slice(0, SUPPORT_REPORT_ACCOUNT_CAP);
|
|
const connectionGroups = new Map<
|
|
string,
|
|
{ provider: string; state: string; reason: string; accounts: number[] }
|
|
>();
|
|
for (const user of readUsers) {
|
|
let health;
|
|
try {
|
|
health = await loadConnectionHealth(user.id);
|
|
} catch {
|
|
continue;
|
|
}
|
|
for (const key of CONNECTION_KEYS) {
|
|
const h = health[key];
|
|
if (!h) continue;
|
|
const groupKey = `${key}|${h.state}|${h.reason}`;
|
|
const group = connectionGroups.get(groupKey) ?? {
|
|
provider: key,
|
|
state: h.state,
|
|
reason: h.reason,
|
|
accounts: [],
|
|
};
|
|
group.accounts.push(accountOf.get(user.id) ?? 0);
|
|
connectionGroups.set(groupKey, group);
|
|
}
|
|
}
|
|
|
|
const incidentRows = await prisma.integrationWatch.findMany({
|
|
where: { incidentReason: { not: null } },
|
|
select: {
|
|
userId: true,
|
|
integration: true,
|
|
incidentReason: true,
|
|
incidentOpenedAt: true,
|
|
alertedAt: true,
|
|
},
|
|
orderBy: { incidentOpenedAt: "desc" },
|
|
take: 100,
|
|
});
|
|
|
|
const providerRows = await prisma.providerHealth.findMany({
|
|
select: {
|
|
providerType: true,
|
|
lastResult: true,
|
|
lastStatus: true,
|
|
lastFailureAt: true,
|
|
nextRetryAt: true,
|
|
},
|
|
});
|
|
const providers = new Map<
|
|
string,
|
|
SupportReportFacts["ai"]["providers"][number]
|
|
>();
|
|
for (const row of providerRows) {
|
|
const p = providers.get(row.providerType) ?? {
|
|
provider: row.providerType,
|
|
ok: 0,
|
|
failing: 0,
|
|
authFailed: 0,
|
|
paused: 0,
|
|
lastFailureAt: null,
|
|
lastStatus: null,
|
|
};
|
|
if (row.lastResult === "ok") p.ok += 1;
|
|
else if (row.lastResult === "auth_failed") p.authFailed += 1;
|
|
else p.failing += 1;
|
|
if (row.nextRetryAt && row.nextRetryAt.getTime() > now.getTime()) {
|
|
p.paused += 1;
|
|
}
|
|
const failedAt = iso(row.lastFailureAt);
|
|
if (failedAt && (!p.lastFailureAt || failedAt > p.lastFailureAt)) {
|
|
p.lastFailureAt = failedAt;
|
|
p.lastStatus = row.lastStatus;
|
|
}
|
|
providers.set(row.providerType, p);
|
|
}
|
|
|
|
const levelCounts = new Map<string, number>();
|
|
for (const u of users) {
|
|
const level = u.aiUsageLevel ?? "instance default";
|
|
levelCounts.set(level, (levelCounts.get(level) ?? 0) + 1);
|
|
}
|
|
|
|
const since = new Date(
|
|
now.getTime() - SUPPORT_REPORT_NOTIFICATION_DAYS * 24 * 60 * 60 * 1000,
|
|
);
|
|
const [attemptCounts, reasonCounts] = await Promise.all([
|
|
prisma.pushAttempt.groupBy({
|
|
by: ["channel", "result"],
|
|
where: { createdAt: { gte: since } },
|
|
_count: { _all: true },
|
|
}),
|
|
prisma.pushAttempt.groupBy({
|
|
by: ["channel", "reason"],
|
|
where: { createdAt: { gte: since }, result: { not: "ok" } },
|
|
_count: { _all: true },
|
|
}),
|
|
]);
|
|
const channels = new Map<
|
|
string,
|
|
SupportReportFacts["notifications"][number]
|
|
>();
|
|
const channelOf = (channel: string) => {
|
|
const c = channels.get(channel) ?? {
|
|
channel,
|
|
ok: 0,
|
|
error: 0,
|
|
skipped: 0,
|
|
topReasons: [],
|
|
};
|
|
channels.set(channel, c);
|
|
return c;
|
|
};
|
|
for (const row of attemptCounts) {
|
|
const c = channelOf(row.channel);
|
|
if (row.result === "ok") c.ok += row._count._all;
|
|
else if (row.result === "error") c.error += row._count._all;
|
|
else c.skipped += row._count._all;
|
|
}
|
|
for (const row of reasonCounts) {
|
|
channelOf(row.channel).topReasons.push([
|
|
supportLine(row.reason ?? "unspecified", 60),
|
|
row._count._all,
|
|
]);
|
|
}
|
|
for (const c of channels.values()) {
|
|
c.topReasons = c.topReasons.sort((a, b) => b[1] - a[1]).slice(0, 3);
|
|
}
|
|
|
|
const [
|
|
localAggregate,
|
|
localCopies,
|
|
offhostRows,
|
|
lastLocalRun,
|
|
lastOffhostRun,
|
|
lastDrill,
|
|
] = await Promise.all([
|
|
prisma.dataBackup.aggregate({ _max: { createdAt: true } }),
|
|
prisma.dataBackup.count(),
|
|
prisma.offhostBackupState.findMany({ select: { lastSuccessAt: true } }),
|
|
readLastQueueRun(DATA_BACKUP_QUEUE),
|
|
readLastQueueRun(OFFHOST_BACKUP_QUEUE),
|
|
readLastQueueRun(RESTORE_DRILL_QUEUE),
|
|
]);
|
|
const offhostDates = offhostRows
|
|
.map((r) => r.lastSuccessAt)
|
|
.filter((d): d is Date => d !== null)
|
|
.map((d) => d.toISOString())
|
|
.sort();
|
|
|
|
const problemEvents = readLogBuffer({ ring: "problems" });
|
|
|
|
const facts: SupportReportFacts = {
|
|
generatedAt: now.toISOString(),
|
|
build: {
|
|
version:
|
|
process.env.NEXT_PUBLIC_APP_VERSION?.trim() || packageJson.version,
|
|
buildSha: process.env.NEXT_PUBLIC_APP_BUILD_SHA?.trim() || null,
|
|
builtAt: process.env.NEXT_PUBLIC_APP_BUILT_AT?.trim() || null,
|
|
node: process.version,
|
|
processType: getProcessType(),
|
|
heapLimitMb: heapLimitMb(),
|
|
uptimeSeconds: Math.round(process.uptime()),
|
|
},
|
|
database,
|
|
config,
|
|
jobs: {
|
|
workerInThisProcess: shouldRunWorker(),
|
|
workerRunning: workerStatus.running,
|
|
workerLastHeartbeat: workerStatus.lastHeartbeat,
|
|
newestFinishedJobAt,
|
|
failing:
|
|
failing === null
|
|
? null
|
|
: failing.map((q) => ({
|
|
queue: q.queue,
|
|
failures: q.failures,
|
|
lastFailedAt: q.lastFailedAt,
|
|
errorClass: errorClassOf(q.lastError),
|
|
})),
|
|
},
|
|
accounts: { total: users.length, read: readUsers.length },
|
|
connections: [...connectionGroups.values()].sort(
|
|
(a, b) =>
|
|
a.provider.localeCompare(b.provider) ||
|
|
a.state.localeCompare(b.state) ||
|
|
a.reason.localeCompare(b.reason),
|
|
),
|
|
incidents: incidentRows.map((row) => ({
|
|
provider: row.integration,
|
|
reason: row.incidentReason ?? "",
|
|
openedAt: iso(row.incidentOpenedAt),
|
|
announced: row.alertedAt !== null,
|
|
account: accountOf.get(row.userId) ?? null,
|
|
})),
|
|
ai: {
|
|
defaultLevel: appSettings?.aiUsageDefaultLevel ?? null,
|
|
maxLevel: appSettings?.aiUsageMaxLevel ?? null,
|
|
levels: [...levelCounts.entries()].sort((a, b) =>
|
|
a[0].localeCompare(b[0]),
|
|
),
|
|
providers: [...providers.values()].sort((a, b) =>
|
|
a.provider.localeCompare(b.provider),
|
|
),
|
|
},
|
|
notifications: [...channels.values()].sort((a, b) =>
|
|
a.channel.localeCompare(b.channel),
|
|
),
|
|
backups: {
|
|
newestLocalAt: iso(localAggregate._max.createdAt),
|
|
localCopies,
|
|
lastLocalRun: runFact(lastLocalRun),
|
|
offhostConfigured: offhostBackupConfigured(),
|
|
offhostNewestAt: offhostDates.at(-1) ?? null,
|
|
offhostOldestAt: offhostDates[0] ?? null,
|
|
offhostAccounts: offhostRows.length,
|
|
offhostNeverStored: offhostRows.length - offhostDates.length,
|
|
lastOffhostRun: runFact(lastOffhostRun),
|
|
lastRestoreDrill: runFact(lastDrill),
|
|
},
|
|
errors: {
|
|
ringSize: problemEvents.length,
|
|
ringMax: PROBLEMS_BUFFER_MAX,
|
|
top: summariseProblemEvents(problemEvents),
|
|
},
|
|
};
|
|
|
|
return { facts, context: { knownValues } };
|
|
}
|
|
|
|
// ─── Render ──────────────────────────────────────────────────────────────
|
|
|
|
function or(value: string | number | null | undefined, fallback = "unknown") {
|
|
return value === null || value === undefined || value === ""
|
|
? fallback
|
|
: String(value);
|
|
}
|
|
|
|
function sizeLine(bytes: number | null): string {
|
|
return bytes === null ? "unknown" : `${megabytes(bytes)} MB`;
|
|
}
|
|
|
|
function runLine(run: QueueRunFact | null): string {
|
|
if (!run) return "no run on record";
|
|
return run.errorClass
|
|
? `${run.state} at ${run.at} (${run.errorClass})`
|
|
: `${run.state} at ${run.at}`;
|
|
}
|
|
|
|
function accountList(accounts: readonly number[]): string {
|
|
const sorted = [...accounts].sort((a, b) => a - b);
|
|
const shown = sorted.slice(0, 10).map((n) => `Account ${n}`);
|
|
const more = sorted.length - shown.length;
|
|
return more > 0 ? `${shown.join(", ")} and ${more} more` : shown.join(", ");
|
|
}
|
|
|
|
/** The report as plain text, before redaction. */
|
|
export function renderSupportReport(facts: SupportReportFacts): string {
|
|
const lines: string[] = [];
|
|
const section = (title: string) => {
|
|
if (lines.length > 0) lines.push("");
|
|
lines.push(`## ${title}`);
|
|
};
|
|
const row = (label: string, value: string | number | null) =>
|
|
lines.push(`- ${label}: ${or(value)}`);
|
|
|
|
lines.push("# HealthLog support report");
|
|
lines.push(`Generated: ${facts.generatedAt}`);
|
|
|
|
section("Build");
|
|
row("Version", facts.build.version);
|
|
row("Build SHA", facts.build.buildSha);
|
|
row("Built at", facts.build.builtAt);
|
|
row("Node", facts.build.node);
|
|
row("Process type", facts.build.processType);
|
|
row("Heap limit", `${facts.build.heapLimitMb} MB`);
|
|
if (isHeapLimitLow(facts.build.heapLimitMb)) {
|
|
row("Heap limit note", lowHeapLimitAdvice(facts.build.heapLimitMb));
|
|
}
|
|
row(
|
|
"Uptime",
|
|
`${Math.floor(facts.build.uptimeSeconds / 3600)} h ${Math.floor((facts.build.uptimeSeconds % 3600) / 60)} min`,
|
|
);
|
|
|
|
section("Database");
|
|
row("PostgreSQL", facts.database.version);
|
|
row("Server time zone", facts.database.timezone);
|
|
row("Size", sizeLine(facts.database.sizeBytes));
|
|
row(
|
|
"Last migration",
|
|
facts.database.lastMigration
|
|
? `${facts.database.lastMigration} (${or(facts.database.lastMigrationAt)})`
|
|
: null,
|
|
);
|
|
row("Unfinished migrations", facts.database.unfinishedMigrations);
|
|
|
|
section("Configuration");
|
|
for (const [label, value] of facts.config) row(label, value);
|
|
|
|
section("Jobs");
|
|
row("Worker in this process", yesNo(facts.jobs.workerInThisProcess));
|
|
if (facts.jobs.workerInThisProcess) {
|
|
row("Worker running", yesNo(facts.jobs.workerRunning));
|
|
row("Worker last heartbeat", facts.jobs.workerLastHeartbeat);
|
|
}
|
|
row("Newest finished job", facts.jobs.newestFinishedJobAt);
|
|
if (facts.jobs.failing === null) {
|
|
row(
|
|
`Failed jobs (${JOB_FAILURE_WINDOW_HOURS} h)`,
|
|
"queue tables not readable",
|
|
);
|
|
} else if (facts.jobs.failing.length === 0) {
|
|
row(`Failed jobs (${JOB_FAILURE_WINDOW_HOURS} h)`, "none");
|
|
} else {
|
|
lines.push(`- Failed jobs (${JOB_FAILURE_WINDOW_HOURS} h):`);
|
|
for (const q of facts.jobs.failing) {
|
|
lines.push(
|
|
` - ${supportLine(q.queue, 60)}: ${q.failures} failed, last ${q.lastFailedAt} (${q.errorClass})`,
|
|
);
|
|
}
|
|
}
|
|
|
|
section("Connections");
|
|
row(
|
|
"Accounts",
|
|
facts.accounts.read < facts.accounts.total
|
|
? `${facts.accounts.total} (connections read for the oldest ${facts.accounts.read})`
|
|
: facts.accounts.total,
|
|
);
|
|
if (facts.connections.length === 0) {
|
|
row("Connected sources", "none");
|
|
} else {
|
|
for (const c of facts.connections) {
|
|
const who = c.state === "ok" ? "" : ` (${accountList(c.accounts)})`;
|
|
lines.push(
|
|
`- ${c.provider}: ${c.state} / ${c.reason}: ${c.accounts.length}${who}`,
|
|
);
|
|
}
|
|
}
|
|
if (facts.incidents.length === 0) {
|
|
row("Open watchdog incidents", "none");
|
|
} else {
|
|
lines.push("- Open watchdog incidents:");
|
|
for (const i of facts.incidents) {
|
|
lines.push(
|
|
` - ${supportLine(i.provider, 40)}: ${supportLine(i.reason, 40)} since ${or(i.openedAt)}, ${i.announced ? "announced" : "not announced"}${i.account ? `, Account ${i.account}` : ""}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
section("AI");
|
|
row("Usage level default", facts.ai.defaultLevel);
|
|
row("Usage level ceiling", facts.ai.maxLevel);
|
|
row(
|
|
"Accounts by usage level",
|
|
facts.ai.levels.map(([level, n]) => `${level} ${n}`).join(", ") || "none",
|
|
);
|
|
if (facts.ai.providers.length === 0) {
|
|
row("Provider ledger", "empty");
|
|
} else {
|
|
lines.push("- Provider ledger (accounts):");
|
|
for (const p of facts.ai.providers) {
|
|
const last = p.lastFailureAt
|
|
? `, last failure ${p.lastFailureAt}${p.lastStatus ? ` (HTTP ${p.lastStatus})` : ""}`
|
|
: "";
|
|
lines.push(
|
|
` - ${supportLine(p.provider, 30)}: ok ${p.ok}, failing ${p.failing}, credential refused ${p.authFailed}, paused ${p.paused}${last}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
section(`Notifications (${SUPPORT_REPORT_NOTIFICATION_DAYS} days)`);
|
|
if (facts.notifications.length === 0) {
|
|
row("Delivery attempts", "none");
|
|
} else {
|
|
for (const c of facts.notifications) {
|
|
const reasons =
|
|
c.topReasons.length > 0
|
|
? `; ${c.topReasons.map(([r, n]) => `${r} ${n}`).join(", ")}`
|
|
: "";
|
|
lines.push(
|
|
`- ${supportLine(c.channel, 20)}: ok ${c.ok}, failed ${c.error}, skipped ${c.skipped}${reasons}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
section("Backups");
|
|
row("Newest in-database copy", facts.backups.newestLocalAt ?? "none");
|
|
row("In-database copies", facts.backups.localCopies);
|
|
row("Last weekly run", runLine(facts.backups.lastLocalRun));
|
|
row("Off-host configured", yesNo(facts.backups.offhostConfigured));
|
|
if (facts.backups.offhostAccounts > 0) {
|
|
row("Off-host newest", facts.backups.offhostNewestAt ?? "none");
|
|
row(
|
|
"Off-host oldest account copy",
|
|
facts.backups.offhostOldestAt ?? "none",
|
|
);
|
|
row(
|
|
"Off-host accounts never stored",
|
|
`${facts.backups.offhostNeverStored} of ${facts.backups.offhostAccounts}`,
|
|
);
|
|
}
|
|
row("Last off-host run", runLine(facts.backups.lastOffhostRun));
|
|
row("Last restore drill", runLine(facts.backups.lastRestoreDrill));
|
|
|
|
section("Recent warnings and errors (this process)");
|
|
row(
|
|
"Problems ring",
|
|
`${facts.errors.ringSize} of ${facts.errors.ringMax} entries`,
|
|
);
|
|
if (facts.errors.top.length === 0) {
|
|
row("Top actions", "none");
|
|
} else {
|
|
for (const e of facts.errors.top) {
|
|
lines.push(
|
|
`- ${e.level} ${e.action}: ${e.count}x, last ${e.lastSeen}, ${e.errorClass}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
return `${lines.join("\n")}\n`;
|
|
}
|
|
|
|
/** Facts to finished, redacted text. */
|
|
export function buildSupportReportText(
|
|
facts: SupportReportFacts,
|
|
context: SupportRedactionContext,
|
|
): string {
|
|
return redactSupportText(renderSupportReport(facts), context);
|
|
}
|