feat(integrations): admin Telegram alert on persistent sync failure (>=3 consecutive)

v1.4.15 Phase B2, criterion 3. The status helper introduced in commit
4db72a8 already trips the dispatcher when the consecutive-failure
counter crosses INTEGRATION_FAILURE_ALERT_THRESHOLD (default 3) and
the 24h re-alert window has lapsed. This commit:

  1. Extracts the message formatter into a pure exported function
     formatAdminAlertPayload() so the message shape is unit-testable
     without standing up Prisma + the dispatcher. Same byte-identical
     output, refactor only.
  2. Locks the alert payload contract via 7 focused unit tests:
       - integration display name uses "Withings" / "moodLog"
         (not the lowercase storage key)
       - subjectLabel falls back to userId when no email/label
       - reauth_required maps to "ask the user to reconnect"
       - transient maps to "investigate the upstream service"
       - 4 KB stack traces are trimmed to 280 chars + ellipsis
         (Telegram cap is 4096; envelope eats ~150 so 280 keeps margin)
       - errorCode renders as "(503)" suffix on the reason line, or
         is omitted entirely when undefined (no "(undefined)" leak)
       - metadata block shapes affectedUserId / integration /
         consecutiveFailures / errorCode (null when absent)

The dispatcher itself is reused as-is — no new sender, no new channel
type, no new retry path. B3 owns the dispatcher reliability surface;
we are exclusively a caller. Admins who haven't configured Telegram
get silently skipped (the dispatcher's documented behaviour).

The 24h re-alert window prevents a flapping integration from paging
on every failure: alertedAt is stamped on the trip, cleared by the
next success, and re-trips only after 24h on the same streak. The
flap-resistance is covered by status.test.ts (commit 4db72a8) and
the integration-status.test.ts case landing in the next commit.

Tests: +7 unit (formatAdminAlertPayload) on top of the +16 already
guarding the threshold/window state machine. 116 / 890 unit pass.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Marc-André BombeckandClaude Opus 4.7 committed 2026-05-09 21:07:23 +02:00
1 parent 65a35904de
commit 604dff057a
2 files changed
+206 -27

No files matched your search

@@ -0,0 +1,152 @@
/**
* v1.4.15 Phase B2, criterion 3 — admin Telegram alert formatting.
*
* The state-machine threshold tests live in `status.test.ts` (when
* does the alert fire?). This file proves WHAT the alert looks like
* — the message body Marc sees on Telegram when an integration
* starts crashing for one of his users.
*
* The formatter is a pure function so we can test deterministically
* without touching Prisma or the dispatcher.
*/
import { describe, it, expect } from "vitest";
import { formatAdminAlertPayload } from "../status";
describe("formatAdminAlertPayload — Withings re-auth", () => {
it("uses the integration display name (Withings, not 'withings')", () => {
const out = formatAdminAlertPayload({
userId: "u-1",
integration: "withings",
kind: "reauth_required",
message: "Withings refresh error: 100 - invalid_grant",
errorCode: "100",
consecutiveFailures: 3,
subjectLabel: "user@example.com",
});
expect(out.title).toBe("Withings sync failing for user@example.com");
expect(out.message).toContain("Withings sync has failed 3 times in a row");
expect(out.message).toContain("re-auth required");
expect(out.message).toContain("(100)");
expect(out.message).toContain(
"Action: ask the user to reconnect the integration.",
);
});
it("falls back to userId when no email/subjectLabel provided", () => {
const out = formatAdminAlertPayload({
userId: "user-no-email",
integration: "withings",
kind: "transient",
message: "Withings refresh error: 503 - upstream",
errorCode: "503",
consecutiveFailures: 5,
});
expect(out.title).toBe("Withings sync failing for user-no-email");
});
});
describe("formatAdminAlertPayload — moodLog transient", () => {
it("uses 'moodLog' display name and the 'investigate the upstream service.' action", () => {
const out = formatAdminAlertPayload({
userId: "u-2",
integration: "moodlog",
kind: "transient",
message: "moodLog sync HTTP 502",
errorCode: "http_502",
consecutiveFailures: 4,
subjectLabel: "marc@example.com",
});
expect(out.title).toBe("moodLog sync failing for marc@example.com");
expect(out.message).toContain(
"Action: investigate the upstream service.",
);
expect(out.message).toContain("(http_502)");
});
});
describe("formatAdminAlertPayload — message trimming", () => {
it("trims a 4 KB stack trace down to 280 chars + ellipsis", () => {
const longMessage = "stack: " + "x".repeat(4000);
const out = formatAdminAlertPayload({
userId: "u-3",
integration: "withings",
kind: "transient",
message: longMessage,
errorCode: undefined,
consecutiveFailures: 3,
subjectLabel: "user@example.com",
});
// Body line includes the trimmed message — full 4 KB MUST NOT
// appear, but the start of the message MUST.
expect(out.message).toContain("stack: ");
expect(out.message).not.toContain("x".repeat(280));
expect(out.message).toContain("...");
// Whole message under 4096 (Telegram cap).
expect(out.message.length).toBeLessThan(4096);
});
it("does NOT add an ellipsis when message is under the trim threshold", () => {
const out = formatAdminAlertPayload({
userId: "u-4",
integration: "withings",
kind: "transient",
message: "short error",
errorCode: undefined,
consecutiveFailures: 3,
subjectLabel: "user@example.com",
});
expect(out.message).toContain("short error");
expect(out.message).not.toMatch(/short error\.\.\./);
});
});
describe("formatAdminAlertPayload — metadata payload", () => {
it("returns the metadata block with affectedUserId, consecutiveFailures, errorCode (or null)", () => {
const withCode = formatAdminAlertPayload({
userId: "u-5",
integration: "withings",
kind: "transient",
message: "503",
errorCode: "503",
consecutiveFailures: 3,
});
expect(withCode.metadata).toEqual({
integration: "withings",
affectedUserId: "u-5",
consecutiveFailures: 3,
errorCode: "503",
});
const withoutCode = formatAdminAlertPayload({
userId: "u-6",
integration: "moodlog",
kind: "transient",
message: "blip",
errorCode: undefined,
consecutiveFailures: 7,
});
expect(withoutCode.metadata).toEqual({
integration: "moodlog",
affectedUserId: "u-6",
consecutiveFailures: 7,
errorCode: null,
});
});
});
describe("formatAdminAlertPayload — omits errorCode parens when undefined", () => {
it("does not render '(undefined)' when no errorCode is supplied", () => {
const out = formatAdminAlertPayload({
userId: "u-7",
integration: "withings",
kind: "transient",
message: "network reset",
errorCode: undefined,
consecutiveFailures: 3,
subjectLabel: "user@example.com",
});
expect(out.message).not.toContain("(undefined)");
expect(out.message).toContain("transient error — network reset");
});
});
+54 -27
View File
@@ -382,13 +382,59 @@ function safeDecryptError(ciphertext: string): string {
}
}
interface AlertInput {
export interface AlertInput {
userId: string;
integration: IntegrationKey;
kind: FailureKind;
message: string;
errorCode: string | undefined;
consecutiveFailures: number;
/** Caller-resolved subject label (usually the user's email). */
subjectLabel?: string;
}
/**
* Pure formatter for the admin-Telegram payload — extracted so we
* can unit-test the message shape without standing up Prisma. Kept
* deterministic on purpose: same input, byte-identical output.
*
* The 280-char trim on the upstream error message protects admins
* from a 4 KB stack trace landing in chat. Telegram's own cap is
* 4096 characters but our envelope (title + summary + action line)
* eats ~150 chars so we keep a comfortable margin.
*/
export function formatAdminAlertPayload(input: AlertInput): {
title: string;
message: string;
metadata: Record<string, unknown>;
} {
const integrationLabel =
input.integration === "withings" ? "Withings" : "moodLog";
const subjectLabel = input.subjectLabel ?? input.userId;
const reasonLabel =
input.kind === "reauth_required" ? "re-auth required" : "transient error";
const codeLabel = input.errorCode ? ` (${input.errorCode})` : "";
const trimmed =
input.message.length > 280
? `${input.message.slice(0, 277)}...`
: input.message;
const title = `${integrationLabel} sync failing for ${subjectLabel}`;
const message =
`${integrationLabel} sync has failed ${input.consecutiveFailures} times in a row for ${subjectLabel}.\n` +
`Last error: ${reasonLabel}${codeLabel} — ${trimmed}\n` +
`Action: ${input.kind === "reauth_required" ? "ask the user to reconnect the integration." : "investigate the upstream service."}`;
return {
title,
message,
metadata: {
integration: input.integration,
affectedUserId: input.userId,
consecutiveFailures: input.consecutiveFailures,
errorCode: input.errorCode ?? null,
},
};
}
/**
@@ -421,37 +467,18 @@ async function maybeAlertAdmins(input: AlertInput): Promise<void> {
return;
}
const integrationLabel =
input.integration === "withings" ? "Withings" : "moodLog";
const subjectLabel = subject?.email ?? input.userId;
const reasonLabel =
input.kind === "reauth_required" ? "re-auth required" : "transient error";
const codeLabel = input.errorCode ? ` (${input.errorCode})` : "";
// Trim message — Telegram caps at 4096 chars and we don't want to
// drown the admin in stack traces.
const trimmed =
input.message.length > 280
? `${input.message.slice(0, 277)}...`
: input.message;
const title = `${integrationLabel} sync failing for ${subjectLabel}`;
const message =
`${integrationLabel} sync has failed ${input.consecutiveFailures} times in a row for ${subjectLabel}.\n` +
`Last error: ${reasonLabel}${codeLabel} — ${trimmed}\n` +
`Action: ${input.kind === "reauth_required" ? "ask the user to reconnect the integration." : "investigate the upstream service."}`;
const payload = formatAdminAlertPayload({
...input,
subjectLabel: subject?.email ?? input.userId,
});
for (const admin of admins) {
await dispatchNotification({
eventType: "SYSTEM_ALERT",
userId: admin.id,
title,
message,
metadata: {
integration: input.integration,
affectedUserId: input.userId,
consecutiveFailures: input.consecutiveFailures,
errorCode: input.errorCode ?? null,
},
title: payload.title,
message: payload.message,
metadata: payload.metadata,
});
}
}