fix(status): expose background failures

This commit is contained in:
Marc-André Bombeck committed 2026-07-28 14:00:44 +02:00
1 parent a689329e79
commit 8ece6f39cc
31 files changed
+854 -44

No files matched your search

@@ -36,7 +36,11 @@ vi.mock("@/lib/auth/audit", () => ({
}));
vi.mock("@/lib/notifications/dispatcher", () => ({
dispatchNotification: vi.fn().mockResolvedValue(undefined),
dispatchNotification: vi.fn().mockResolvedValue({
dispatched: true,
channelsAttempted: 1,
channelsSucceeded: 1,
}),
}));
vi.mock("@/lib/crypto", () => ({
+105 -6
View File
@@ -9,6 +9,15 @@
*/
import { describe, it, expect, vi, beforeEach } from "vitest";
const { addWarningMock } = vi.hoisted(() => ({
addWarningMock: vi.fn(),
}));
vi.mock("@/lib/logging/context", () => ({
annotate: vi.fn(),
getEvent: () => ({ addWarning: addWarningMock }),
}));
vi.mock("@/lib/db", () => ({
prisma: {
integrationStatus: {
@@ -28,7 +37,7 @@ vi.mock("@/lib/auth/audit", () => ({
}));
vi.mock("@/lib/notifications/dispatcher", () => ({
dispatchNotification: vi.fn().mockResolvedValue(undefined),
dispatchNotification: vi.fn(),
}));
vi.mock("@/lib/crypto", () => ({
@@ -56,6 +65,11 @@ import { dispatchNotification } from "@/lib/notifications/dispatcher";
beforeEach(() => {
vi.resetAllMocks();
vi.mocked(dispatchNotification).mockResolvedValue({
dispatched: true,
channelsAttempted: 1,
channelsSucceeded: 1,
});
delete process.env.INTEGRATION_FAILURE_ALERT_THRESHOLD;
});
@@ -207,6 +221,90 @@ describe("recordSyncFailure — at threshold", () => {
);
});
it("does not stamp alertedAt when no admin channel is available", async () => {
vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({
consecutiveFailuresByKind: {
transient: 2,
reauth_required: 0,
persistent: 0,
},
persistentFailureStartedAt: null,
alertedAt: null,
} as never);
vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce({
alertedAt: null,
} as never);
vi.mocked(prisma.user.findUnique).mockResolvedValueOnce({
email: "user@example.com",
} as never);
vi.mocked(prisma.user.findMany).mockResolvedValueOnce([
{ id: "admin-1" },
] as never);
vi.mocked(dispatchNotification).mockResolvedValueOnce({
dispatched: false,
channelsAttempted: 0,
channelsSucceeded: 0,
});
await recordSyncFailure({
userId: "u1",
integration: "withings",
kind: "transient",
message: "Withings refresh error: 503 - upstream service down",
errorCode: "503",
});
expect(dispatchNotification).toHaveBeenCalledOnce();
expect(prisma.integrationStatus.update).not.toHaveBeenCalled();
expect(addWarningMock).toHaveBeenCalledWith(
"No admin notification channel available for withings failure alert",
);
});
it("stamps alertedAt after an attempted delivery that failed", async () => {
vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({
consecutiveFailuresByKind: {
transient: 2,
reauth_required: 0,
persistent: 0,
},
persistentFailureStartedAt: null,
alertedAt: null,
} as never);
vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce({
alertedAt: null,
} as never);
vi.mocked(prisma.user.findUnique).mockResolvedValueOnce({
email: "user@example.com",
} as never);
vi.mocked(prisma.user.findMany).mockResolvedValueOnce([
{ id: "admin-1" },
] as never);
vi.mocked(dispatchNotification).mockResolvedValueOnce({
dispatched: false,
channelsAttempted: 1,
channelsSucceeded: 0,
});
vi.mocked(prisma.integrationStatus.update).mockResolvedValueOnce(
{} as never,
);
await recordSyncFailure({
userId: "u1",
integration: "withings",
kind: "transient",
message: "Withings refresh error: 503 - upstream service down",
errorCode: "503",
});
expect(prisma.integrationStatus.update).toHaveBeenCalledWith({
where: {
userId_integration: { userId: "u1", integration: "withings" },
},
data: { alertedAt: expect.any(Date) },
});
});
it("uses the custom threshold from INTEGRATION_FAILURE_ALERT_THRESHOLD", async () => {
process.env.INTEGRATION_FAILURE_ALERT_THRESHOLD = "5";
// 4 < 5 → should not page yet. Seed the existing bucket at 3 so
@@ -398,8 +496,8 @@ describe("recordSyncFailure — persistent (v1.4.42 W6)", () => {
});
});
describe("recordSyncFailure — admin Telegram skipped silently when no admins", () => {
it("logs a wide-event warning but does NOT throw when no admin users exist", async () => {
describe("recordSyncFailure — no admin recipients", () => {
it("surfaces the unavailable delivery and leaves alertedAt unstamped", async () => {
// Seed bucket at 2 so the in-memory increment hits the threshold
// and the no-admins branch is the one being exercised.
vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({
@@ -418,9 +516,6 @@ describe("recordSyncFailure — admin Telegram skipped silently when no admins",
email: "u@example.com",
} as never);
vi.mocked(prisma.user.findMany).mockResolvedValueOnce([] as never);
vi.mocked(prisma.integrationStatus.update).mockResolvedValueOnce(
{} as never,
);
await expect(
recordSyncFailure({
@@ -432,6 +527,10 @@ describe("recordSyncFailure — admin Telegram skipped silently when no admins",
).resolves.toBeUndefined();
expect(dispatchNotification).not.toHaveBeenCalled();
expect(prisma.integrationStatus.update).not.toHaveBeenCalled();
expect(addWarningMock).toHaveBeenCalledWith(
"No admin notification channel available for withings failure alert",
);
});
});
+46 -24
View File
@@ -35,7 +35,10 @@ import { prisma } from "@/lib/db";
import { encrypt, decrypt } from "@/lib/crypto";
import { auditLog } from "@/lib/auth/audit";
import { annotate, getEvent } from "@/lib/logging/context";
import { dispatchNotification } from "@/lib/notifications/dispatcher";
import {
dispatchNotification,
type DispatchOutcome,
} from "@/lib/notifications/dispatcher";
import type { IntegrationClassification } from "@/lib/integrations/http-status-classifier";
export type IntegrationKey =
@@ -755,7 +758,7 @@ export async function recordSyncFailure(
row.alertedAt &&
now.getTime() - row.alertedAt.getTime() < ALERT_REPEAT_WINDOW_MS;
if (!previouslyAlerted) {
await maybeAlertAdmins({
const outcome = await maybeAlertAdmins({
userId,
integration,
kind: effectiveKind,
@@ -766,14 +769,25 @@ export async function recordSyncFailure(
getEvent()?.addWarning(
`Admin alert dispatch failed: ${err instanceof Error ? err.message : String(err)}`,
);
return null;
});
// Stamp the alert window even if dispatch failed — better to
// miss one notification than to flood admins on a backend
// outage.
await prisma.integrationStatus.update({
where: { userId_integration: { userId, integration } },
data: { alertedAt: now },
});
if (outcome?.channelsAttempted) {
// An attempted channel owns its own retry/cooldown policy. Stamp the
// alert window even when every attempt failed so a backend outage
// cannot turn this failure streak into an hourly notification flood.
await prisma.integrationStatus.update({
where: { userId_integration: { userId, integration } },
data: { alertedAt: now },
});
} else if (outcome) {
// No recipient channel accepted an attempt. Leaving alertedAt empty
// keeps the missing delivery visible instead of recording a page that
// never left the process.
getEvent()?.addWarning(
`No admin notification channel available for ${integration} failure alert`,
);
}
}
}
@@ -1203,18 +1217,15 @@ export function formatAdminAlertPayload(input: AlertInput): {
}
/**
* Page admins on Telegram via the existing dispatcher. We do NOT add
* a new sender, channel type, or retry path — B3 owns that surface.
* The dispatcher is opt-in per-event and per-channel, so an admin
* who has not configured Telegram simply gets no message (silent
* fall-through is the documented behaviour).
* Page admins through the existing dispatcher. We do NOT add a new sender,
* channel type, or retry path. The dispatcher is opt-in per event and channel,
* and its outcome is the authority on whether a delivery was attempted.
*
* The notification is sent to EACH admin user with a Telegram channel
* configured. We resolve recipients once per failure burst (gated by
* `alertedAt` upstream) so a deployment with a single admin doesn't
* pay for a query per user.
* The notification is sent to each admin user. The returned outcome aggregates
* the dispatcher's real per-admin outcomes so one attempted delivery cannot be
* hidden by another admin having no enabled channel.
*/
async function maybeAlertAdmins(input: AlertInput): Promise<void> {
async function maybeAlertAdmins(input: AlertInput): Promise<DispatchOutcome> {
const subject = await prisma.user.findUnique({
where: { id: input.userId },
select: { email: true },
@@ -1226,24 +1237,35 @@ async function maybeAlertAdmins(input: AlertInput): Promise<void> {
});
if (admins.length === 0) {
getEvent()?.addWarning(
`No admin user found to alert about persistent ${input.integration} failure for user ${input.userId}`,
);
return;
return {
dispatched: false,
channelsAttempted: 0,
channelsSucceeded: 0,
};
}
const payload = formatAdminAlertPayload({
...input,
subjectLabel: subject?.email ?? input.userId,
});
const outcome: DispatchOutcome = {
dispatched: false,
channelsAttempted: 0,
channelsSucceeded: 0,
};
for (const admin of admins) {
await dispatchNotification({
const adminOutcome = await dispatchNotification({
eventType: "SYSTEM_ALERT",
userId: admin.id,
title: payload.title,
message: payload.message,
metadata: payload.metadata,
});
outcome.dispatched ||= adminOutcome.dispatched;
outcome.channelsAttempted += adminOutcome.channelsAttempted;
outcome.channelsSucceeded += adminOutcome.channelsSucceeded;
}
return outcome;
}