mirror of
https://github.com/MBombeck/HealthLog.git
synced 2026-11-18 08:02:12 +00:00
fix(status): expose background failures
This commit is contained in:
1 parent
a689329e79
commit
8ece6f39cc
31 files changed
+854
-44
No files matched your search
@@ -36,7 +36,11 @@ vi.mock("@/lib/auth/audit", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/notifications/dispatcher", () => ({
|
||||
dispatchNotification: vi.fn().mockResolvedValue(undefined),
|
||||
dispatchNotification: vi.fn().mockResolvedValue({
|
||||
dispatched: true,
|
||||
channelsAttempted: 1,
|
||||
channelsSucceeded: 1,
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/crypto", () => ({
|
||||
|
||||
@@ -9,6 +9,15 @@
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
|
||||
const { addWarningMock } = vi.hoisted(() => ({
|
||||
addWarningMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/logging/context", () => ({
|
||||
annotate: vi.fn(),
|
||||
getEvent: () => ({ addWarning: addWarningMock }),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({
|
||||
prisma: {
|
||||
integrationStatus: {
|
||||
@@ -28,7 +37,7 @@ vi.mock("@/lib/auth/audit", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/notifications/dispatcher", () => ({
|
||||
dispatchNotification: vi.fn().mockResolvedValue(undefined),
|
||||
dispatchNotification: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/crypto", () => ({
|
||||
@@ -56,6 +65,11 @@ import { dispatchNotification } from "@/lib/notifications/dispatcher";
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.mocked(dispatchNotification).mockResolvedValue({
|
||||
dispatched: true,
|
||||
channelsAttempted: 1,
|
||||
channelsSucceeded: 1,
|
||||
});
|
||||
delete process.env.INTEGRATION_FAILURE_ALERT_THRESHOLD;
|
||||
});
|
||||
|
||||
@@ -207,6 +221,90 @@ describe("recordSyncFailure — at threshold", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("does not stamp alertedAt when no admin channel is available", async () => {
|
||||
vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({
|
||||
consecutiveFailuresByKind: {
|
||||
transient: 2,
|
||||
reauth_required: 0,
|
||||
persistent: 0,
|
||||
},
|
||||
persistentFailureStartedAt: null,
|
||||
alertedAt: null,
|
||||
} as never);
|
||||
vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce({
|
||||
alertedAt: null,
|
||||
} as never);
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValueOnce({
|
||||
email: "user@example.com",
|
||||
} as never);
|
||||
vi.mocked(prisma.user.findMany).mockResolvedValueOnce([
|
||||
{ id: "admin-1" },
|
||||
] as never);
|
||||
vi.mocked(dispatchNotification).mockResolvedValueOnce({
|
||||
dispatched: false,
|
||||
channelsAttempted: 0,
|
||||
channelsSucceeded: 0,
|
||||
});
|
||||
|
||||
await recordSyncFailure({
|
||||
userId: "u1",
|
||||
integration: "withings",
|
||||
kind: "transient",
|
||||
message: "Withings refresh error: 503 - upstream service down",
|
||||
errorCode: "503",
|
||||
});
|
||||
|
||||
expect(dispatchNotification).toHaveBeenCalledOnce();
|
||||
expect(prisma.integrationStatus.update).not.toHaveBeenCalled();
|
||||
expect(addWarningMock).toHaveBeenCalledWith(
|
||||
"No admin notification channel available for withings failure alert",
|
||||
);
|
||||
});
|
||||
|
||||
it("stamps alertedAt after an attempted delivery that failed", async () => {
|
||||
vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({
|
||||
consecutiveFailuresByKind: {
|
||||
transient: 2,
|
||||
reauth_required: 0,
|
||||
persistent: 0,
|
||||
},
|
||||
persistentFailureStartedAt: null,
|
||||
alertedAt: null,
|
||||
} as never);
|
||||
vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce({
|
||||
alertedAt: null,
|
||||
} as never);
|
||||
vi.mocked(prisma.user.findUnique).mockResolvedValueOnce({
|
||||
email: "user@example.com",
|
||||
} as never);
|
||||
vi.mocked(prisma.user.findMany).mockResolvedValueOnce([
|
||||
{ id: "admin-1" },
|
||||
] as never);
|
||||
vi.mocked(dispatchNotification).mockResolvedValueOnce({
|
||||
dispatched: false,
|
||||
channelsAttempted: 1,
|
||||
channelsSucceeded: 0,
|
||||
});
|
||||
vi.mocked(prisma.integrationStatus.update).mockResolvedValueOnce(
|
||||
{} as never,
|
||||
);
|
||||
|
||||
await recordSyncFailure({
|
||||
userId: "u1",
|
||||
integration: "withings",
|
||||
kind: "transient",
|
||||
message: "Withings refresh error: 503 - upstream service down",
|
||||
errorCode: "503",
|
||||
});
|
||||
|
||||
expect(prisma.integrationStatus.update).toHaveBeenCalledWith({
|
||||
where: {
|
||||
userId_integration: { userId: "u1", integration: "withings" },
|
||||
},
|
||||
data: { alertedAt: expect.any(Date) },
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the custom threshold from INTEGRATION_FAILURE_ALERT_THRESHOLD", async () => {
|
||||
process.env.INTEGRATION_FAILURE_ALERT_THRESHOLD = "5";
|
||||
// 4 < 5 → should not page yet. Seed the existing bucket at 3 so
|
||||
@@ -398,8 +496,8 @@ describe("recordSyncFailure — persistent (v1.4.42 W6)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordSyncFailure — admin Telegram skipped silently when no admins", () => {
|
||||
it("logs a wide-event warning but does NOT throw when no admin users exist", async () => {
|
||||
describe("recordSyncFailure — no admin recipients", () => {
|
||||
it("surfaces the unavailable delivery and leaves alertedAt unstamped", async () => {
|
||||
// Seed bucket at 2 so the in-memory increment hits the threshold
|
||||
// and the no-admins branch is the one being exercised.
|
||||
vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({
|
||||
@@ -418,9 +516,6 @@ describe("recordSyncFailure — admin Telegram skipped silently when no admins",
|
||||
email: "u@example.com",
|
||||
} as never);
|
||||
vi.mocked(prisma.user.findMany).mockResolvedValueOnce([] as never);
|
||||
vi.mocked(prisma.integrationStatus.update).mockResolvedValueOnce(
|
||||
{} as never,
|
||||
);
|
||||
|
||||
await expect(
|
||||
recordSyncFailure({
|
||||
@@ -432,6 +527,10 @@ describe("recordSyncFailure — admin Telegram skipped silently when no admins",
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(dispatchNotification).not.toHaveBeenCalled();
|
||||
expect(prisma.integrationStatus.update).not.toHaveBeenCalled();
|
||||
expect(addWarningMock).toHaveBeenCalledWith(
|
||||
"No admin notification channel available for withings failure alert",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -35,7 +35,10 @@ import { prisma } from "@/lib/db";
|
||||
import { encrypt, decrypt } from "@/lib/crypto";
|
||||
import { auditLog } from "@/lib/auth/audit";
|
||||
import { annotate, getEvent } from "@/lib/logging/context";
|
||||
import { dispatchNotification } from "@/lib/notifications/dispatcher";
|
||||
import {
|
||||
dispatchNotification,
|
||||
type DispatchOutcome,
|
||||
} from "@/lib/notifications/dispatcher";
|
||||
import type { IntegrationClassification } from "@/lib/integrations/http-status-classifier";
|
||||
|
||||
export type IntegrationKey =
|
||||
@@ -755,7 +758,7 @@ export async function recordSyncFailure(
|
||||
row.alertedAt &&
|
||||
now.getTime() - row.alertedAt.getTime() < ALERT_REPEAT_WINDOW_MS;
|
||||
if (!previouslyAlerted) {
|
||||
await maybeAlertAdmins({
|
||||
const outcome = await maybeAlertAdmins({
|
||||
userId,
|
||||
integration,
|
||||
kind: effectiveKind,
|
||||
@@ -766,14 +769,25 @@ export async function recordSyncFailure(
|
||||
getEvent()?.addWarning(
|
||||
`Admin alert dispatch failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return null;
|
||||
});
|
||||
// Stamp the alert window even if dispatch failed — better to
|
||||
// miss one notification than to flood admins on a backend
|
||||
// outage.
|
||||
await prisma.integrationStatus.update({
|
||||
where: { userId_integration: { userId, integration } },
|
||||
data: { alertedAt: now },
|
||||
});
|
||||
|
||||
if (outcome?.channelsAttempted) {
|
||||
// An attempted channel owns its own retry/cooldown policy. Stamp the
|
||||
// alert window even when every attempt failed so a backend outage
|
||||
// cannot turn this failure streak into an hourly notification flood.
|
||||
await prisma.integrationStatus.update({
|
||||
where: { userId_integration: { userId, integration } },
|
||||
data: { alertedAt: now },
|
||||
});
|
||||
} else if (outcome) {
|
||||
// No recipient channel accepted an attempt. Leaving alertedAt empty
|
||||
// keeps the missing delivery visible instead of recording a page that
|
||||
// never left the process.
|
||||
getEvent()?.addWarning(
|
||||
`No admin notification channel available for ${integration} failure alert`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1203,18 +1217,15 @@ export function formatAdminAlertPayload(input: AlertInput): {
|
||||
}
|
||||
|
||||
/**
|
||||
* Page admins on Telegram via the existing dispatcher. We do NOT add
|
||||
* a new sender, channel type, or retry path — B3 owns that surface.
|
||||
* The dispatcher is opt-in per-event and per-channel, so an admin
|
||||
* who has not configured Telegram simply gets no message (silent
|
||||
* fall-through is the documented behaviour).
|
||||
* Page admins through the existing dispatcher. We do NOT add a new sender,
|
||||
* channel type, or retry path. The dispatcher is opt-in per event and channel,
|
||||
* and its outcome is the authority on whether a delivery was attempted.
|
||||
*
|
||||
* The notification is sent to EACH admin user with a Telegram channel
|
||||
* configured. We resolve recipients once per failure burst (gated by
|
||||
* `alertedAt` upstream) so a deployment with a single admin doesn't
|
||||
* pay for a query per user.
|
||||
* The notification is sent to each admin user. The returned outcome aggregates
|
||||
* the dispatcher's real per-admin outcomes so one attempted delivery cannot be
|
||||
* hidden by another admin having no enabled channel.
|
||||
*/
|
||||
async function maybeAlertAdmins(input: AlertInput): Promise<void> {
|
||||
async function maybeAlertAdmins(input: AlertInput): Promise<DispatchOutcome> {
|
||||
const subject = await prisma.user.findUnique({
|
||||
where: { id: input.userId },
|
||||
select: { email: true },
|
||||
@@ -1226,24 +1237,35 @@ async function maybeAlertAdmins(input: AlertInput): Promise<void> {
|
||||
});
|
||||
|
||||
if (admins.length === 0) {
|
||||
getEvent()?.addWarning(
|
||||
`No admin user found to alert about persistent ${input.integration} failure for user ${input.userId}`,
|
||||
);
|
||||
return;
|
||||
return {
|
||||
dispatched: false,
|
||||
channelsAttempted: 0,
|
||||
channelsSucceeded: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const payload = formatAdminAlertPayload({
|
||||
...input,
|
||||
subjectLabel: subject?.email ?? input.userId,
|
||||
});
|
||||
const outcome: DispatchOutcome = {
|
||||
dispatched: false,
|
||||
channelsAttempted: 0,
|
||||
channelsSucceeded: 0,
|
||||
};
|
||||
|
||||
for (const admin of admins) {
|
||||
await dispatchNotification({
|
||||
const adminOutcome = await dispatchNotification({
|
||||
eventType: "SYSTEM_ALERT",
|
||||
userId: admin.id,
|
||||
title: payload.title,
|
||||
message: payload.message,
|
||||
metadata: payload.metadata,
|
||||
});
|
||||
outcome.dispatched ||= adminOutcome.dispatched;
|
||||
outcome.channelsAttempted += adminOutcome.channelsAttempted;
|
||||
outcome.channelsSucceeded += adminOutcome.channelsSucceeded;
|
||||
}
|
||||
|
||||
return outcome;
|
||||
}
|
||||
Reference in new issue
Block a user