From 0e5afe76a44b4fc888d10d22b62a7e94f3dffc45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 15 Oct 2026 16:15:32 +0200 Subject: [PATCH] fix(integrations): skip the ledger page when the watchdog announced it A failure streak reaching the alert threshold paged the operator even when the watchdog had already announced an open incident for the same connection. The ledger now checks for that announced incident first, stamps the streak as announced and sends nothing; the watchdog's recovery note still ends it. --- src/lib/integrations/__tests__/status.test.ts | 47 +++++++++++++++++++ src/lib/integrations/status.ts | 28 ++++++++++- .../integration/integration-watchdog.test.ts | 29 ++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/src/lib/integrations/__tests__/status.test.ts b/src/lib/integrations/__tests__/status.test.ts index a4d365a01..0c0ea8351 100644 --- a/src/lib/integrations/__tests__/status.test.ts +++ b/src/lib/integrations/__tests__/status.test.ts @@ -30,6 +30,9 @@ vi.mock("@/lib/db", () => ({ findUnique: vi.fn(), findMany: vi.fn(), }, + integrationWatch: { + findFirst: vi.fn(), + }, }, })); @@ -161,6 +164,50 @@ describe("recordSyncFailure — under threshold", () => { }); describe("recordSyncFailure — at threshold", () => { + it("does not page a streak the watchdog already announced for the same connection", async () => { + vi.mocked(prisma.integrationStatus.findUnique).mockResolvedValueOnce({ + consecutiveFailuresByKind: { + transient: 2, + reauth_required: 0, + persistent: 0, + }, + persistentFailureStartedAt: null, + alertedAt: null, + } as never); + vi.mocked(prisma.integrationStatus.upsert).mockResolvedValueOnce({ + alertedAt: null, + } as never); + vi.mocked(prisma.integrationWatch.findFirst).mockResolvedValueOnce({ + id: "watch-1", + } as never); + + await recordSyncFailure({ + userId: "u1", + integration: "withings", + kind: "transient", + message: "Withings refresh error: 503", + errorCode: "503", + }); + + expect(prisma.integrationWatch.findFirst).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + userId: "u1", + integration: "withings", + incidentReason: { not: null }, + alertedAt: { not: null }, + }), + }), + ); + expect(dispatchNotification).not.toHaveBeenCalled(); + // Stamped as announced, so later failures of the streak do not ask again. + expect(prisma.integrationStatus.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: { alertedAt: expect.any(Date) }, + }), + ); + }); + it("dispatches an admin Telegram alert when failure count reaches default threshold (3)", async () => { // v1.4.47 W1 — the alert ladder reads Math.max(...buckets) so the // existing row must seed the bucket at N-1 before the in-memory diff --git a/src/lib/integrations/status.ts b/src/lib/integrations/status.ts index 57bac23f8..d3f0675cb 100644 --- a/src/lib/integrations/status.ts +++ b/src/lib/integrations/status.ts @@ -798,7 +798,33 @@ export async function recordSyncFailure( const alertSignal = Math.max(...Object.values(buckets)); if (alertSignal >= threshold) { const previouslyAlerted = row.alertedAt != null; - if (!previouslyAlerted) { + // The watchdog may have announced this connection already (it pages a + // day-long provider error streak, a reauth or a silence on its own). Its + // open incident covers this streak too, and its recovery note ends it; + // a second page about the same trouble would only repeat it. + const watchdogAnnounced = + !previouslyAlerted && + !!(await prisma.integrationWatch.findFirst({ + where: { + userId, + integration, + incidentReason: { not: null }, + alertedAt: { not: null }, + }, + select: { id: true }, + })); + if (watchdogAnnounced) { + await prisma.integrationStatus.update({ + where: { userId_integration: { userId, integration } }, + data: { alertedAt: now }, + }); + annotate({ + action: { + name: "integrations.alert.covered_by_watchdog", + details: { integration }, + }, + }); + } else if (!previouslyAlerted) { const outcome = await maybeAlertAdmins({ userId, integration, diff --git a/tests/integration/integration-watchdog.test.ts b/tests/integration/integration-watchdog.test.ts index 0ef329cc8..1c616f03f 100644 --- a/tests/integration/integration-watchdog.test.ts +++ b/tests/integration/integration-watchdog.test.ts @@ -361,6 +361,35 @@ describe("what pages and what does not", () => { }); }); +describe("the ledger and the watchdog page once between them", () => { + it("does not page a failure streak the watchdog already announced for the connection", async () => { + await prisma().integrationWatch.create({ + data: { + userId: USER, + integration: "oura", + silentSince: new Date(NOW.getTime() - 3 * DAY), + incidentReason: "unexpected_silence", + incidentOpenedAt: new Date(NOW.getTime() - DAY), + alertedAt: new Date(NOW.getTime() - DAY), + }, + }); + for (let i = 0; i < 3; i++) { + await recordSyncFailure({ + userId: USER, + integration: "oura", + kind: "transient", + message: "Oura error: 503", + errorCode: "503", + }); + } + expect(dispatchNotification).not.toHaveBeenCalled(); + const ledger = await prisma().integrationStatus.findUniqueOrThrow({ + where: { userId_integration: { userId: USER, integration: "oura" } }, + }); + expect(ledger.alertedAt).not.toBeNull(); + }); +}); + describe("recent-call ring", () => { it("keeps the newest ten attempts, newest first, without any message text", async () => { for (let i = 0; i < 7; i++) {