From 346825149d92fd94cd90d72479235f021efd95d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Bombeck?= Date: Thu, 15 Oct 2026 16:42:41 +0200 Subject: [PATCH] fix(integrations): send no recovery for an incident the first run recorded silently The watchdog's first pass after a deploy records standing incidents as announced without paging. Their end then sent a recovery note for something the operator was never told about. Such incidents now carry the epoch as their announcement time, which no real announcement can have, and close without a note. --- .../__tests__/watchdog-classify.test.ts | 22 +++++++++++++++++++ src/lib/integrations/watchdog-classify.ts | 18 ++++++++++++++- src/lib/integrations/watchdog.ts | 6 +++-- .../integration/integration-watchdog.test.ts | 18 +++++++++++++++ 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/lib/integrations/__tests__/watchdog-classify.test.ts b/src/lib/integrations/__tests__/watchdog-classify.test.ts index 9de4196db..35dd48473 100644 --- a/src/lib/integrations/__tests__/watchdog-classify.test.ts +++ b/src/lib/integrations/__tests__/watchdog-classify.test.ts @@ -4,6 +4,7 @@ import { NORMAL_DELIVERY_MIN_DAYS, SOURCE_PROFILES, classifySilence, + FIRST_RUN_ALERTED_AT, currentIncident, decideWatchTransition, type WatchRow, @@ -293,6 +294,27 @@ describe("currentIncident", () => { }); }); +describe("decideWatchTransition: an incident recorded on the first run", () => { + it("closes without a recovery note, since none was ever sent", () => { + expect( + decideWatchTransition({ + row: { + silentSince: ago(30), + incidentReason: "unexpected_silence", + incidentOpenedAt: ago(2), + alertedAt: FIRST_RUN_ALERTED_AT, + }, + incident: null, + ledgerAlerted: false, + }), + ).toEqual({ + kind: "close", + notify: false, + previousReason: "unexpected_silence", + }); + }); +}); + describe("decideWatchTransition", () => { const incident = { reason: "unexpected_silence" as const, since: ago(30) }; diff --git a/src/lib/integrations/watchdog-classify.ts b/src/lib/integrations/watchdog-classify.ts index 73f27aa2f..8b10705e3 100644 --- a/src/lib/integrations/watchdog-classify.ts +++ b/src/lib/integrations/watchdog-classify.ts @@ -189,6 +189,22 @@ export function currentIncident(input: { return null; } +/** + * The `alertedAt` an incident gets when the watchdog's first pass after a + * deploy found it already standing: counted as announced, so it is never + * paged, but never actually sent either, so its end sends no recovery note. + * The epoch, because no real announcement can carry it, and it keeps the + * marker inside the existing row. + */ +export const FIRST_RUN_ALERTED_AT = new Date(0); + +/** Whether an `alertedAt` records a message that was really sent. */ +export function wasAnnounced(alertedAt: Date | null | undefined): boolean { + return ( + alertedAt != null && alertedAt.getTime() !== FIRST_RUN_ALERTED_AT.getTime() + ); +} + export type WatchTransition = /** No incident before, none now. */ | { kind: "none" } @@ -220,7 +236,7 @@ export function decideWatchTransition(input: { if (input.incident === null) { return { kind: "close", - notify: input.row?.alertedAt != null, + notify: wasAnnounced(input.row?.alertedAt), previousReason: open, }; } diff --git a/src/lib/integrations/watchdog.ts b/src/lib/integrations/watchdog.ts index 0cbdcd64d..797087652 100644 --- a/src/lib/integrations/watchdog.ts +++ b/src/lib/integrations/watchdog.ts @@ -31,6 +31,7 @@ import { } from "./connection-health-load"; import { INTEGRATION_MEASUREMENT_SOURCE } from "./metric-freshness"; import { + FIRST_RUN_ALERTED_AT, HISTORY_DAYS, SOURCE_PROFILES, classifySilence, @@ -529,8 +530,9 @@ async function watchConnection(input: { incidentOpenedAt = now; if (summary.firstRun) { // Standing since before the watchdog first looked: recorded as - // announced, not paged (see `WatchdogSummary.firstRun`). - alertedAt = now; + // announced, not paged (see `WatchdogSummary.firstRun`), and marked + // so its end sends no recovery note either. + alertedAt = FIRST_RUN_ALERTED_AT; } else { announce(transition.reason, transition.since); } diff --git a/tests/integration/integration-watchdog.test.ts b/tests/integration/integration-watchdog.test.ts index e6884d876..777e3ce80 100644 --- a/tests/integration/integration-watchdog.test.ts +++ b/tests/integration/integration-watchdog.test.ts @@ -333,6 +333,24 @@ describe("what pages and what does not", () => { expect(second.firstRun).toBe(false); expect(dispatchNotification).not.toHaveBeenCalled(); + // It ends without a recovery note: nothing was ever sent about it. + await prisma().measurement.create({ + data: { + userId: USER, + type: "RESTING_HEART_RATE", + value: 61, + unit: "bpm", + source: "OURA", + measuredAt: new Date(later.getTime() - 10 * 60 * 1000), + }, + }); + const recovered = await runIntegrationWatchdog({ + mode: "hourly", + now: new Date(later.getTime() + HOUR), + }); + expect(recovered.incidentsClosed).toBe(1); + expect(dispatchNotification).not.toHaveBeenCalled(); + // A new incident after the first pass pages as usual. for (let i = 0; i < 3; i++) { await recordSyncFailure({